diff options
| author | Aiden Grossman <aidengrossman@google.com> | 2025-09-12 01:05:48 +0000 |
|---|---|---|
| committer | Aiden Grossman <aidengrossman@google.com> | 2025-09-12 01:05:48 +0000 |
| commit | 88a52e1fc6d3e153132f0e0a86431762adf8c0c4 (patch) | |
| tree | bae8c1b720736edc54705c325c5bfb95b459eda2 /llvm/lib | |
| parent | 1873dd7e8bb03319500a9f4b51e9e498a8fb70de (diff) | |
| parent | 2740e4b73682eb7a6869c333991a608304938952 (diff) | |
[𝘀𝗽𝗿] changes introduced through rebaseusers/boomanaiden154/main.clang-invoke-shell-script-with-bash
Created using spr 1.3.6
[skip ci]
Diffstat (limited to 'llvm/lib')
214 files changed, 7543 insertions, 3072 deletions
diff --git a/llvm/lib/Analysis/ConstantFolding.cpp b/llvm/lib/Analysis/ConstantFolding.cpp index 40e176c2ab5c..a3b2e62a1b8b 100644..100755 --- a/llvm/lib/Analysis/ConstantFolding.cpp +++ b/llvm/lib/Analysis/ConstantFolding.cpp @@ -1652,6 +1652,13 @@ bool llvm::canConstantFoldCallTo(const CallBase *Call, const Function *F) { case Intrinsic::amdgcn_perm: case Intrinsic::amdgcn_wave_reduce_umin: case Intrinsic::amdgcn_wave_reduce_umax: + case Intrinsic::amdgcn_wave_reduce_max: + case Intrinsic::amdgcn_wave_reduce_min: + case Intrinsic::amdgcn_wave_reduce_add: + case Intrinsic::amdgcn_wave_reduce_sub: + case Intrinsic::amdgcn_wave_reduce_and: + case Intrinsic::amdgcn_wave_reduce_or: + case Intrinsic::amdgcn_wave_reduce_xor: case Intrinsic::amdgcn_s_wqm: case Intrinsic::amdgcn_s_quadmask: case Intrinsic::amdgcn_s_bitreplicate: @@ -3672,6 +3679,13 @@ static Constant *ConstantFoldIntrinsicCall2(Intrinsic::ID IntrinsicID, Type *Ty, return ConstantInt::get(Ty, C0->abs()); case Intrinsic::amdgcn_wave_reduce_umin: case Intrinsic::amdgcn_wave_reduce_umax: + case Intrinsic::amdgcn_wave_reduce_max: + case Intrinsic::amdgcn_wave_reduce_min: + case Intrinsic::amdgcn_wave_reduce_add: + case Intrinsic::amdgcn_wave_reduce_sub: + case Intrinsic::amdgcn_wave_reduce_and: + case Intrinsic::amdgcn_wave_reduce_or: + case Intrinsic::amdgcn_wave_reduce_xor: return dyn_cast<Constant>(Operands[0]); } @@ -4238,6 +4252,13 @@ static Constant *ConstantFoldScalableVectorCall( return ConstantInt::getFalse(SVTy); } + case Intrinsic::get_active_lane_mask: { + auto *Op0 = dyn_cast<ConstantInt>(Operands[0]); + auto *Op1 = dyn_cast<ConstantInt>(Operands[1]); + if (Op0 && Op1 && Op0->getValue().uge(Op1->getValue())) + return ConstantVector::getNullValue(SVTy); + break; + } default: break; } diff --git a/llvm/lib/Analysis/LazyValueInfo.cpp b/llvm/lib/Analysis/LazyValueInfo.cpp index c7b0ca97a8e4..90bae77bcf70 100644 --- a/llvm/lib/Analysis/LazyValueInfo.cpp +++ b/llvm/lib/Analysis/LazyValueInfo.cpp @@ -1493,6 +1493,24 @@ LazyValueInfoImpl::getEdgeValueLocal(Value *Val, BasicBlock *BBFrom, // br %Condition, label %then, label %else APInt ConditionVal(1, isTrueDest ? 1 : 0); Result = constantFoldUser(Usr, Condition, ConditionVal, DL); + } else if (isa<TruncInst, ZExtInst, SExtInst>(Usr)) { + ValueLatticeElement OpLatticeVal = + *getValueFromCondition(Usr->getOperand(0), Condition, + isTrueDest, /*UseBlockValue*/ false); + + if (!OpLatticeVal.isConstantRange()) + return OpLatticeVal; + + const unsigned ResultBitWidth = + Usr->getType()->getScalarSizeInBits(); + if (auto *Trunc = dyn_cast<TruncInst>(Usr)) + return ValueLatticeElement::getRange( + OpLatticeVal.getConstantRange().truncate( + ResultBitWidth, Trunc->getNoWrapKind())); + + return ValueLatticeElement::getRange( + OpLatticeVal.getConstantRange().castOp( + cast<CastInst>(Usr)->getOpcode(), ResultBitWidth)); } else { // If one of Val's operand has an inferred value, we may be able to // infer the value of Val. diff --git a/llvm/lib/Analysis/Lint.cpp b/llvm/lib/Analysis/Lint.cpp index 1168005f48c0..32a4264c0343 100644 --- a/llvm/lib/Analysis/Lint.cpp +++ b/llvm/lib/Analysis/Lint.cpp @@ -374,13 +374,6 @@ void Lint::visitCallBase(CallBase &I) { visitMemoryReference(I, MemoryLocation::getForArgument(&I, 0, TLI), std::nullopt, nullptr, MemRef::Read | MemRef::Write); break; - case Intrinsic::get_active_lane_mask: - if (auto *TripCount = dyn_cast<ConstantInt>(I.getArgOperand(1))) - Check(!TripCount->isZero(), - "get_active_lane_mask: operand #2 " - "must be greater than 0", - &I); - break; } } diff --git a/llvm/lib/Analysis/Loads.cpp b/llvm/lib/Analysis/Loads.cpp index 47fdb41e8b44..0c4e3a2e3b23 100644 --- a/llvm/lib/Analysis/Loads.cpp +++ b/llvm/lib/Analysis/Loads.cpp @@ -21,6 +21,7 @@ #include "llvm/Analysis/ScalarEvolutionExpressions.h" #include "llvm/Analysis/ValueTracking.h" #include "llvm/IR/DataLayout.h" +#include "llvm/IR/GetElementPtrTypeIterator.h" #include "llvm/IR/IntrinsicInst.h" #include "llvm/IR/Operator.h" @@ -876,3 +877,66 @@ bool llvm::isReadOnlyLoop( } return true; } + +LinearExpression llvm::decomposeLinearExpression(const DataLayout &DL, + Value *Ptr) { + assert(Ptr->getType()->isPointerTy() && "Must be called with pointer arg"); + + unsigned BitWidth = DL.getIndexTypeSizeInBits(Ptr->getType()); + LinearExpression Expr(Ptr, BitWidth); + + while (true) { + auto *GEP = dyn_cast<GEPOperator>(Expr.BasePtr); + if (!GEP || GEP->getSourceElementType()->isScalableTy()) + return Expr; + + Value *VarIndex = nullptr; + for (Value *Index : GEP->indices()) { + if (isa<ConstantInt>(Index)) + continue; + // Only allow a single variable index. We do not bother to handle the + // case of the same variable index appearing multiple times. + if (Expr.Index || VarIndex) + return Expr; + VarIndex = Index; + } + + // Don't return non-canonical indexes. + if (VarIndex && !VarIndex->getType()->isIntegerTy(BitWidth)) + return Expr; + + // We have verified that we can fully handle this GEP, so we can update Expr + // members past this point. + Expr.BasePtr = GEP->getPointerOperand(); + Expr.Flags = Expr.Flags.intersectForOffsetAdd(GEP->getNoWrapFlags()); + for (gep_type_iterator GTI = gep_type_begin(GEP), GTE = gep_type_end(GEP); + GTI != GTE; ++GTI) { + Value *Index = GTI.getOperand(); + if (auto *ConstOffset = dyn_cast<ConstantInt>(Index)) { + if (ConstOffset->isZero()) + continue; + if (StructType *STy = GTI.getStructTypeOrNull()) { + unsigned ElementIdx = ConstOffset->getZExtValue(); + const StructLayout *SL = DL.getStructLayout(STy); + Expr.Offset += SL->getElementOffset(ElementIdx); + continue; + } + // Truncate if type size exceeds index space. + APInt IndexedSize(BitWidth, GTI.getSequentialElementStride(DL), + /*isSigned=*/false, + /*implcitTrunc=*/true); + Expr.Offset += ConstOffset->getValue() * IndexedSize; + continue; + } + + // FIXME: Also look through a mul/shl in the index. + assert(Expr.Index == nullptr && "Shouldn't have index yet"); + Expr.Index = Index; + // Truncate if type size exceeds index space. + Expr.Scale = APInt(BitWidth, GTI.getSequentialElementStride(DL), + /*isSigned=*/false, /*implicitTrunc=*/true); + } + } + + return Expr; +} diff --git a/llvm/lib/Analysis/LoopAccessAnalysis.cpp b/llvm/lib/Analysis/LoopAccessAnalysis.cpp index cfddd9980c86..87fae92977cd 100644 --- a/llvm/lib/Analysis/LoopAccessAnalysis.cpp +++ b/llvm/lib/Analysis/LoopAccessAnalysis.cpp @@ -1231,48 +1231,40 @@ static void findForkedSCEVs( } } -static SmallVector<PointerIntPair<const SCEV *, 1, bool>> -findForkedPointer(PredicatedScalarEvolution &PSE, - const DenseMap<Value *, const SCEV *> &StridesMap, Value *Ptr, - const Loop *L) { - ScalarEvolution *SE = PSE.getSE(); - assert(SE->isSCEVable(Ptr->getType()) && "Value is not SCEVable!"); - SmallVector<PointerIntPair<const SCEV *, 1, bool>> Scevs; - findForkedSCEVs(SE, L, Ptr, Scevs, MaxForkedSCEVDepth); - - // For now, we will only accept a forked pointer with two possible SCEVs - // that are either SCEVAddRecExprs or loop invariant. - if (Scevs.size() == 2 && - (isa<SCEVAddRecExpr>(get<0>(Scevs[0])) || - SE->isLoopInvariant(get<0>(Scevs[0]), L)) && - (isa<SCEVAddRecExpr>(get<0>(Scevs[1])) || - SE->isLoopInvariant(get<0>(Scevs[1]), L))) { - LLVM_DEBUG(dbgs() << "LAA: Found forked pointer: " << *Ptr << "\n"); - LLVM_DEBUG(dbgs() << "\t(1) " << *get<0>(Scevs[0]) << "\n"); - LLVM_DEBUG(dbgs() << "\t(2) " << *get<0>(Scevs[1]) << "\n"); - return Scevs; - } - - return {{replaceSymbolicStrideSCEV(PSE, StridesMap, Ptr), false}}; -} - bool AccessAnalysis::createCheckForAccess( RuntimePointerChecking &RtCheck, MemAccessInfo Access, Type *AccessTy, const DenseMap<Value *, const SCEV *> &StridesMap, DenseMap<Value *, unsigned> &DepSetId, Loop *TheLoop, unsigned &RunningDepId, unsigned ASId, bool Assume) { Value *Ptr = Access.getPointer(); + ScalarEvolution *SE = PSE.getSE(); + assert(SE->isSCEVable(Ptr->getType()) && "Value is not SCEVable!"); - SmallVector<PointerIntPair<const SCEV *, 1, bool>> TranslatedPtrs = - findForkedPointer(PSE, StridesMap, Ptr, TheLoop); - assert(!TranslatedPtrs.empty() && "must have some translated pointers"); + SmallVector<PointerIntPair<const SCEV *, 1, bool>> RTCheckPtrs; + findForkedSCEVs(SE, TheLoop, Ptr, RTCheckPtrs, MaxForkedSCEVDepth); + assert(!RTCheckPtrs.empty() && + "Must have some runtime-check pointer candidates"); + + // RTCheckPtrs must have size 2 if there are forked pointers. Otherwise, there + // are no forked pointers; replaceSymbolicStridesSCEV in this case. + auto IsLoopInvariantOrAR = + [&SE, &TheLoop](const PointerIntPair<const SCEV *, 1, bool> &P) { + return SE->isLoopInvariant(P.getPointer(), TheLoop) || + isa<SCEVAddRecExpr>(P.getPointer()); + }; + if (RTCheckPtrs.size() == 2 && all_of(RTCheckPtrs, IsLoopInvariantOrAR)) { + LLVM_DEBUG(dbgs() << "LAA: Found forked pointer: " << *Ptr << "\n"; + for (const auto &[Idx, Q] : enumerate(RTCheckPtrs)) dbgs() + << "\t(" << Idx << ") " << *Q.getPointer() << "\n"); + } else { + RTCheckPtrs = {{replaceSymbolicStrideSCEV(PSE, StridesMap, Ptr), false}}; + } /// Check whether all pointers can participate in a runtime bounds check. They - /// must either be invariant or AddRecs. If ShouldCheckWrap is true, they also - /// must not wrap. - for (auto &P : TranslatedPtrs) { + /// must either be invariant or non-wrapping affine AddRecs. + for (auto &P : RTCheckPtrs) { // The bounds for loop-invariant pointer is trivial. - if (PSE.getSE()->isLoopInvariant(P.getPointer(), TheLoop)) + if (SE->isLoopInvariant(P.getPointer(), TheLoop)) continue; const SCEVAddRecExpr *AR = dyn_cast<SCEVAddRecExpr>(P.getPointer()); @@ -1283,21 +1275,18 @@ bool AccessAnalysis::createCheckForAccess( // If there's only one option for Ptr, look it up after bounds and wrap // checking, because assumptions might have been added to PSE. - if (TranslatedPtrs.size() == 1) { + if (RTCheckPtrs.size() == 1) { AR = cast<SCEVAddRecExpr>(replaceSymbolicStrideSCEV(PSE, StridesMap, Ptr)); P.setPointer(AR); } - // When we run after a failing dependency check we have to make sure - // we don't have wrapping pointers. - if (!isNoWrap(PSE, AR, TranslatedPtrs.size() == 1 ? Ptr : nullptr, AccessTy, - TheLoop, Assume)) { + if (!isNoWrap(PSE, AR, RTCheckPtrs.size() == 1 ? Ptr : nullptr, AccessTy, + TheLoop, Assume)) return false; - } } - for (auto [PtrExpr, NeedsFreeze] : TranslatedPtrs) { + for (const auto &[PtrExpr, NeedsFreeze] : RTCheckPtrs) { // The id of the dependence set. unsigned DepId; diff --git a/llvm/lib/Analysis/ScalarEvolution.cpp b/llvm/lib/Analysis/ScalarEvolution.cpp index bd57d1192eb9..ebb863076d2c 100644 --- a/llvm/lib/Analysis/ScalarEvolution.cpp +++ b/llvm/lib/Analysis/ScalarEvolution.cpp @@ -3216,13 +3216,28 @@ const SCEV *ScalarEvolution::getMulExpr(SmallVectorImpl<const SCEV *> &Ops, }; } - // Try to fold (C * D /u C) -> D, if C is a power-of-2 and D is a multiple - // of C. + // Try to fold (C1 * D /u C2) -> C1/C2 * D, if C1 and C2 are powers-of-2, + // D is a multiple of C2, and C1 is a multiple of C2. If C2 is a multiple + // of C1, fold to (D /u (C2 /u C1)). const SCEV *D; - if (match(Ops[1], m_scev_UDiv(m_SCEV(D), m_scev_Specific(LHSC))) && - LHSC->getAPInt().isPowerOf2() && - LHSC->getAPInt().logBase2() <= getMinTrailingZeros(D)) { - return D; + APInt C1V = LHSC->getAPInt(); + // (C1 * D /u C2) == -1 * -C1 * D /u C2 when C1 != INT_MIN. Don't treat -1 + // as -1 * 1, as it won't enable additional folds. + if (C1V.isNegative() && !C1V.isMinSignedValue() && !C1V.isAllOnes()) + C1V = C1V.abs(); + const SCEVConstant *C2; + if (C1V.isPowerOf2() && + match(Ops[1], m_scev_UDiv(m_SCEV(D), m_SCEVConstant(C2))) && + C2->getAPInt().isPowerOf2() && + C1V.logBase2() <= getMinTrailingZeros(D)) { + const SCEV *NewMul; + if (C1V.uge(C2->getAPInt())) { + NewMul = getMulExpr(getUDivExpr(getConstant(C1V), C2), D); + } else { + assert(C1V.ugt(1) && "C1 <= 1 should have been folded earlier"); + NewMul = getUDivExpr(D, getUDivExpr(C2, getConstant(C1V))); + } + return C1V == LHSC->getAPInt() ? NewMul : getNegativeSCEV(NewMul); } } } diff --git a/llvm/lib/CodeGen/ExpandFp.cpp b/llvm/lib/CodeGen/ExpandFp.cpp index 1fc2277c7169..9cc6c6a706c5 100644 --- a/llvm/lib/CodeGen/ExpandFp.cpp +++ b/llvm/lib/CodeGen/ExpandFp.cpp @@ -1135,7 +1135,7 @@ void ExpandFpPass::printPipeline( static_cast<PassInfoMixin<ExpandFpPass> *>(this)->printPipeline( OS, MapClassName2PassName); OS << '<'; - OS << "opt-level=" << (int)OptLevel; + OS << "O" << (int)OptLevel; OS << '>'; } diff --git a/llvm/lib/CodeGen/ExpandVectorPredication.cpp b/llvm/lib/CodeGen/ExpandVectorPredication.cpp index 673c4f7434d6..17fae8f6c818 100644 --- a/llvm/lib/CodeGen/ExpandVectorPredication.cpp +++ b/llvm/lib/CodeGen/ExpandVectorPredication.cpp @@ -111,6 +111,10 @@ static void transferDecorations(Value &NewVal, VPIntrinsic &VPI) { /// OldVP gets erased. static void replaceOperation(Value &NewOp, VPIntrinsic &OldOp) { transferDecorations(NewOp, OldOp); + + if (isa<Instruction>(NewOp) && !NewOp.hasName() && OldOp.hasName()) + NewOp.takeName(&OldOp); + OldOp.replaceAllUsesWith(&NewOp); OldOp.eraseFromParent(); } @@ -253,7 +257,7 @@ bool CachingVPExpander::expandPredicationInBinaryOperator(IRBuilder<> &Builder, } } - Value *NewBinOp = Builder.CreateBinOp(OC, Op0, Op1, VPI.getName()); + Value *NewBinOp = Builder.CreateBinOp(OC, Op0, Op1); replaceOperation(*NewBinOp, VPI); return true; @@ -268,8 +272,8 @@ bool CachingVPExpander::expandPredicationToIntCall(IRBuilder<> &Builder, for (unsigned i = 0; i < VPI.getNumOperands() - 3; i++) { Argument.push_back(VPI.getOperand(i)); } - Value *NewOp = Builder.CreateIntrinsic(FID.value(), {VPI.getType()}, Argument, - /*FMFSource=*/nullptr, VPI.getName()); + Value *NewOp = + Builder.CreateIntrinsic(FID.value(), {VPI.getType()}, Argument); replaceOperation(*NewOp, VPI); return true; } @@ -288,9 +292,8 @@ bool CachingVPExpander::expandPredicationToFPCall( for (unsigned i = 0; i < VPI.getNumOperands() - 3; i++) { Argument.push_back(VPI.getOperand(i)); } - Value *NewOp = Builder.CreateIntrinsic( - UnpredicatedIntrinsicID, {VPI.getType()}, Argument, - /*FMFSource=*/nullptr, VPI.getName()); + Value *NewOp = Builder.CreateIntrinsic(UnpredicatedIntrinsicID, + {VPI.getType()}, Argument); replaceOperation(*NewOp, VPI); return true; } @@ -305,10 +308,9 @@ bool CachingVPExpander::expandPredicationToFPCall( VPI.getModule(), UnpredicatedIntrinsicID, {VPI.getType()}); Value *NewOp; if (Intrinsic::isConstrainedFPIntrinsic(UnpredicatedIntrinsicID)) - NewOp = - Builder.CreateConstrainedFPCall(Fn, {Op0, Op1, Op2}, VPI.getName()); + NewOp = Builder.CreateConstrainedFPCall(Fn, {Op0, Op1, Op2}); else - NewOp = Builder.CreateCall(Fn, {Op0, Op1, Op2}, VPI.getName()); + NewOp = Builder.CreateCall(Fn, {Op0, Op1, Op2}); replaceOperation(*NewOp, VPI); return true; } @@ -393,9 +395,8 @@ bool CachingVPExpander::expandPredicationToCastIntrinsic(IRBuilder<> &Builder, Intrinsic::ID VPID = VPI.getIntrinsicID(); unsigned CastOpcode = VPIntrinsic::getFunctionalOpcodeForVP(VPID).value(); assert(Instruction::isCast(CastOpcode)); - Value *CastOp = - Builder.CreateCast(Instruction::CastOps(CastOpcode), VPI.getOperand(0), - VPI.getType(), VPI.getName()); + Value *CastOp = Builder.CreateCast(Instruction::CastOps(CastOpcode), + VPI.getOperand(0), VPI.getType()); replaceOperation(*CastOp, VPI); return true; @@ -454,8 +455,8 @@ bool CachingVPExpander::expandPredicationInMemoryIntrinsic(IRBuilder<> &Builder, auto *ElementType = cast<VectorType>(VPI.getType())->getElementType(); NewMemoryInst = Builder.CreateMaskedGather( VPI.getType(), PtrParam, - AlignOpt.value_or(DL.getPrefTypeAlign(ElementType)), MaskParam, nullptr, - VPI.getName()); + AlignOpt.value_or(DL.getPrefTypeAlign(ElementType)), MaskParam, + nullptr); break; } } @@ -521,7 +522,8 @@ bool CachingVPExpander::foldEVLIntoMask(VPIntrinsic &VPI) { // Only VP intrinsics can have an %evl parameter. Value *OldMaskParam = VPI.getMaskParam(); if (!OldMaskParam) { - assert(VPI.getIntrinsicID() == Intrinsic::vp_merge && + assert((VPI.getIntrinsicID() == Intrinsic::vp_merge || + VPI.getIntrinsicID() == Intrinsic::vp_select) && "Unexpected VP intrinsic without mask operand"); OldMaskParam = VPI.getArgOperand(0); } @@ -537,7 +539,8 @@ bool CachingVPExpander::foldEVLIntoMask(VPIntrinsic &VPI) { ElementCount ElemCount = VPI.getStaticVectorLength(); Value *VLMask = convertEVLToMask(Builder, OldEVLParam, ElemCount); Value *NewMaskParam = Builder.CreateAnd(VLMask, OldMaskParam); - if (VPI.getIntrinsicID() == Intrinsic::vp_merge) + if (VPI.getIntrinsicID() == Intrinsic::vp_merge || + VPI.getIntrinsicID() == Intrinsic::vp_select) VPI.setArgOperand(0, NewMaskParam); else VPI.setMaskParam(NewMaskParam); @@ -575,10 +578,18 @@ bool CachingVPExpander::expandPredication(VPIntrinsic &VPI) { default: break; case Intrinsic::vp_fneg: { - Value *NewNegOp = Builder.CreateFNeg(VPI.getOperand(0), VPI.getName()); + Value *NewNegOp = Builder.CreateFNeg(VPI.getOperand(0)); replaceOperation(*NewNegOp, VPI); return NewNegOp; } + case Intrinsic::vp_select: + case Intrinsic::vp_merge: { + assert(maySpeculateLanes(VPI) || VPI.canIgnoreVectorLengthParam()); + Value *NewSelectOp = Builder.CreateSelect( + VPI.getOperand(0), VPI.getOperand(1), VPI.getOperand(2)); + replaceOperation(*NewSelectOp, VPI); + return NewSelectOp; + } case Intrinsic::vp_abs: case Intrinsic::vp_smax: case Intrinsic::vp_smin: diff --git a/llvm/lib/CodeGen/GlobalISel/CombinerHelper.cpp b/llvm/lib/CodeGen/GlobalISel/CombinerHelper.cpp index 0674f5fd1ae0..0ebee2cfd868 100644 --- a/llvm/lib/CodeGen/GlobalISel/CombinerHelper.cpp +++ b/llvm/lib/CodeGen/GlobalISel/CombinerHelper.cpp @@ -2094,6 +2094,68 @@ bool CombinerHelper::matchCommuteShift(MachineInstr &MI, return true; } +bool CombinerHelper::matchLshrOfTruncOfLshr(MachineInstr &MI, + LshrOfTruncOfLshr &MatchInfo, + MachineInstr &ShiftMI) const { + assert(MI.getOpcode() == TargetOpcode::G_LSHR && "Expected a G_LSHR"); + + Register N0 = MI.getOperand(1).getReg(); + Register N1 = MI.getOperand(2).getReg(); + unsigned OpSizeInBits = MRI.getType(N0).getScalarSizeInBits(); + + APInt N1C, N001C; + if (!mi_match(N1, MRI, m_ICstOrSplat(N1C))) + return false; + auto N001 = ShiftMI.getOperand(2).getReg(); + if (!mi_match(N001, MRI, m_ICstOrSplat(N001C))) + return false; + + if (N001C.getBitWidth() > N1C.getBitWidth()) + N1C = N1C.zext(N001C.getBitWidth()); + else + N001C = N001C.zext(N1C.getBitWidth()); + + Register InnerShift = ShiftMI.getOperand(0).getReg(); + LLT InnerShiftTy = MRI.getType(InnerShift); + uint64_t InnerShiftSize = InnerShiftTy.getScalarSizeInBits(); + if ((N1C + N001C).ult(InnerShiftSize)) { + MatchInfo.Src = ShiftMI.getOperand(1).getReg(); + MatchInfo.ShiftAmt = N1C + N001C; + MatchInfo.ShiftAmtTy = MRI.getType(N001); + MatchInfo.InnerShiftTy = InnerShiftTy; + + if ((N001C + OpSizeInBits) == InnerShiftSize) + return true; + if (MRI.hasOneUse(N0) && MRI.hasOneUse(InnerShift)) { + MatchInfo.Mask = true; + MatchInfo.MaskVal = APInt(N1C.getBitWidth(), OpSizeInBits) - N1C; + return true; + } + } + return false; +} + +void CombinerHelper::applyLshrOfTruncOfLshr( + MachineInstr &MI, LshrOfTruncOfLshr &MatchInfo) const { + assert(MI.getOpcode() == TargetOpcode::G_LSHR && "Expected a G_LSHR"); + + Register Dst = MI.getOperand(0).getReg(); + auto ShiftAmt = + Builder.buildConstant(MatchInfo.ShiftAmtTy, MatchInfo.ShiftAmt); + auto Shift = + Builder.buildLShr(MatchInfo.InnerShiftTy, MatchInfo.Src, ShiftAmt); + if (MatchInfo.Mask == true) { + APInt MaskVal = + APInt::getLowBitsSet(MatchInfo.InnerShiftTy.getScalarSizeInBits(), + MatchInfo.MaskVal.getZExtValue()); + auto Mask = Builder.buildConstant(MatchInfo.InnerShiftTy, MaskVal); + auto And = Builder.buildAnd(MatchInfo.InnerShiftTy, Shift, Mask); + Builder.buildTrunc(Dst, And); + } else + Builder.buildTrunc(Dst, Shift); + MI.eraseFromParent(); +} + bool CombinerHelper::matchCombineMulToShl(MachineInstr &MI, unsigned &ShiftVal) const { assert(MI.getOpcode() == TargetOpcode::G_MUL && "Expected a G_MUL"); diff --git a/llvm/lib/CodeGen/GlobalISel/GISelValueTracking.cpp b/llvm/lib/CodeGen/GlobalISel/GISelValueTracking.cpp index 974fc40de622..0cf44e02254d 100644 --- a/llvm/lib/CodeGen/GlobalISel/GISelValueTracking.cpp +++ b/llvm/lib/CodeGen/GlobalISel/GISelValueTracking.cpp @@ -1892,6 +1892,44 @@ unsigned GISelValueTracking::computeNumSignBits(Register R, FirstAnswer = std::min<uint64_t>(FirstAnswer + *C, TyBits); break; } + case TargetOpcode::G_SHL: { + Register Src1 = MI.getOperand(1).getReg(); + Register Src2 = MI.getOperand(2).getReg(); + if (std::optional<ConstantRange> ShAmtRange = + getValidShiftAmountRange(Src2, DemandedElts, Depth + 1)) { + uint64_t MaxShAmt = ShAmtRange->getUnsignedMax().getZExtValue(); + uint64_t MinShAmt = ShAmtRange->getUnsignedMin().getZExtValue(); + + MachineInstr &ExtMI = *MRI.getVRegDef(Src1); + unsigned ExtOpc = ExtMI.getOpcode(); + + // Try to look through ZERO/SIGN/ANY_EXTEND. If all extended bits are + // shifted out, then we can compute the number of sign bits for the + // operand being extended. A future improvement could be to pass along the + // "shifted left by" information in the recursive calls to + // ComputeKnownSignBits. Allowing us to handle this more generically. + if (ExtOpc == TargetOpcode::G_SEXT || ExtOpc == TargetOpcode::G_ZEXT || + ExtOpc == TargetOpcode::G_ANYEXT) { + LLT ExtTy = MRI.getType(Src1); + Register Extendee = ExtMI.getOperand(1).getReg(); + LLT ExtendeeTy = MRI.getType(Extendee); + uint64_t SizeDiff = + ExtTy.getScalarSizeInBits() - ExtendeeTy.getScalarSizeInBits(); + + if (SizeDiff <= MinShAmt) { + unsigned Tmp = + SizeDiff + computeNumSignBits(Extendee, DemandedElts, Depth + 1); + if (MaxShAmt < Tmp) + return Tmp - MaxShAmt; + } + } + // shl destroys sign bits, ensure it doesn't shift out all sign bits. + unsigned Tmp = computeNumSignBits(Src1, DemandedElts, Depth + 1); + if (MaxShAmt < Tmp) + return Tmp - MaxShAmt; + } + break; + } case TargetOpcode::G_TRUNC: { Register Src = MI.getOperand(1).getReg(); LLT SrcTy = MRI.getType(Src); diff --git a/llvm/lib/CodeGen/GlobalISel/LegalizerHelper.cpp b/llvm/lib/CodeGen/GlobalISel/LegalizerHelper.cpp index a38d305a8bb5..d87e335f837d 100644 --- a/llvm/lib/CodeGen/GlobalISel/LegalizerHelper.cpp +++ b/llvm/lib/CodeGen/GlobalISel/LegalizerHelper.cpp @@ -487,6 +487,10 @@ static RTLIB::Libcall getRTLibDesc(unsigned Opcode, unsigned Size) { RTLIBCASE(FMIN_F); case TargetOpcode::G_FMAXNUM: RTLIBCASE(FMAX_F); + case TargetOpcode::G_FMINIMUMNUM: + RTLIBCASE(FMINIMUM_NUM_F); + case TargetOpcode::G_FMAXIMUMNUM: + RTLIBCASE(FMAXIMUM_NUM_F); case TargetOpcode::G_FSQRT: RTLIBCASE(SQRT_F); case TargetOpcode::G_FRINT: @@ -1307,6 +1311,8 @@ LegalizerHelper::libcall(MachineInstr &MI, LostDebugLocObserver &LocObserver) { case TargetOpcode::G_FFLOOR: case TargetOpcode::G_FMINNUM: case TargetOpcode::G_FMAXNUM: + case TargetOpcode::G_FMINIMUMNUM: + case TargetOpcode::G_FMAXIMUMNUM: case TargetOpcode::G_FSQRT: case TargetOpcode::G_FRINT: case TargetOpcode::G_FNEARBYINT: @@ -2916,6 +2922,7 @@ LegalizerHelper::widenScalar(MachineInstr &MI, unsigned TypeIdx, LLT WideTy) { case TargetOpcode::G_SREM: case TargetOpcode::G_SMIN: case TargetOpcode::G_SMAX: + case TargetOpcode::G_ABDS: Observer.changingInstr(MI); widenScalarSrc(MI, WideTy, 1, TargetOpcode::G_SEXT); widenScalarSrc(MI, WideTy, 2, TargetOpcode::G_SEXT); @@ -2953,6 +2960,7 @@ LegalizerHelper::widenScalar(MachineInstr &MI, unsigned TypeIdx, LLT WideTy) { return Legalized; case TargetOpcode::G_UDIV: case TargetOpcode::G_UREM: + case TargetOpcode::G_ABDU: Observer.changingInstr(MI); widenScalarSrc(MI, WideTy, 1, TargetOpcode::G_ZEXT); widenScalarSrc(MI, WideTy, 2, TargetOpcode::G_ZEXT); @@ -4447,6 +4455,8 @@ LegalizerHelper::lower(MachineInstr &MI, unsigned TypeIdx, LLT LowerHintTy) { case TargetOpcode::G_SADDO: case TargetOpcode::G_SSUBO: return lowerSADDO_SSUBO(MI); + case TargetOpcode::G_SADDE: + return lowerSADDE(MI); case TargetOpcode::G_UMULH: case TargetOpcode::G_SMULH: return lowerSMULH_UMULH(MI); @@ -9298,6 +9308,28 @@ LegalizerHelper::lowerSADDO_SSUBO(MachineInstr &MI) { return Legalized; } +LegalizerHelper::LegalizeResult LegalizerHelper::lowerSADDE(MachineInstr &MI) { + auto [Res, OvOut, LHS, RHS, CarryIn] = MI.getFirst5Regs(); + const LLT Ty = MRI.getType(Res); + + // sum = LHS + RHS + zext(CarryIn) + auto Tmp = MIRBuilder.buildAdd(Ty, LHS, RHS); + auto CarryZ = MIRBuilder.buildZExt(Ty, CarryIn); + auto Sum = MIRBuilder.buildAdd(Ty, Tmp, CarryZ); + MIRBuilder.buildCopy(Res, Sum); + + // OvOut = icmp slt ((sum ^ lhs) & (sum ^ rhs)), 0 + auto AX = MIRBuilder.buildXor(Ty, Sum, LHS); + auto BX = MIRBuilder.buildXor(Ty, Sum, RHS); + auto T = MIRBuilder.buildAnd(Ty, AX, BX); + + auto Zero = MIRBuilder.buildConstant(Ty, 0); + MIRBuilder.buildICmp(CmpInst::ICMP_SLT, OvOut, T, Zero); + + MI.eraseFromParent(); + return Legalized; +} + LegalizerHelper::LegalizeResult LegalizerHelper::lowerAddSubSatToMinMax(MachineInstr &MI) { auto [Res, LHS, RHS] = MI.getFirst3Regs(); diff --git a/llvm/lib/CodeGen/GlobalISel/Utils.cpp b/llvm/lib/CodeGen/GlobalISel/Utils.cpp index 58d631e569b3..1ccc549e0ec6 100644 --- a/llvm/lib/CodeGen/GlobalISel/Utils.cpp +++ b/llvm/lib/CodeGen/GlobalISel/Utils.cpp @@ -1409,7 +1409,8 @@ bool llvm::isBuildVectorConstantSplat(const Register Reg, bool llvm::isBuildVectorConstantSplat(const Register Reg, const MachineRegisterInfo &MRI, - APInt SplatValue, bool AllowUndef) { + const APInt &SplatValue, + bool AllowUndef) { if (auto SplatValAndReg = getAnyConstantSplat(Reg, MRI, AllowUndef)) { if (SplatValAndReg->Value.getBitWidth() < SplatValue.getBitWidth()) return APInt::isSameValue( @@ -1431,7 +1432,8 @@ bool llvm::isBuildVectorConstantSplat(const MachineInstr &MI, bool llvm::isBuildVectorConstantSplat(const MachineInstr &MI, const MachineRegisterInfo &MRI, - APInt SplatValue, bool AllowUndef) { + const APInt &SplatValue, + bool AllowUndef) { return isBuildVectorConstantSplat(MI.getOperand(0).getReg(), MRI, SplatValue, AllowUndef); } @@ -1758,9 +1760,11 @@ bool llvm::isPreISelGenericFloatingPointOpcode(unsigned Opc) { case TargetOpcode::G_FMA: case TargetOpcode::G_FMAD: case TargetOpcode::G_FMAXIMUM: + case TargetOpcode::G_FMAXIMUMNUM: case TargetOpcode::G_FMAXNUM: case TargetOpcode::G_FMAXNUM_IEEE: case TargetOpcode::G_FMINIMUM: + case TargetOpcode::G_FMINIMUMNUM: case TargetOpcode::G_FMINNUM: case TargetOpcode::G_FMINNUM_IEEE: case TargetOpcode::G_FMUL: diff --git a/llvm/lib/CodeGen/MachineInstrBundle.cpp b/llvm/lib/CodeGen/MachineInstrBundle.cpp index 5830ecfe4aa2..da29ffc9d2fe 100644 --- a/llvm/lib/CodeGen/MachineInstrBundle.cpp +++ b/llvm/lib/CodeGen/MachineInstrBundle.cpp @@ -133,7 +133,6 @@ void llvm::finalizeBundle(MachineBasicBlock &MBB, SmallSetVector<Register, 32> LocalDefs; BitVector LocalDefsP(TRI->getNumRegUnits()); SmallSet<Register, 8> DeadDefSet; - SmallSet<Register, 16> KilledDefSet; SmallSetVector<Register, 8> ExternUses; SmallSet<Register, 8> KilledUseSet; SmallSet<Register, 8> UndefUseSet; @@ -151,7 +150,7 @@ void llvm::finalizeBundle(MachineBasicBlock &MBB, MO.setIsInternalRead(); if (MO.isKill()) { // Internal def is now killed. - KilledDefSet.insert(Reg); + DeadDefSet.insert(Reg); } } else { if (ExternUses.insert(Reg)) { @@ -171,19 +170,18 @@ void llvm::finalizeBundle(MachineBasicBlock &MBB, continue; if (LocalDefs.insert(Reg)) { - if (MO.isDead()) - DeadDefSet.insert(Reg); - else if (Reg.isPhysical()) + if (!MO.isDead() && Reg.isPhysical()) { for (MCRegUnit Unit : TRI->regunits(Reg.asMCReg())) LocalDefsP.set(Unit); + } } else { - // Re-defined inside the bundle, it's no longer killed. - KilledDefSet.erase(Reg); if (!MO.isDead()) { - // Previously defined but dead. + // Re-defined inside the bundle, it's no longer dead. DeadDefSet.erase(Reg); } } + if (MO.isDead()) + DeadDefSet.insert(Reg); } // Set FrameSetup/FrameDestroy for the bundle. If any of the instructions @@ -196,7 +194,7 @@ void llvm::finalizeBundle(MachineBasicBlock &MBB, for (Register Reg : LocalDefs) { // If it's not live beyond end of the bundle, mark it dead. - bool isDead = DeadDefSet.contains(Reg) || KilledDefSet.contains(Reg); + bool isDead = DeadDefSet.contains(Reg); MIB.addReg(Reg, getDefRegState(true) | getDeadRegState(isDead) | getImplRegState(true)); } diff --git a/llvm/lib/CodeGen/MachineOutliner.cpp b/llvm/lib/CodeGen/MachineOutliner.cpp index c7f09d577c4f..fdae3b470de0 100644 --- a/llvm/lib/CodeGen/MachineOutliner.cpp +++ b/llvm/lib/CodeGen/MachineOutliner.cpp @@ -59,8 +59,10 @@ #include "llvm/ADT/SmallSet.h" #include "llvm/ADT/Statistic.h" #include "llvm/ADT/Twine.h" +#include "llvm/Analysis/BlockFrequencyInfo.h" #include "llvm/Analysis/ModuleSummaryAnalysis.h" #include "llvm/Analysis/OptimizationRemarkEmitter.h" +#include "llvm/Analysis/ProfileSummaryInfo.h" #include "llvm/CGData/CodeGenDataReader.h" #include "llvm/CodeGen/LivePhysRegs.h" #include "llvm/CodeGen/MachineModuleInfo.h" @@ -107,6 +109,16 @@ STATISTIC(StableHashAttempts, STATISTIC(StableHashDropped, "Count of unsuccessful hashing attempts for outlined functions"); STATISTIC(NumRemovedLOHs, "Total number of Linker Optimization Hints removed"); +STATISTIC(NumPGOBlockedOutlined, + "Number of times outlining was blocked by PGO"); +STATISTIC(NumPGOAllowedCold, + "Number of times outlining was allowed from cold functions"); +STATISTIC(NumPGOConservativeBlockedOutlined, + "Number of times outlining was blocked conservatively when profile " + "counts were missing"); +STATISTIC(NumPGOOptimisticOutlined, + "Number of times outlining was allowed optimistically when profile " + "counts were missing"); // Set to true if the user wants the outliner to run on linkonceodr linkage // functions. This is false by default because the linker can dedupe linkonceodr @@ -438,11 +450,10 @@ struct MachineOutliner : public ModulePass { /// The current repeat number of machine outlining. unsigned OutlineRepeatedNum = 0; - /// Set to true if the outliner should run on all functions in the module - /// considered safe for outlining. - /// Set to true by default for compatibility with llc's -run-pass option. - /// Set when the pass is constructed in TargetPassConfig. - bool RunOnAllFunctions = true; + /// The mode for whether to run the outliner + /// Set to always-outline by default for compatibility with llc's -run-pass + /// option. + RunOutliner RunOutlinerMode = RunOutliner::AlwaysOutline; /// This is a compact representation of hash sequences of outlined functions. /// It is used when OutlinerMode = CGDataMode::Write. @@ -468,6 +479,11 @@ struct MachineOutliner : public ModulePass { AU.addRequired<TargetPassConfig>(); AU.addPreserved<MachineModuleInfoWrapperPass>(); AU.addUsedIfAvailable<ImmutableModuleSummaryIndexWrapperPass>(); + if (RunOutlinerMode == RunOutliner::OptimisticPGO || + RunOutlinerMode == RunOutliner::ConservativePGO) { + AU.addRequired<BlockFrequencyInfoWrapperPass>(); + AU.addRequired<ProfileSummaryInfoWrapperPass>(); + } AU.setPreservesAll(); ModulePass::getAnalysisUsage(AU); } @@ -578,9 +594,9 @@ struct MachineOutliner : public ModulePass { char MachineOutliner::ID = 0; namespace llvm { -ModulePass *createMachineOutlinerPass(bool RunOnAllFunctions) { +ModulePass *createMachineOutlinerPass(RunOutliner RunOutlinerMode) { MachineOutliner *OL = new MachineOutliner(); - OL->RunOnAllFunctions = RunOnAllFunctions; + OL->RunOutlinerMode = RunOutlinerMode; return OL; } @@ -1198,10 +1214,49 @@ bool MachineOutliner::outline( return OutlinedSomething; } +static bool allowPGOOutlining(RunOutliner RunOutlinerMode, + const ProfileSummaryInfo *PSI, + const BlockFrequencyInfo *BFI, + MachineBasicBlock &MBB) { + if (RunOutlinerMode != RunOutliner::OptimisticPGO && + RunOutlinerMode != RunOutliner::ConservativePGO) + return true; + auto *MF = MBB.getParent(); + if (MF->getFunction().hasFnAttribute(Attribute::Cold)) { + ++NumPGOAllowedCold; + return true; + } + + auto *BB = MBB.getBasicBlock(); + if (BB && PSI && BFI) + if (auto Count = BFI->getBlockProfileCount(BB)) + return *Count <= PSI->getOrCompColdCountThreshold(); + + if (RunOutlinerMode == RunOutliner::OptimisticPGO) { + auto *TII = MF->getSubtarget().getInstrInfo(); + if (TII->shouldOutlineFromFunctionByDefault(*MF)) { + // Profile data is unavailable, but we optimistically allow outlining + ++NumPGOOptimisticOutlined; + return true; + } + return false; + } + assert(RunOutlinerMode == RunOutliner::ConservativePGO); + // Profile data is unavailable, so we conservatively block outlining + ++NumPGOConservativeBlockedOutlined; + return false; +} + void MachineOutliner::populateMapper(InstructionMapper &Mapper, Module &M) { // Build instruction mappings for each function in the module. Start by // iterating over each Function in M. LLVM_DEBUG(dbgs() << "*** Populating mapper ***\n"); + bool EnableProfileGuidedOutlining = + RunOutlinerMode == RunOutliner::OptimisticPGO || + RunOutlinerMode == RunOutliner::ConservativePGO; + ProfileSummaryInfo *PSI = nullptr; + if (EnableProfileGuidedOutlining) + PSI = &getAnalysis<ProfileSummaryInfoWrapperPass>().getPSI(); for (Function &F : M) { LLVM_DEBUG(dbgs() << "MAPPING FUNCTION: " << F.getName() << "\n"); @@ -1222,7 +1277,11 @@ void MachineOutliner::populateMapper(InstructionMapper &Mapper, Module &M) { } const TargetInstrInfo *TII = MF->getSubtarget().getInstrInfo(); - if (!RunOnAllFunctions && !TII->shouldOutlineFromFunctionByDefault(*MF)) { + BlockFrequencyInfo *BFI = nullptr; + if (EnableProfileGuidedOutlining && F.hasProfileData()) + BFI = &getAnalysis<BlockFrequencyInfoWrapperPass>(F).getBFI(); + if (RunOutlinerMode == RunOutliner::TargetDefault && + !TII->shouldOutlineFromFunctionByDefault(*MF)) { LLVM_DEBUG(dbgs() << "SKIP: Target does not want to outline from " "function by default\n"); continue; @@ -1262,6 +1321,11 @@ void MachineOutliner::populateMapper(InstructionMapper &Mapper, Module &M) { continue; } + if (!allowPGOOutlining(RunOutlinerMode, PSI, BFI, MBB)) { + ++NumPGOBlockedOutlined; + continue; + } + // MBB is suitable for outlining. Map it to a list of unsigneds. Mapper.convertToUnsignedVec(MBB, *TII); } @@ -1434,10 +1498,22 @@ bool MachineOutliner::doOutline(Module &M, unsigned &OutlinedFunctionNum) { // the user how the outliner is running. LLVM_DEBUG({ dbgs() << "Machine Outliner: Running on "; - if (RunOnAllFunctions) + switch (RunOutlinerMode) { + case RunOutliner::AlwaysOutline: dbgs() << "all functions"; - else + break; + case RunOutliner::OptimisticPGO: + dbgs() << "optimistically cold functions"; + break; + case RunOutliner::ConservativePGO: + dbgs() << "conservatively cold functions"; + break; + case RunOutliner::TargetDefault: dbgs() << "target-default functions"; + break; + case RunOutliner::NeverOutline: + llvm_unreachable("should not outline"); + } dbgs() << "\n"; }); diff --git a/llvm/lib/CodeGen/TargetPassConfig.cpp b/llvm/lib/CodeGen/TargetPassConfig.cpp index f83973c30b48..b6169e6c4dc3 100644 --- a/llvm/lib/CodeGen/TargetPassConfig.cpp +++ b/llvm/lib/CodeGen/TargetPassConfig.cpp @@ -134,12 +134,18 @@ static cl::opt<cl::boolOrDefault> DebugifyCheckAndStripAll( static cl::opt<RunOutliner> EnableMachineOutliner( "enable-machine-outliner", cl::desc("Enable the machine outliner"), cl::Hidden, cl::ValueOptional, cl::init(RunOutliner::TargetDefault), - cl::values(clEnumValN(RunOutliner::AlwaysOutline, "always", - "Run on all functions guaranteed to be beneficial"), - clEnumValN(RunOutliner::NeverOutline, "never", - "Disable all outlining"), - // Sentinel value for unspecified option. - clEnumValN(RunOutliner::AlwaysOutline, "", ""))); + cl::values( + clEnumValN(RunOutliner::AlwaysOutline, "always", + "Run on all functions guaranteed to be beneficial"), + clEnumValN(RunOutliner::OptimisticPGO, "optimistic-pgo", + "Outline cold code only. If a code block does not have " + "profile data, optimistically assume it is cold."), + clEnumValN(RunOutliner::ConservativePGO, "conservative-pgo", + "Outline cold code only. If a code block does not have " + "profile, data, conservatively assume it is hot."), + clEnumValN(RunOutliner::NeverOutline, "never", "Disable all outlining"), + // Sentinel value for unspecified option. + clEnumValN(RunOutliner::AlwaysOutline, "", ""))); static cl::opt<bool> EnableGlobalMergeFunc( "enable-global-merge-func", cl::Hidden, cl::desc("Enable global merge functions that are based on hash function")); @@ -1224,12 +1230,9 @@ void TargetPassConfig::addMachinePasses() { if (TM->Options.EnableMachineOutliner && getOptLevel() != CodeGenOptLevel::None && EnableMachineOutliner != RunOutliner::NeverOutline) { - bool RunOnAllFunctions = - (EnableMachineOutliner == RunOutliner::AlwaysOutline); - bool AddOutliner = - RunOnAllFunctions || TM->Options.SupportsDefaultOutlining; - if (AddOutliner) - addPass(createMachineOutlinerPass(RunOnAllFunctions)); + if (EnableMachineOutliner != RunOutliner::TargetDefault || + TM->Options.SupportsDefaultOutlining) + addPass(createMachineOutlinerPass(EnableMachineOutliner)); } if (GCEmptyBlocks) diff --git a/llvm/lib/CodeGen/ValueTypes.cpp b/llvm/lib/CodeGen/ValueTypes.cpp index 10970b719fca..0743c92c5b95 100644 --- a/llvm/lib/CodeGen/ValueTypes.cpp +++ b/llvm/lib/CodeGen/ValueTypes.cpp @@ -176,6 +176,8 @@ std::string EVT::getEVTString() const { return "i" + utostr(getSizeInBits()); if (isFloatingPoint()) return "f" + utostr(getSizeInBits()); + if (isCheriCapability()) + return "c" + utostr(getSizeInBits()); llvm_unreachable("Invalid EVT!"); case MVT::bf16: return "bf16"; case MVT::ppcf128: return "ppcf128"; diff --git a/llvm/lib/DebugInfo/PDB/Native/PublicsStream.cpp b/llvm/lib/DebugInfo/PDB/Native/PublicsStream.cpp index c350e0e0b3e1..0453eea26605 100644 --- a/llvm/lib/DebugInfo/PDB/Native/PublicsStream.cpp +++ b/llvm/lib/DebugInfo/PDB/Native/PublicsStream.cpp @@ -22,9 +22,12 @@ //===----------------------------------------------------------------------===// #include "llvm/DebugInfo/PDB/Native/PublicsStream.h" +#include "llvm/DebugInfo/CodeView/SymbolDeserializer.h" +#include "llvm/DebugInfo/CodeView/SymbolRecord.h" #include "llvm/DebugInfo/MSF/MappedBlockStream.h" #include "llvm/DebugInfo/PDB/Native/RawError.h" #include "llvm/DebugInfo/PDB/Native/RawTypes.h" +#include "llvm/DebugInfo/PDB/Native/SymbolStream.h" #include "llvm/Support/BinaryStreamReader.h" #include "llvm/Support/Error.h" #include <cstdint> @@ -96,3 +99,50 @@ Error PublicsStream::reload() { "Corrupted publics stream."); return Error::success(); } + +// This is a reimplementation of NearestSym: +// https://github.com/microsoft/microsoft-pdb/blob/805655a28bd8198004be2ac27e6e0290121a5e89/PDB/dbi/gsi.cpp#L1492-L1581 +std::optional<std::pair<codeview::PublicSym32, size_t>> +PublicsStream::findByAddress(const SymbolStream &Symbols, uint16_t Segment, + uint32_t Offset) const { + // The address map is sorted by address, so we can use lower_bound to find the + // position. Each element is an offset into the symbols for a public symbol. + auto It = llvm::lower_bound( + AddressMap, std::tuple(Segment, Offset), + [&](support::ulittle32_t Cur, auto Addr) { + auto Sym = Symbols.readRecord(Cur.value()); + if (Sym.kind() != codeview::S_PUB32) + return false; // stop here, this is most likely corrupted debug info + + auto Psym = + codeview::SymbolDeserializer::deserializeAs<codeview::PublicSym32>( + Sym); + if (!Psym) { + consumeError(Psym.takeError()); + return false; + } + + return std::tie(Psym->Segment, Psym->Offset) < Addr; + }); + + if (It == AddressMap.end()) + return std::nullopt; + + auto Sym = Symbols.readRecord(It->value()); + if (Sym.kind() != codeview::S_PUB32) + return std::nullopt; // this is most likely corrupted debug info + + auto MaybePsym = + codeview::SymbolDeserializer::deserializeAs<codeview::PublicSym32>(Sym); + if (!MaybePsym) { + consumeError(MaybePsym.takeError()); + return std::nullopt; + } + codeview::PublicSym32 Psym = std::move(*MaybePsym); + + if (std::tuple(Segment, Offset) != std::tuple(Psym.Segment, Psym.Offset)) + return std::nullopt; + + std::ptrdiff_t IterOffset = It - AddressMap.begin(); + return std::pair{Psym, static_cast<size_t>(IterOffset)}; +} diff --git a/llvm/lib/ExecutionEngine/JITLink/ELF_riscv.cpp b/llvm/lib/ExecutionEngine/JITLink/ELF_riscv.cpp index 71a0f14368ac..0dc97104610b 100644 --- a/llvm/lib/ExecutionEngine/JITLink/ELF_riscv.cpp +++ b/llvm/lib/ExecutionEngine/JITLink/ELF_riscv.cpp @@ -154,18 +154,22 @@ public: std::unique_ptr<LinkGraph> G, PassConfiguration PassConfig) : JITLinker(std::move(Ctx), std::move(G), std::move(PassConfig)) { JITLinkerBase::getPassConfig().PostAllocationPasses.push_back( - [this](LinkGraph &G) { return gatherRISCVPCRelHi20(G); }); + [this](LinkGraph &G) { return gatherRISCVPairs(G); }); } private: DenseMap<std::pair<const Block *, orc::ExecutorAddrDiff>, const Edge *> RelHi20; + DenseMap<std::pair<const Block *, orc::ExecutorAddrDiff>, const Edge *> + SetULEB128; - Error gatherRISCVPCRelHi20(LinkGraph &G) { + Error gatherRISCVPairs(LinkGraph &G) { for (Block *B : G.blocks()) for (Edge &E : B->edges()) if (E.getKind() == R_RISCV_PCREL_HI20) RelHi20[{B, E.getOffset()}] = &E; + else if (E.getKind() == R_RISCV_SET_ULEB128) + SetULEB128[{B, E.getOffset()}] = &E; return Error::success(); } @@ -189,6 +193,20 @@ private: "for LO12 PCREL relocation type"); } + Expected<const Edge &> getRISCVSetULEB128(const Block &B, + const Edge &E) const { + using namespace riscv; + assert(E.getKind() == R_RISCV_SUB_ULEB128 && + "Can only have pair relocation for R_RISCV_SUB_ULEB128"); + + auto It = SetULEB128.find({&B, E.getOffset()}); + if (It != SetULEB128.end()) + return *It->second; + + return make_error<JITLinkError>( + "No RISCV_SET_ULEB128 relocation type be found"); + } + Error applyFixup(LinkGraph &G, Block &B, const Edge &E) const { using namespace riscv; using namespace llvm::support; @@ -467,6 +485,21 @@ private: *(little32_t *)FixupPtr = static_cast<uint32_t>(Value); break; } + case R_RISCV_SET_ULEB128: + break; + case R_RISCV_SUB_ULEB128: { + auto SetULEB128 = getRISCVSetULEB128(B, E); + if (!SetULEB128) + return SetULEB128.takeError(); + uint64_t Value = SetULEB128->getTarget().getAddress() + + SetULEB128->getAddend() - E.getTarget().getAddress() - + E.getAddend(); + if (overwriteULEB128(reinterpret_cast<uint8_t *>(FixupPtr), Value) >= + 0x80) + return make_error<StringError>("ULEB128 value exceeds available space", + inconvertibleErrorCode()); + break; + } } return Error::success(); } @@ -843,6 +876,10 @@ private: return EdgeKind_riscv::R_RISCV_32_PCREL; case ELF::R_RISCV_ALIGN: return EdgeKind_riscv::AlignRelaxable; + case ELF::R_RISCV_SET_ULEB128: + return EdgeKind_riscv::R_RISCV_SET_ULEB128; + case ELF::R_RISCV_SUB_ULEB128: + return EdgeKind_riscv::R_RISCV_SUB_ULEB128; } return make_error<JITLinkError>( diff --git a/llvm/lib/ExecutionEngine/JITLink/riscv.cpp b/llvm/lib/ExecutionEngine/JITLink/riscv.cpp index a4e4daef97fb..9e9f4433a9fc 100644 --- a/llvm/lib/ExecutionEngine/JITLink/riscv.cpp +++ b/llvm/lib/ExecutionEngine/JITLink/riscv.cpp @@ -84,6 +84,10 @@ const char *getEdgeKindName(Edge::Kind K) { return "AlignRelaxable"; case NegDelta32: return "NegDelta32"; + case R_RISCV_SET_ULEB128: + return "R_RISCV_SET_ULEB128"; + case R_RISCV_SUB_ULEB128: + return "R_RISCV_SUB_ULEB128"; } return getGenericEdgeKindName(K); } diff --git a/llvm/lib/ExecutionEngine/RuntimeDyld/RuntimeDyldChecker.cpp b/llvm/lib/ExecutionEngine/RuntimeDyld/RuntimeDyldChecker.cpp index df702130450d..afe3b671547d 100644 --- a/llvm/lib/ExecutionEngine/RuntimeDyld/RuntimeDyldChecker.cpp +++ b/llvm/lib/ExecutionEngine/RuntimeDyld/RuntimeDyldChecker.cpp @@ -756,13 +756,10 @@ private: Expected<TargetInfo> getTargetInfo(const Triple &TT, const StringRef &CPU, const SubtargetFeatures &TF) const { - - auto TripleName = TT.str(); std::string ErrorStr; - const Target *TheTarget = - TargetRegistry::lookupTarget(TripleName, ErrorStr); + const Target *TheTarget = TargetRegistry::lookupTarget(TT, ErrorStr); if (!TheTarget) - return make_error<StringError>("Error accessing target '" + TripleName + + return make_error<StringError>("Error accessing target '" + TT.str() + "': " + ErrorStr, inconvertibleErrorCode()); @@ -770,14 +767,14 @@ private: TheTarget->createMCSubtargetInfo(TT, CPU, TF.getString())); if (!STI) return make_error<StringError>("Unable to create subtarget for " + - TripleName, + TT.str(), inconvertibleErrorCode()); std::unique_ptr<MCRegisterInfo> MRI(TheTarget->createMCRegInfo(TT)); if (!MRI) return make_error<StringError>("Unable to create target register info " "for " + - TripleName, + TT.str(), inconvertibleErrorCode()); MCTargetOptions MCOptions; @@ -785,30 +782,30 @@ private: TheTarget->createMCAsmInfo(*MRI, TT, MCOptions)); if (!MAI) return make_error<StringError>("Unable to create target asm info " + - TripleName, + TT.str(), inconvertibleErrorCode()); - auto Ctx = std::make_unique<MCContext>(Triple(TripleName), MAI.get(), + auto Ctx = std::make_unique<MCContext>(Triple(TT.str()), MAI.get(), MRI.get(), STI.get()); std::unique_ptr<MCDisassembler> Disassembler( TheTarget->createMCDisassembler(*STI, *Ctx)); if (!Disassembler) return make_error<StringError>("Unable to create disassembler for " + - TripleName, + TT.str(), inconvertibleErrorCode()); std::unique_ptr<MCInstrInfo> MII(TheTarget->createMCInstrInfo()); if (!MII) return make_error<StringError>("Unable to create instruction info for" + - TripleName, + TT.str(), inconvertibleErrorCode()); - std::unique_ptr<MCInstPrinter> InstPrinter(TheTarget->createMCInstPrinter( - Triple(TripleName), 0, *MAI, *MII, *MRI)); + std::unique_ptr<MCInstPrinter> InstPrinter( + TheTarget->createMCInstPrinter(TT, 0, *MAI, *MII, *MRI)); if (!InstPrinter) return make_error<StringError>( - "Unable to create instruction printer for" + TripleName, + "Unable to create instruction printer for" + TT.str(), inconvertibleErrorCode()); return TargetInfo({TheTarget, std::move(STI), std::move(MRI), diff --git a/llvm/lib/FileCheck/FileCheck.cpp b/llvm/lib/FileCheck/FileCheck.cpp index ce35a5bad761..9245db442611 100644 --- a/llvm/lib/FileCheck/FileCheck.cpp +++ b/llvm/lib/FileCheck/FileCheck.cpp @@ -1218,6 +1218,14 @@ Pattern::MatchResult Pattern::match(StringRef Buffer, StringRef MatchedValue = MatchInfo[CaptureParenGroup]; ExpressionFormat Format = DefinedNumericVariable->getImplicitFormat(); APInt Value = Format.valueFromStringRepr(MatchedValue, SM); + // Numeric variables are already inserted into GlobalNumericVariableTable + // during parsing, but clearLocalVars might remove them, so we must + // reinsert them. Numeric-variable resolution does not access + // GlobalNumericVariableTable; it directly uses a pointer to the variable. + // However, other functions (such as clearLocalVars) may require active + // variables to be in the table. + Context->GlobalNumericVariableTable.try_emplace(NumericVariableDef.getKey(), + DefinedNumericVariable); DefinedNumericVariable->setValue(Value, MatchedValue); } diff --git a/llvm/lib/Frontend/HLSL/RootSignatureMetadata.cpp b/llvm/lib/Frontend/HLSL/RootSignatureMetadata.cpp index a5a92cbd2d61..f29f2c7602fc 100644 --- a/llvm/lib/Frontend/HLSL/RootSignatureMetadata.cpp +++ b/llvm/lib/Frontend/HLSL/RootSignatureMetadata.cpp @@ -52,13 +52,15 @@ static std::optional<StringRef> extractMdStringValue(MDNode *Node, return NodeText->getString(); } -static Expected<dxbc::ShaderVisibility> -extractShaderVisibility(MDNode *Node, unsigned int OpId) { +template <typename T, typename = std::enable_if_t< + std::is_enum_v<T> && + std::is_same_v<std::underlying_type_t<T>, uint32_t>>> +Expected<T> extractEnumValue(MDNode *Node, unsigned int OpId, StringRef ErrText, + llvm::function_ref<bool(uint32_t)> VerifyFn) { if (std::optional<uint32_t> Val = extractMdIntValue(Node, OpId)) { - if (!dxbc::isValidShaderVisibility(*Val)) - return make_error<RootSignatureValidationError<uint32_t>>( - "ShaderVisibility", *Val); - return dxbc::ShaderVisibility(*Val); + if (!VerifyFn(*Val)) + return make_error<RootSignatureValidationError<uint32_t>>(ErrText, *Val); + return static_cast<T>(*Val); } return make_error<InvalidRSMetadataValue>("ShaderVisibility"); } @@ -233,7 +235,9 @@ Error MetadataParser::parseRootConstants(mcdxbc::RootSignatureDesc &RSD, return make_error<InvalidRSMetadataFormat>("RootConstants Element"); Expected<dxbc::ShaderVisibility> Visibility = - extractShaderVisibility(RootConstantNode, 1); + extractEnumValue<dxbc::ShaderVisibility>(RootConstantNode, 1, + "ShaderVisibility", + dxbc::isValidShaderVisibility); if (auto E = Visibility.takeError()) return Error(std::move(E)); @@ -287,7 +291,9 @@ Error MetadataParser::parseRootDescriptors( } Expected<dxbc::ShaderVisibility> Visibility = - extractShaderVisibility(RootDescriptorNode, 1); + extractEnumValue<dxbc::ShaderVisibility>(RootDescriptorNode, 1, + "ShaderVisibility", + dxbc::isValidShaderVisibility); if (auto E = Visibility.takeError()) return Error(std::move(E)); @@ -322,7 +328,7 @@ Error MetadataParser::parseDescriptorRange(mcdxbc::DescriptorTable &Table, if (RangeDescriptorNode->getNumOperands() != 6) return make_error<InvalidRSMetadataFormat>("Descriptor Range"); - dxbc::RTS0::v2::DescriptorRange Range; + mcdxbc::DescriptorRange Range; std::optional<StringRef> ElementText = extractMdStringValue(RangeDescriptorNode, 0); @@ -330,15 +336,15 @@ Error MetadataParser::parseDescriptorRange(mcdxbc::DescriptorTable &Table, if (!ElementText.has_value()) return make_error<InvalidRSMetadataFormat>("Descriptor Range"); - Range.RangeType = - StringSwitch<uint32_t>(*ElementText) - .Case("CBV", to_underlying(dxbc::DescriptorRangeType::CBV)) - .Case("SRV", to_underlying(dxbc::DescriptorRangeType::SRV)) - .Case("UAV", to_underlying(dxbc::DescriptorRangeType::UAV)) - .Case("Sampler", to_underlying(dxbc::DescriptorRangeType::Sampler)) - .Default(~0U); - - if (Range.RangeType == ~0U) + if (*ElementText == "CBV") + Range.RangeType = dxil::ResourceClass::CBuffer; + else if (*ElementText == "SRV") + Range.RangeType = dxil::ResourceClass::SRV; + else if (*ElementText == "UAV") + Range.RangeType = dxil::ResourceClass::UAV; + else if (*ElementText == "Sampler") + Range.RangeType = dxil::ResourceClass::Sampler; + else return make_error<GenericRSMetadataError>("Invalid Descriptor Range type.", RangeDescriptorNode); @@ -380,7 +386,9 @@ Error MetadataParser::parseDescriptorTable(mcdxbc::RootSignatureDesc &RSD, return make_error<InvalidRSMetadataFormat>("Descriptor Table"); Expected<dxbc::ShaderVisibility> Visibility = - extractShaderVisibility(DescriptorTableNode, 1); + extractEnumValue<dxbc::ShaderVisibility>(DescriptorTableNode, 1, + "ShaderVisibility", + dxbc::isValidShaderVisibility); if (auto E = Visibility.takeError()) return Error(std::move(E)); @@ -406,26 +414,34 @@ Error MetadataParser::parseStaticSampler(mcdxbc::RootSignatureDesc &RSD, if (StaticSamplerNode->getNumOperands() != 14) return make_error<InvalidRSMetadataFormat>("Static Sampler"); - dxbc::RTS0::v1::StaticSampler Sampler; - if (std::optional<uint32_t> Val = extractMdIntValue(StaticSamplerNode, 1)) - Sampler.Filter = *Val; - else - return make_error<InvalidRSMetadataValue>("Filter"); + mcdxbc::StaticSampler Sampler; - if (std::optional<uint32_t> Val = extractMdIntValue(StaticSamplerNode, 2)) - Sampler.AddressU = *Val; - else - return make_error<InvalidRSMetadataValue>("AddressU"); + Expected<dxbc::SamplerFilter> Filter = extractEnumValue<dxbc::SamplerFilter>( + StaticSamplerNode, 1, "Filter", dxbc::isValidSamplerFilter); + if (auto E = Filter.takeError()) + return Error(std::move(E)); + Sampler.Filter = *Filter; - if (std::optional<uint32_t> Val = extractMdIntValue(StaticSamplerNode, 3)) - Sampler.AddressV = *Val; - else - return make_error<InvalidRSMetadataValue>("AddressV"); + Expected<dxbc::TextureAddressMode> AddressU = + extractEnumValue<dxbc::TextureAddressMode>( + StaticSamplerNode, 2, "AddressU", dxbc::isValidAddress); + if (auto E = AddressU.takeError()) + return Error(std::move(E)); + Sampler.AddressU = *AddressU; - if (std::optional<uint32_t> Val = extractMdIntValue(StaticSamplerNode, 4)) - Sampler.AddressW = *Val; - else - return make_error<InvalidRSMetadataValue>("AddressW"); + Expected<dxbc::TextureAddressMode> AddressV = + extractEnumValue<dxbc::TextureAddressMode>( + StaticSamplerNode, 3, "AddressV", dxbc::isValidAddress); + if (auto E = AddressV.takeError()) + return Error(std::move(E)); + Sampler.AddressV = *AddressV; + + Expected<dxbc::TextureAddressMode> AddressW = + extractEnumValue<dxbc::TextureAddressMode>( + StaticSamplerNode, 4, "AddressW", dxbc::isValidAddress); + if (auto E = AddressW.takeError()) + return Error(std::move(E)); + Sampler.AddressW = *AddressW; if (std::optional<float> Val = extractMdFloatValue(StaticSamplerNode, 5)) Sampler.MipLODBias = *Val; @@ -437,15 +453,19 @@ Error MetadataParser::parseStaticSampler(mcdxbc::RootSignatureDesc &RSD, else return make_error<InvalidRSMetadataValue>("MaxAnisotropy"); - if (std::optional<uint32_t> Val = extractMdIntValue(StaticSamplerNode, 7)) - Sampler.ComparisonFunc = *Val; - else - return make_error<InvalidRSMetadataValue>("ComparisonFunc"); + Expected<dxbc::ComparisonFunc> ComparisonFunc = + extractEnumValue<dxbc::ComparisonFunc>( + StaticSamplerNode, 7, "ComparisonFunc", dxbc::isValidComparisonFunc); + if (auto E = ComparisonFunc.takeError()) + return Error(std::move(E)); + Sampler.ComparisonFunc = *ComparisonFunc; - if (std::optional<uint32_t> Val = extractMdIntValue(StaticSamplerNode, 8)) - Sampler.BorderColor = *Val; - else - return make_error<InvalidRSMetadataValue>("ComparisonFunc"); + Expected<dxbc::StaticBorderColor> BorderColor = + extractEnumValue<dxbc::StaticBorderColor>( + StaticSamplerNode, 8, "BorderColor", dxbc::isValidBorderColor); + if (auto E = BorderColor.takeError()) + return Error(std::move(E)); + Sampler.BorderColor = *BorderColor; if (std::optional<float> Val = extractMdFloatValue(StaticSamplerNode, 9)) Sampler.MinLOD = *Val; @@ -467,10 +487,13 @@ Error MetadataParser::parseStaticSampler(mcdxbc::RootSignatureDesc &RSD, else return make_error<InvalidRSMetadataValue>("RegisterSpace"); - if (std::optional<uint32_t> Val = extractMdIntValue(StaticSamplerNode, 13)) - Sampler.ShaderVisibility = *Val; - else - return make_error<InvalidRSMetadataValue>("ShaderVisibility"); + Expected<dxbc::ShaderVisibility> Visibility = + extractEnumValue<dxbc::ShaderVisibility>(StaticSamplerNode, 13, + "ShaderVisibility", + dxbc::isValidShaderVisibility); + if (auto E = Visibility.takeError()) + return Error(std::move(E)); + Sampler.ShaderVisibility = *Visibility; RSD.StaticSamplers.push_back(Sampler); return Error::success(); @@ -568,13 +591,7 @@ Error MetadataParser::validateRootSignature( case dxbc::RootParameterType::DescriptorTable: { const mcdxbc::DescriptorTable &Table = RSD.ParametersContainer.getDescriptorTable(Info.Location); - for (const dxbc::RTS0::v2::DescriptorRange &Range : Table) { - if (!hlsl::rootsig::verifyRangeType(Range.RangeType)) - DeferredErrs = - joinErrors(std::move(DeferredErrs), - make_error<RootSignatureValidationError<uint32_t>>( - "RangeType", Range.RangeType)); - + for (const mcdxbc::DescriptorRange &Range : Table) { if (!hlsl::rootsig::verifyRegisterSpace(Range.RegisterSpace)) DeferredErrs = joinErrors(std::move(DeferredErrs), @@ -588,7 +605,8 @@ Error MetadataParser::validateRootSignature( "NumDescriptors", Range.NumDescriptors)); if (!hlsl::rootsig::verifyDescriptorRangeFlag( - RSD.Version, Range.RangeType, Range.Flags)) + RSD.Version, Range.RangeType, + dxbc::DescriptorRangeFlags(Range.Flags))) DeferredErrs = joinErrors(std::move(DeferredErrs), make_error<RootSignatureValidationError<uint32_t>>( @@ -599,30 +617,7 @@ Error MetadataParser::validateRootSignature( } } - for (const dxbc::RTS0::v1::StaticSampler &Sampler : RSD.StaticSamplers) { - if (!hlsl::rootsig::verifySamplerFilter(Sampler.Filter)) - DeferredErrs = - joinErrors(std::move(DeferredErrs), - make_error<RootSignatureValidationError<uint32_t>>( - "Filter", Sampler.Filter)); - - if (!hlsl::rootsig::verifyAddress(Sampler.AddressU)) - DeferredErrs = - joinErrors(std::move(DeferredErrs), - make_error<RootSignatureValidationError<uint32_t>>( - "AddressU", Sampler.AddressU)); - - if (!hlsl::rootsig::verifyAddress(Sampler.AddressV)) - DeferredErrs = - joinErrors(std::move(DeferredErrs), - make_error<RootSignatureValidationError<uint32_t>>( - "AddressV", Sampler.AddressV)); - - if (!hlsl::rootsig::verifyAddress(Sampler.AddressW)) - DeferredErrs = - joinErrors(std::move(DeferredErrs), - make_error<RootSignatureValidationError<uint32_t>>( - "AddressW", Sampler.AddressW)); + for (const mcdxbc::StaticSampler &Sampler : RSD.StaticSamplers) { if (!hlsl::rootsig::verifyMipLODBias(Sampler.MipLODBias)) DeferredErrs = joinErrors(std::move(DeferredErrs), @@ -635,18 +630,6 @@ Error MetadataParser::validateRootSignature( make_error<RootSignatureValidationError<uint32_t>>( "MaxAnisotropy", Sampler.MaxAnisotropy)); - if (!hlsl::rootsig::verifyComparisonFunc(Sampler.ComparisonFunc)) - DeferredErrs = - joinErrors(std::move(DeferredErrs), - make_error<RootSignatureValidationError<uint32_t>>( - "ComparisonFunc", Sampler.ComparisonFunc)); - - if (!hlsl::rootsig::verifyBorderColor(Sampler.BorderColor)) - DeferredErrs = - joinErrors(std::move(DeferredErrs), - make_error<RootSignatureValidationError<uint32_t>>( - "BorderColor", Sampler.BorderColor)); - if (!hlsl::rootsig::verifyLOD(Sampler.MinLOD)) DeferredErrs = joinErrors(std::move(DeferredErrs), make_error<RootSignatureValidationError<float>>( @@ -668,12 +651,6 @@ Error MetadataParser::validateRootSignature( joinErrors(std::move(DeferredErrs), make_error<RootSignatureValidationError<uint32_t>>( "RegisterSpace", Sampler.RegisterSpace)); - - if (!dxbc::isValidShaderVisibility(Sampler.ShaderVisibility)) - DeferredErrs = - joinErrors(std::move(DeferredErrs), - make_error<RootSignatureValidationError<uint32_t>>( - "ShaderVisibility", Sampler.ShaderVisibility)); } return DeferredErrs; diff --git a/llvm/lib/Frontend/HLSL/RootSignatureValidations.cpp b/llvm/lib/Frontend/HLSL/RootSignatureValidations.cpp index 72308a3de5fd..0970977b5064 100644 --- a/llvm/lib/Frontend/HLSL/RootSignatureValidations.cpp +++ b/llvm/lib/Frontend/HLSL/RootSignatureValidations.cpp @@ -51,25 +51,11 @@ bool verifyRootDescriptorFlag(uint32_t Version, uint32_t FlagsVal) { return (Flags | DataFlags) == DataFlags; } -bool verifyRangeType(uint32_t Type) { - switch (Type) { - case llvm::to_underlying(dxbc::DescriptorRangeType::CBV): - case llvm::to_underlying(dxbc::DescriptorRangeType::SRV): - case llvm::to_underlying(dxbc::DescriptorRangeType::UAV): - case llvm::to_underlying(dxbc::DescriptorRangeType::Sampler): - return true; - }; - - return false; -} - -bool verifyDescriptorRangeFlag(uint32_t Version, uint32_t Type, - uint32_t FlagsVal) { +bool verifyDescriptorRangeFlag(uint32_t Version, dxil::ResourceClass Type, + dxbc::DescriptorRangeFlags Flags) { using FlagT = dxbc::DescriptorRangeFlags; - FlagT Flags = FlagT(FlagsVal); - const bool IsSampler = - (Type == llvm::to_underlying(dxbc::DescriptorRangeType::Sampler)); + const bool IsSampler = (Type == dxil::ResourceClass::Sampler); if (Version == 1) { // Since the metadata is unversioned, we expect to explicitly see the values @@ -129,27 +115,6 @@ bool verifyNumDescriptors(uint32_t NumDescriptors) { return NumDescriptors > 0; } -bool verifySamplerFilter(uint32_t Value) { - switch (Value) { -#define FILTER(Num, Val) case llvm::to_underlying(dxbc::SamplerFilter::Val): -#include "llvm/BinaryFormat/DXContainerConstants.def" - return true; - } - return false; -} - -// Values allowed here: -// https://learn.microsoft.com/en-us/windows/win32/api/d3d12/ne-d3d12-d3d12_texture_address_mode#syntax -bool verifyAddress(uint32_t Address) { - switch (Address) { -#define TEXTURE_ADDRESS_MODE(Num, Val) \ - case llvm::to_underlying(dxbc::TextureAddressMode::Val): -#include "llvm/BinaryFormat/DXContainerConstants.def" - return true; - } - return false; -} - bool verifyMipLODBias(float MipLODBias) { return MipLODBias >= -16.f && MipLODBias <= 15.99f; } @@ -158,27 +123,23 @@ bool verifyMaxAnisotropy(uint32_t MaxAnisotropy) { return MaxAnisotropy <= 16u; } -bool verifyComparisonFunc(uint32_t ComparisonFunc) { - switch (ComparisonFunc) { -#define COMPARISON_FUNC(Num, Val) \ - case llvm::to_underlying(dxbc::ComparisonFunc::Val): -#include "llvm/BinaryFormat/DXContainerConstants.def" - return true; - } - return false; +bool verifyLOD(float LOD) { return !std::isnan(LOD); } + +bool verifyBoundOffset(uint32_t Offset) { + return Offset != NumDescriptorsUnbounded; } -bool verifyBorderColor(uint32_t BorderColor) { - switch (BorderColor) { -#define STATIC_BORDER_COLOR(Num, Val) \ - case llvm::to_underlying(dxbc::StaticBorderColor::Val): -#include "llvm/BinaryFormat/DXContainerConstants.def" - return true; - } - return false; +bool verifyNoOverflowedOffset(uint64_t Offset) { + return Offset <= std::numeric_limits<uint32_t>::max(); } -bool verifyLOD(float LOD) { return !std::isnan(LOD); } +uint64_t computeRangeBound(uint32_t Offset, uint32_t Size) { + assert(0 < Size && "Must be a non-empty range"); + if (Size == NumDescriptorsUnbounded) + return NumDescriptorsUnbounded; + + return uint64_t(Offset) + uint64_t(Size) - 1; +} } // namespace rootsig } // namespace hlsl diff --git a/llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp b/llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp index 3d5e487c8990..c955ecd40363 100644 --- a/llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp +++ b/llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp @@ -50,6 +50,7 @@ #include "llvm/Support/CommandLine.h" #include "llvm/Support/ErrorHandling.h" #include "llvm/Support/FileSystem.h" +#include "llvm/Support/VirtualFileSystem.h" #include "llvm/Target/TargetMachine.h" #include "llvm/Target/TargetOptions.h" #include "llvm/Transforms/Utils/BasicBlockUtils.h" @@ -10531,11 +10532,12 @@ void OpenMPIRBuilder::loadOffloadInfoMetadata(Module &M) { } } -void OpenMPIRBuilder::loadOffloadInfoMetadata(StringRef HostFilePath) { +void OpenMPIRBuilder::loadOffloadInfoMetadata(vfs::FileSystem &VFS, + StringRef HostFilePath) { if (HostFilePath.empty()) return; - auto Buf = MemoryBuffer::getFile(HostFilePath); + auto Buf = VFS.getBufferForFile(HostFilePath); if (std::error_code Err = Buf.getError()) { report_fatal_error(("error opening host file from host file path inside of " "OpenMPIRBuilder: " + diff --git a/llvm/lib/IR/AutoUpgrade.cpp b/llvm/lib/IR/AutoUpgrade.cpp index 8034b3ffe273..8d8120ac9ed9 100644 --- a/llvm/lib/IR/AutoUpgrade.cpp +++ b/llvm/lib/IR/AutoUpgrade.cpp @@ -107,6 +107,24 @@ static bool upgradeX86MaskedFPCompare(Function *F, Intrinsic::ID IID, return true; } +// Upgrade the declaration of multiply and add bytes intrinsics whose input +// arguments' types have changed from vectors of i32 to vectors of i8 +static bool upgradeX86MultiplyAddBytes(Function *F, Intrinsic::ID IID, + Function *&NewFn) { + // check if input argument type is a vector of i8 + Type *Arg1Type = F->getFunctionType()->getParamType(1); + Type *Arg2Type = F->getFunctionType()->getParamType(2); + if (Arg1Type->isVectorTy() && + cast<VectorType>(Arg1Type)->getElementType()->isIntegerTy(8) && + Arg2Type->isVectorTy() && + cast<VectorType>(Arg2Type)->getElementType()->isIntegerTy(8)) + return false; + + rename(F); + NewFn = Intrinsic::getOrInsertDeclaration(F->getParent(), IID); + return true; +} + static bool upgradeX86BF16Intrinsic(Function *F, Intrinsic::ID IID, Function *&NewFn) { if (F->getReturnType()->getScalarType()->isBFloatTy()) @@ -546,19 +564,34 @@ static bool upgradeX86IntrinsicFunction(Function *F, StringRef Name, if (ID != Intrinsic::not_intrinsic) return upgradeX86IntrinsicsWith8BitMask(F, ID, NewFn); - if (Name.consume_front("avx512.mask.cmp.")) { - // Added in 7.0 - ID = StringSwitch<Intrinsic::ID>(Name) - .Case("pd.128", Intrinsic::x86_avx512_mask_cmp_pd_128) - .Case("pd.256", Intrinsic::x86_avx512_mask_cmp_pd_256) - .Case("pd.512", Intrinsic::x86_avx512_mask_cmp_pd_512) - .Case("ps.128", Intrinsic::x86_avx512_mask_cmp_ps_128) - .Case("ps.256", Intrinsic::x86_avx512_mask_cmp_ps_256) - .Case("ps.512", Intrinsic::x86_avx512_mask_cmp_ps_512) - .Default(Intrinsic::not_intrinsic); - if (ID != Intrinsic::not_intrinsic) - return upgradeX86MaskedFPCompare(F, ID, NewFn); - return false; // No other 'x86.avx523.mask.cmp.*'. + if (Name.consume_front("avx512.")) { + if (Name.consume_front("mask.cmp.")) { + // Added in 7.0 + ID = StringSwitch<Intrinsic::ID>(Name) + .Case("pd.128", Intrinsic::x86_avx512_mask_cmp_pd_128) + .Case("pd.256", Intrinsic::x86_avx512_mask_cmp_pd_256) + .Case("pd.512", Intrinsic::x86_avx512_mask_cmp_pd_512) + .Case("ps.128", Intrinsic::x86_avx512_mask_cmp_ps_128) + .Case("ps.256", Intrinsic::x86_avx512_mask_cmp_ps_256) + .Case("ps.512", Intrinsic::x86_avx512_mask_cmp_ps_512) + .Default(Intrinsic::not_intrinsic); + if (ID != Intrinsic::not_intrinsic) + return upgradeX86MaskedFPCompare(F, ID, NewFn); + } else if (Name.starts_with("vpdpbusd.") || + Name.starts_with("vpdpbusds.")) { + // Added in 21.1 + ID = StringSwitch<Intrinsic::ID>(Name) + .Case("vpdpbusd.128", Intrinsic::x86_avx512_vpdpbusd_128) + .Case("vpdpbusd.256", Intrinsic::x86_avx512_vpdpbusd_256) + .Case("vpdpbusd.512", Intrinsic::x86_avx512_vpdpbusd_512) + .Case("vpdpbusds.128", Intrinsic::x86_avx512_vpdpbusds_128) + .Case("vpdpbusds.256", Intrinsic::x86_avx512_vpdpbusds_256) + .Case("vpdpbusds.512", Intrinsic::x86_avx512_vpdpbusds_512) + .Default(Intrinsic::not_intrinsic); + if (ID != Intrinsic::not_intrinsic) + return upgradeX86MultiplyAddBytes(F, ID, NewFn); + } + return false; // No other 'x86.avx512.*'. } if (Name.consume_front("avx512bf16.")) { @@ -4149,6 +4182,32 @@ static Value *upgradeX86IntrinsicCall(StringRef Name, CallBase *CI, Function *F, Value *Args[] = {CI->getArgOperand(0), CI->getArgOperand(1), CI->getArgOperand(2)}; + + // Input arguments types were incorrectly set to vectors of i32 before but + // they should be vectors of i8. Insert bit cast when encountering the old + // types + if (Args[1]->getType()->isVectorTy() && + cast<VectorType>(Args[1]->getType()) + ->getElementType() + ->isIntegerTy(32) && + Args[2]->getType()->isVectorTy() && + cast<VectorType>(Args[2]->getType()) + ->getElementType() + ->isIntegerTy(32)) { + Type *NewArgType = nullptr; + if (VecWidth == 128) + NewArgType = VectorType::get(Builder.getInt8Ty(), 16, false); + else if (VecWidth == 256) + NewArgType = VectorType::get(Builder.getInt8Ty(), 32, false); + else if (VecWidth == 512) + NewArgType = VectorType::get(Builder.getInt8Ty(), 64, false); + else + llvm_unreachable("Unexpected vector bit width"); + + Args[1] = Builder.CreateBitCast(Args[1], NewArgType); + Args[2] = Builder.CreateBitCast(Args[2], NewArgType); + } + Rep = Builder.CreateIntrinsic(IID, Args); Value *PassThru = ZeroMask ? ConstantAggregateZero::get(CI->getType()) : CI->getArgOperand(0); @@ -5156,6 +5215,23 @@ void llvm::UpgradeIntrinsicCall(CallBase *CI, Function *NewFn) { CI->eraseFromParent(); return; } + + case Intrinsic::x86_avx512_vpdpbusd_128: + case Intrinsic::x86_avx512_vpdpbusd_256: + case Intrinsic::x86_avx512_vpdpbusd_512: + case Intrinsic::x86_avx512_vpdpbusds_128: + case Intrinsic::x86_avx512_vpdpbusds_256: + case Intrinsic::x86_avx512_vpdpbusds_512: { + unsigned NumElts = CI->getType()->getPrimitiveSizeInBits() / 8; + Value *Args[] = {CI->getArgOperand(0), CI->getArgOperand(1), + CI->getArgOperand(2)}; + Type *NewArgType = VectorType::get(Builder.getInt8Ty(), NumElts, false); + Args[1] = Builder.CreateBitCast(Args[1], NewArgType); + Args[2] = Builder.CreateBitCast(Args[2], NewArgType); + + NewCall = Builder.CreateCall(NewFn, Args); + break; + } } assert(NewCall && "Should have either set this variable or returned through " "the default case"); diff --git a/llvm/lib/IR/DataLayout.cpp b/llvm/lib/IR/DataLayout.cpp index ed629d4e5ea2..77f9b997a2eb 100644 --- a/llvm/lib/IR/DataLayout.cpp +++ b/llvm/lib/IR/DataLayout.cpp @@ -172,18 +172,6 @@ struct LessPointerAddrSpace { }; } // namespace -const char *DataLayout::getManglingComponent(const Triple &T) { - if (T.isOSBinFormatGOFF()) - return "-m:l"; - if (T.isOSBinFormatMachO()) - return "-m:o"; - if ((T.isOSWindows() || T.isUEFI()) && T.isOSBinFormatCOFF()) - return T.getArch() == Triple::x86 ? "-m:x" : "-m:w"; - if (T.isOSBinFormatXCOFF()) - return "-m:a"; - return "-m:e"; -} - // Default primitive type specifications. // NOTE: These arrays must be sorted by type bit width. constexpr DataLayout::PrimitiveSpec DefaultIntSpecs[] = { diff --git a/llvm/lib/IR/DebugLoc.cpp b/llvm/lib/IR/DebugLoc.cpp index 79c5b896f8f2..01dafcab94ce 100644 --- a/llvm/lib/IR/DebugLoc.cpp +++ b/llvm/lib/IR/DebugLoc.cpp @@ -181,10 +181,19 @@ DebugLoc DebugLoc::getMergedLocations(ArrayRef<DebugLoc> Locs) { return Merged; } DebugLoc DebugLoc::getMergedLocation(DebugLoc LocA, DebugLoc LocB) { - if (!LocA) - return LocA; - if (!LocB) + if (!LocA || !LocB) { + // If coverage tracking is enabled, prioritize returning empty non-annotated + // locations to empty annotated locations. +#if LLVM_ENABLE_DEBUGLOC_TRACKING_COVERAGE + if (!LocA && LocA.getKind() == DebugLocKind::Normal) + return LocA; + if (!LocB && LocB.getKind() == DebugLocKind::Normal) + return LocB; +#endif // LLVM_ENABLE_DEBUGLOC_TRACKING_COVERAGE + if (!LocA) + return LocA; return LocB; + } return DILocation::getMergedLocation(LocA, LocB); } diff --git a/llvm/lib/IR/ProfDataUtils.cpp b/llvm/lib/IR/ProfDataUtils.cpp index b41256f59909..5827292cee39 100644 --- a/llvm/lib/IR/ProfDataUtils.cpp +++ b/llvm/lib/IR/ProfDataUtils.cpp @@ -95,6 +95,7 @@ const char *MDProfLabels::FunctionEntryCount = "function_entry_count"; const char *MDProfLabels::SyntheticFunctionEntryCount = "synthetic_function_entry_count"; const char *MDProfLabels::UnknownBranchWeightsMarker = "unknown"; +const char *LLVMLoopEstimatedTripCount = "llvm.loop.estimated_trip_count"; bool hasProfMD(const Instruction &I) { return I.hasMetadata(LLVMContext::MD_prof); @@ -242,24 +243,26 @@ bool extractProfTotalWeight(const Instruction &I, uint64_t &TotalVal) { return extractProfTotalWeight(I.getMetadata(LLVMContext::MD_prof), TotalVal); } -void setExplicitlyUnknownBranchWeights(Instruction &I) { +void setExplicitlyUnknownBranchWeights(Instruction &I, StringRef PassName) { MDBuilder MDB(I.getContext()); I.setMetadata( LLVMContext::MD_prof, MDNode::get(I.getContext(), - MDB.createString(MDProfLabels::UnknownBranchWeightsMarker))); + {MDB.createString(MDProfLabels::UnknownBranchWeightsMarker), + MDB.createString(PassName)})); } -void setExplicitlyUnknownFunctionEntryCount(Function &F) { +void setExplicitlyUnknownFunctionEntryCount(Function &F, StringRef PassName) { MDBuilder MDB(F.getContext()); F.setMetadata( LLVMContext::MD_prof, MDNode::get(F.getContext(), - MDB.createString(MDProfLabels::UnknownBranchWeightsMarker))); + {MDB.createString(MDProfLabels::UnknownBranchWeightsMarker), + MDB.createString(PassName)})); } bool isExplicitlyUnknownProfileMetadata(const MDNode &MD) { - if (MD.getNumOperands() != 1) + if (MD.getNumOperands() != 2) return false; return MD.getOperand(0).equalsStr(MDProfLabels::UnknownBranchWeightsMarker); } diff --git a/llvm/lib/IR/RuntimeLibcalls.cpp b/llvm/lib/IR/RuntimeLibcalls.cpp index 4fe5714a74e3..7ea2e46a5418 100644 --- a/llvm/lib/IR/RuntimeLibcalls.cpp +++ b/llvm/lib/IR/RuntimeLibcalls.cpp @@ -33,31 +33,6 @@ void RuntimeLibcallsInfo::initLibcalls(const Triple &TT, EABI EABIVersion, StringRef ABIName) { setTargetRuntimeLibcallSets(TT, ExceptionModel, FloatABI, EABIVersion, ABIName); - - if (TT.isARM() || TT.isThumb()) { - // The half <-> float conversion functions are always soft-float on - // non-watchos platforms, but are needed for some targets which use a - // hard-float calling convention by default. - if (!TT.isWatchABI()) { - if (isAAPCS_ABI(TT, ABIName)) { - setLibcallImplCallingConv(RTLIB::impl___truncsfhf2, - CallingConv::ARM_AAPCS); - setLibcallImplCallingConv(RTLIB::impl___truncdfhf2, - CallingConv::ARM_AAPCS); - setLibcallImplCallingConv(RTLIB::impl___extendhfsf2, - CallingConv::ARM_AAPCS); - } else { - setLibcallImplCallingConv(RTLIB::impl___truncsfhf2, - CallingConv::ARM_APCS); - setLibcallImplCallingConv(RTLIB::impl___truncdfhf2, - CallingConv::ARM_APCS); - setLibcallImplCallingConv(RTLIB::impl___extendhfsf2, - CallingConv::ARM_APCS); - } - } - - return; - } } LLVM_ATTRIBUTE_ALWAYS_INLINE diff --git a/llvm/lib/IR/Verifier.cpp b/llvm/lib/IR/Verifier.cpp index f38871f09f35..c06b60fd2d9a 100644 --- a/llvm/lib/IR/Verifier.cpp +++ b/llvm/lib/IR/Verifier.cpp @@ -631,7 +631,7 @@ private: void verifyFunctionAttrs(FunctionType *FT, AttributeList Attrs, const Value *V, bool IsIntrinsic, bool IsInlineAsm); void verifyFunctionMetadata(ArrayRef<std::pair<unsigned, MDNode *>> MDs); - + void verifyUnknownProfileMetadata(MDNode *MD); void visitConstantExprsRecursively(const Constant *EntryC); void visitConstantExpr(const ConstantExpr *CE); void visitConstantPtrAuth(const ConstantPtrAuth *CPA); @@ -1076,6 +1076,18 @@ void Verifier::visitMDNode(const MDNode &MD, AreDebugLocsAllowed AllowLocs) { } } + // Check llvm.loop.estimated_trip_count. + if (MD.getNumOperands() > 0 && + MD.getOperand(0).equalsStr(LLVMLoopEstimatedTripCount)) { + Check(MD.getNumOperands() == 2, "Expected two operands", &MD); + auto *Count = dyn_cast_or_null<ConstantAsMetadata>(MD.getOperand(1)); + Check(Count && Count->getType()->isIntegerTy() && + cast<IntegerType>(Count->getType())->getBitWidth() <= 32, + "Expected second operand to be an integer constant of type i32 or " + "smaller", + &MD); + } + // Check these last, so we diagnose problems in operands first. Check(!MD.isTemporary(), "Expected no forward declarations!", &MD); Check(MD.isResolved(), "All nodes should be resolved!", &MD); @@ -2515,19 +2527,31 @@ void Verifier::verifyFunctionAttrs(FunctionType *FT, AttributeList Attrs, V); } } +void Verifier::verifyUnknownProfileMetadata(MDNode *MD) { + Check(MD->getNumOperands() == 2, + "'unknown' !prof should have a single additional operand", MD); + auto *PassName = dyn_cast<MDString>(MD->getOperand(1)); + Check(PassName != nullptr, + "'unknown' !prof should have an additional operand of type " + "string"); + Check(!PassName->getString().empty(), + "the 'unknown' !prof operand should not be an empty string"); +} void Verifier::verifyFunctionMetadata( ArrayRef<std::pair<unsigned, MDNode *>> MDs) { for (const auto &Pair : MDs) { if (Pair.first == LLVMContext::MD_prof) { MDNode *MD = Pair.second; + Check(MD->getNumOperands() >= 2, + "!prof annotations should have no less than 2 operands", MD); // We may have functions that are synthesized by the compiler, e.g. in // WPD, that we can't currently determine the entry count. - if (isExplicitlyUnknownProfileMetadata(*MD)) + if (MD->getOperand(0).equalsStr( + MDProfLabels::UnknownBranchWeightsMarker)) { + verifyUnknownProfileMetadata(MD); continue; - - Check(MD->getNumOperands() >= 2, - "!prof annotations should have no less than 2 operands", MD); + } // Check first operand. Check(MD->getOperand(0) != nullptr, "first operand should not be null", @@ -3185,7 +3209,7 @@ void Verifier::visitFunction(const Function &F) { // Scope and SP could be the same MDNode and we don't want to skip // validation in that case - if (SP && ((Scope != SP) && !Seen.insert(SP).second)) + if ((Scope != SP) && !Seen.insert(SP).second) return; CheckDI(SP->describes(&F), @@ -5054,8 +5078,7 @@ void Verifier::visitProfMetadata(Instruction &I, MDNode *MD) { "'unknown' !prof should only appear on instructions on which " "'branch_weights' would", MD); - Check(MD->getNumOperands() == 1, - "'unknown' !prof should have no additional operands", MD); + verifyUnknownProfileMetadata(MD); return; } diff --git a/llvm/lib/LTO/LTOCodeGenerator.cpp b/llvm/lib/LTO/LTOCodeGenerator.cpp index 09b91d81225a..d8a96f73110f 100644 --- a/llvm/lib/LTO/LTOCodeGenerator.cpp +++ b/llvm/lib/LTO/LTOCodeGenerator.cpp @@ -375,16 +375,12 @@ bool LTOCodeGenerator::determineTarget() { if (TargetMach) return true; - TripleStr = MergedModule->getTargetTriple().str(); - llvm::Triple Triple(TripleStr); - if (TripleStr.empty()) { - TripleStr = sys::getDefaultTargetTriple(); - MergedModule->setTargetTriple(Triple); - } + if (MergedModule->getTargetTriple().empty()) + MergedModule->setTargetTriple(Triple(sys::getDefaultTargetTriple())); // create target machine from info for merged modules std::string ErrMsg; - MArch = TargetRegistry::lookupTarget(TripleStr, ErrMsg); + MArch = TargetRegistry::lookupTarget(MergedModule->getTargetTriple(), ErrMsg); if (!MArch) { emitError(ErrMsg); return false; @@ -393,10 +389,10 @@ bool LTOCodeGenerator::determineTarget() { // Construct LTOModule, hand over ownership of module and target. Use MAttr as // the default set of features. SubtargetFeatures Features(join(Config.MAttrs, "")); - Features.getDefaultSubtargetFeatures(Triple); + Features.getDefaultSubtargetFeatures(MergedModule->getTargetTriple()); FeatureStr = Features.getString(); if (Config.CPU.empty()) - Config.CPU = lto::getThinLTODefaultCPU(Triple); + Config.CPU = lto::getThinLTODefaultCPU(MergedModule->getTargetTriple()); // If data-sections is not explicitly set or unset, set data-sections by // default to match the behaviour of lld and gold plugin. @@ -412,7 +408,7 @@ bool LTOCodeGenerator::determineTarget() { std::unique_ptr<TargetMachine> LTOCodeGenerator::createTargetMachine() { assert(MArch && "MArch is not set!"); return std::unique_ptr<TargetMachine>(MArch->createTargetMachine( - Triple(TripleStr), Config.CPU, FeatureStr, Config.Options, + MergedModule->getTargetTriple(), Config.CPU, FeatureStr, Config.Options, Config.RelocModel, std::nullopt, Config.CGOptLevel)); } diff --git a/llvm/lib/MC/DXContainerRootSignature.cpp b/llvm/lib/MC/DXContainerRootSignature.cpp index ce7d5c91bc7c..b9ebb7a9e789 100644 --- a/llvm/lib/MC/DXContainerRootSignature.cpp +++ b/llvm/lib/MC/DXContainerRootSignature.cpp @@ -137,7 +137,8 @@ void RootSignatureDesc::write(raw_ostream &OS) const { llvm::endianness::little); rewriteOffsetToCurrentByte(BOS, writePlaceholder(BOS)); for (const auto &Range : Table) { - support::endian::write(BOS, Range.RangeType, llvm::endianness::little); + support::endian::write(BOS, static_cast<uint32_t>(Range.RangeType), + llvm::endianness::little); support::endian::write(BOS, Range.NumDescriptors, llvm::endianness::little); support::endian::write(BOS, Range.BaseShaderRegister, diff --git a/llvm/lib/MC/MCDisassembler/Disassembler.cpp b/llvm/lib/MC/MCDisassembler/Disassembler.cpp index ba5e9feb601c..0429227f0fec 100644 --- a/llvm/lib/MC/MCDisassembler/Disassembler.cpp +++ b/llvm/lib/MC/MCDisassembler/Disassembler.cpp @@ -44,13 +44,14 @@ LLVMCreateDisasmCPUFeatures(const char *TT, const char *CPU, const char *Features, void *DisInfo, int TagType, LLVMOpInfoCallback GetOpInfo, LLVMSymbolLookupCallback SymbolLookUp) { + Triple TheTriple(TT); + // Get the target. std::string Error; - const Target *TheTarget = TargetRegistry::lookupTarget(TT, Error); + const Target *TheTarget = TargetRegistry::lookupTarget(TheTriple, Error); if (!TheTarget) return nullptr; - Triple TheTriple(TT); std::unique_ptr<const MCRegisterInfo> MRI( TheTarget->createMCRegInfo(TheTriple)); if (!MRI) diff --git a/llvm/lib/ObjCopy/ConfigManager.cpp b/llvm/lib/ObjCopy/ConfigManager.cpp index 24f3caccf7dc..eef8a2190c4d 100644 --- a/llvm/lib/ObjCopy/ConfigManager.cpp +++ b/llvm/lib/ObjCopy/ConfigManager.cpp @@ -13,6 +13,13 @@ using namespace llvm; using namespace llvm::objcopy; +Expected<const ELFConfig &> ConfigManager::getELFConfig() const { + if (!Common.ExtractSection.empty()) + return createStringError(llvm::errc::invalid_argument, + "option is not supported for ELF"); + return ELF; +} + Expected<const COFFConfig &> ConfigManager::getCOFFConfig() const { if (!Common.SplitDWO.empty() || !Common.SymbolsPrefix.empty() || !Common.SymbolsPrefixRemove.empty() || !Common.SymbolsToSkip.empty() || @@ -27,7 +34,7 @@ Expected<const COFFConfig &> ConfigManager::getCOFFConfig() const { Common.DiscardMode == DiscardType::Locals || !Common.SymbolsToAdd.empty() || Common.GapFill != 0 || Common.PadTo != 0 || Common.ChangeSectionLMAValAll != 0 || - !Common.ChangeSectionAddress.empty()) + !Common.ChangeSectionAddress.empty() || !Common.ExtractSection.empty()) return createStringError(llvm::errc::invalid_argument, "option is not supported for COFF"); @@ -48,7 +55,7 @@ Expected<const MachOConfig &> ConfigManager::getMachOConfig() const { Common.DiscardMode == DiscardType::Locals || !Common.SymbolsToAdd.empty() || Common.GapFill != 0 || Common.PadTo != 0 || Common.ChangeSectionLMAValAll != 0 || - !Common.ChangeSectionAddress.empty()) + !Common.ChangeSectionAddress.empty() || !Common.ExtractSection.empty()) return createStringError(llvm::errc::invalid_argument, "option is not supported for MachO"); @@ -69,7 +76,7 @@ Expected<const WasmConfig &> ConfigManager::getWasmConfig() const { !Common.SetSectionFlags.empty() || !Common.SetSectionType.empty() || !Common.SymbolsToRename.empty() || Common.GapFill != 0 || Common.PadTo != 0 || Common.ChangeSectionLMAValAll != 0 || - !Common.ChangeSectionAddress.empty()) + !Common.ChangeSectionAddress.empty() || !Common.ExtractSection.empty()) return createStringError(llvm::errc::invalid_argument, "only flags for section dumping, removal, and " "addition are supported"); @@ -99,7 +106,7 @@ Expected<const XCOFFConfig &> ConfigManager::getXCOFFConfig() const { Common.Weaken || Common.StripUnneeded || Common.DecompressDebugSections || Common.GapFill != 0 || Common.PadTo != 0 || Common.ChangeSectionLMAValAll != 0 || - !Common.ChangeSectionAddress.empty()) { + !Common.ChangeSectionAddress.empty() || !Common.ExtractSection.empty()) { return createStringError( llvm::errc::invalid_argument, "no flags are supported yet, only basic copying is allowed"); @@ -124,9 +131,8 @@ ConfigManager::getDXContainerConfig() const { Common.DecompressDebugSections || Common.GapFill != 0 || Common.PadTo != 0 || Common.ChangeSectionLMAValAll != 0 || !Common.ChangeSectionAddress.empty()) { - return createStringError( - llvm::errc::invalid_argument, - "no flags are supported yet, only basic copying is allowed"); + return createStringError(llvm::errc::invalid_argument, + "option is not supported for DXContainer"); } return DXContainer; } diff --git a/llvm/lib/ObjCopy/DXContainer/DXContainerObjcopy.cpp b/llvm/lib/ObjCopy/DXContainer/DXContainerObjcopy.cpp index ac97dc462c14..d7f3c0d1f7b3 100644 --- a/llvm/lib/ObjCopy/DXContainer/DXContainerObjcopy.cpp +++ b/llvm/lib/ObjCopy/DXContainer/DXContainerObjcopy.cpp @@ -11,6 +11,7 @@ #include "DXContainerWriter.h" #include "llvm/ObjCopy/CommonConfig.h" #include "llvm/ObjCopy/DXContainer/DXContainerConfig.h" +#include "llvm/Support/raw_ostream.h" namespace llvm { namespace objcopy { @@ -18,7 +19,40 @@ namespace dxbc { using namespace object; +static Error extractPartAsObject(StringRef PartName, StringRef OutFilename, + StringRef InputFilename, const Object &Obj) { + for (const Part &P : Obj.Parts) + if (P.Name == PartName) { + Object PartObj; + PartObj.Header = Obj.Header; + PartObj.Parts.push_back({P.Name, P.Data}); + PartObj.recomputeHeader(); + + auto Write = [&OutFilename, &PartObj](raw_ostream &Out) -> Error { + DXContainerWriter Writer(PartObj, Out); + if (Error E = Writer.write()) + return createFileError(OutFilename, std::move(E)); + return Error::success(); + }; + + return writeToOutput(OutFilename, Write); + } + + return createFileError(InputFilename, object_error::parse_failed, + "part '%s' not found", PartName.str().c_str()); +} + static Error handleArgs(const CommonConfig &Config, Object &Obj) { + // Extract all sections before any modifications. + for (StringRef Flag : Config.ExtractSection) { + StringRef SectionName; + StringRef FileName; + std::tie(SectionName, FileName) = Flag.split('='); + if (Error E = extractPartAsObject(SectionName, FileName, + Config.InputFilename, Obj)) + return E; + } + std::function<bool(const Part &)> RemovePred = [](const Part &) { return false; }; diff --git a/llvm/lib/ObjectYAML/DXContainerEmitter.cpp b/llvm/lib/ObjectYAML/DXContainerEmitter.cpp index fa3abd274480..73dfa9899d61 100644 --- a/llvm/lib/ObjectYAML/DXContainerEmitter.cpp +++ b/llvm/lib/ObjectYAML/DXContainerEmitter.cpp @@ -313,9 +313,10 @@ Error DXContainerWriter::writeParts(raw_ostream &OS) { P.RootSignature->Parameters.getOrInsertTable(L); mcdxbc::DescriptorTable Table; for (const auto &R : TableYaml.Ranges) { - - dxbc::RTS0::v2::DescriptorRange Range; - Range.RangeType = R.RangeType; + assert(dxbc::isValidRangeType(R.RangeType) && + "Invalid Descriptor Range Type"); + mcdxbc::DescriptorRange Range; + Range.RangeType = dxil::ResourceClass(R.RangeType); Range.NumDescriptors = R.NumDescriptors; Range.BaseShaderRegister = R.BaseShaderRegister; Range.RegisterSpace = R.RegisterSpace; @@ -334,20 +335,30 @@ Error DXContainerWriter::writeParts(raw_ostream &OS) { } for (const auto &Param : P.RootSignature->samplers()) { - dxbc::RTS0::v1::StaticSampler NewSampler; - NewSampler.Filter = Param.Filter; - NewSampler.AddressU = Param.AddressU; - NewSampler.AddressV = Param.AddressV; - NewSampler.AddressW = Param.AddressW; + assert(dxbc::isValidSamplerFilter(Param.Filter) && + dxbc::isValidAddress(Param.AddressU) && + dxbc::isValidAddress(Param.AddressV) && + dxbc::isValidAddress(Param.AddressW) && + dxbc::isValidComparisonFunc(Param.ComparisonFunc) && + dxbc::isValidBorderColor(Param.BorderColor) && + dxbc::isValidShaderVisibility(Param.ShaderVisibility) && + "Invalid enum value in static sampler"); + + mcdxbc::StaticSampler NewSampler; + NewSampler.Filter = dxbc::SamplerFilter(Param.Filter); + NewSampler.AddressU = dxbc::TextureAddressMode(Param.AddressU); + NewSampler.AddressV = dxbc::TextureAddressMode(Param.AddressV); + NewSampler.AddressW = dxbc::TextureAddressMode(Param.AddressW); NewSampler.MipLODBias = Param.MipLODBias; NewSampler.MaxAnisotropy = Param.MaxAnisotropy; - NewSampler.ComparisonFunc = Param.ComparisonFunc; - NewSampler.BorderColor = Param.BorderColor; + NewSampler.ComparisonFunc = dxbc::ComparisonFunc(Param.ComparisonFunc); + NewSampler.BorderColor = dxbc::StaticBorderColor(Param.BorderColor); NewSampler.MinLOD = Param.MinLOD; NewSampler.MaxLOD = Param.MaxLOD; NewSampler.ShaderRegister = Param.ShaderRegister; NewSampler.RegisterSpace = Param.RegisterSpace; - NewSampler.ShaderVisibility = Param.ShaderVisibility; + NewSampler.ShaderVisibility = + dxbc::ShaderVisibility(Param.ShaderVisibility); RS.StaticSamplers.push_back(NewSampler); } diff --git a/llvm/lib/Passes/PassBuilder.cpp b/llvm/lib/Passes/PassBuilder.cpp index 587f0ece0859..8cf277657a54 100644 --- a/llvm/lib/Passes/PassBuilder.cpp +++ b/llvm/lib/Passes/PassBuilder.cpp @@ -1493,25 +1493,23 @@ parseBoundsCheckingOptions(StringRef Params) { return Options; } -Expected<CodeGenOptLevel> parseExpandFpOptions(StringRef Params) { - if (Params.empty()) +Expected<CodeGenOptLevel> parseExpandFpOptions(StringRef Param) { + if (Param.empty()) return CodeGenOptLevel::None; - StringRef Param; - std::tie(Param, Params) = Params.split(';'); - if (!Params.empty()) - return createStringError("too many expand-fp pass parameters"); + // Parse a CodeGenOptLevel, e.g. "O1", "O2", "O3". + auto [Prefix, Digit] = Param.split('O'); - auto [Name, Val] = Param.split('='); - if (Name != "opt-level") + uint8_t N; + if (!Prefix.empty() || Digit.getAsInteger(10, N)) return createStringError("invalid expand-fp pass parameter '%s'", Param.str().c_str()); - int8_t N; - Val.getAsInteger(10, N); + std::optional<CodeGenOptLevel> Level = CodeGenOpt::getLevel(N); if (!Level.has_value()) - return createStringError("invalid expand-fp opt-level value: %s", - Val.str().c_str()); + return createStringError( + "invalid optimization level for expand-fp pass: %s", + Digit.str().c_str()); return *Level; } diff --git a/llvm/lib/Passes/PassRegistry.def b/llvm/lib/Passes/PassRegistry.def index 299aaa801439..1d015971dfbd 100644 --- a/llvm/lib/Passes/PassRegistry.def +++ b/llvm/lib/Passes/PassRegistry.def @@ -724,7 +724,7 @@ FUNCTION_PASS_WITH_PARAMS( [TM = TM](CodeGenOptLevel OL) { return ExpandFpPass(TM, OL); }, - parseExpandFpOptions, "opt-level=N") + parseExpandFpOptions, "O0;O1;O2;O3") #undef FUNCTION_PASS_WITH_PARAMS diff --git a/llvm/lib/Remarks/BitstreamRemarkParser.cpp b/llvm/lib/Remarks/BitstreamRemarkParser.cpp index 20a8ebbadc68..86a6c6dffb18 100644 --- a/llvm/lib/Remarks/BitstreamRemarkParser.cpp +++ b/llvm/lib/Remarks/BitstreamRemarkParser.cpp @@ -11,7 +11,6 @@ // //===----------------------------------------------------------------------===// -#include "llvm/Remarks/BitstreamRemarkParser.h" #include "BitstreamRemarkParser.h" #include "llvm/Remarks/Remark.h" #include "llvm/Support/MemoryBuffer.h" diff --git a/llvm/lib/Remarks/BitstreamRemarkParser.h b/llvm/lib/Remarks/BitstreamRemarkParser.h index 061206471fee..cba805dc24b5 100644 --- a/llvm/lib/Remarks/BitstreamRemarkParser.h +++ b/llvm/lib/Remarks/BitstreamRemarkParser.h @@ -13,10 +13,14 @@ #ifndef LLVM_LIB_REMARKS_BITSTREAM_REMARK_PARSER_H #define LLVM_LIB_REMARKS_BITSTREAM_REMARK_PARSER_H +#include "llvm/ADT/ArrayRef.h" +#include "llvm/ADT/StringRef.h" +#include "llvm/Bitstream/BitstreamReader.h" #include "llvm/Remarks/BitstreamRemarkContainer.h" -#include "llvm/Remarks/BitstreamRemarkParser.h" #include "llvm/Remarks/RemarkFormat.h" #include "llvm/Remarks/RemarkParser.h" +#include "llvm/Support/Error.h" +#include <array> #include <cstdint> #include <memory> #include <optional> @@ -26,6 +30,91 @@ namespace remarks { struct Remark; +/// Helper to parse a META_BLOCK for a bitstream remark container. +struct BitstreamMetaParserHelper { + /// The Bitstream reader. + BitstreamCursor &Stream; + /// Reference to the storage for the block info. + BitstreamBlockInfo &BlockInfo; + /// The parsed content: depending on the container type, some fields might be + /// empty. + std::optional<uint64_t> ContainerVersion; + std::optional<uint8_t> ContainerType; + std::optional<StringRef> StrTabBuf; + std::optional<StringRef> ExternalFilePath; + std::optional<uint64_t> RemarkVersion; + + /// Continue parsing with \p Stream. \p Stream is expected to contain a + /// ENTER_SUBBLOCK to the META_BLOCK at the current position. + /// \p Stream is expected to have a BLOCKINFO_BLOCK set. + BitstreamMetaParserHelper(BitstreamCursor &Stream, + BitstreamBlockInfo &BlockInfo); + + /// Parse the META_BLOCK and fill the available entries. + /// This helper does not check for the validity of the fields. + Error parse(); +}; + +/// Helper to parse a REMARK_BLOCK for a bitstream remark container. +struct BitstreamRemarkParserHelper { + /// The Bitstream reader. + BitstreamCursor &Stream; + /// The parsed content: depending on the remark, some fields might be empty. + std::optional<uint8_t> Type; + std::optional<uint64_t> RemarkNameIdx; + std::optional<uint64_t> PassNameIdx; + std::optional<uint64_t> FunctionNameIdx; + std::optional<uint64_t> SourceFileNameIdx; + std::optional<uint32_t> SourceLine; + std::optional<uint32_t> SourceColumn; + std::optional<uint64_t> Hotness; + struct Argument { + std::optional<uint64_t> KeyIdx; + std::optional<uint64_t> ValueIdx; + std::optional<uint64_t> SourceFileNameIdx; + std::optional<uint32_t> SourceLine; + std::optional<uint32_t> SourceColumn; + }; + std::optional<ArrayRef<Argument>> Args; + /// Avoid re-allocating a vector every time. + SmallVector<Argument, 8> TmpArgs; + + /// Continue parsing with \p Stream. \p Stream is expected to contain a + /// ENTER_SUBBLOCK to the REMARK_BLOCK at the current position. + /// \p Stream is expected to have a BLOCKINFO_BLOCK set and to have already + /// parsed the META_BLOCK. + BitstreamRemarkParserHelper(BitstreamCursor &Stream); + + /// Parse the REMARK_BLOCK and fill the available entries. + /// This helper does not check for the validity of the fields. + Error parse(); +}; + +/// Helper to parse any bitstream remark container. +struct BitstreamParserHelper { + /// The Bitstream reader. + BitstreamCursor Stream; + /// The block info block. + BitstreamBlockInfo BlockInfo; + /// Start parsing at \p Buffer. + BitstreamParserHelper(StringRef Buffer); + /// Parse the magic number. + Expected<std::array<char, 4>> parseMagic(); + /// Parse the block info block containing all the abbrevs. + /// This needs to be called before calling any other parsing function. + Error parseBlockInfoBlock(); + /// Return true if the next block is a META_BLOCK. This function does not move + /// the cursor. + Expected<bool> isMetaBlock(); + /// Return true if the next block is a REMARK_BLOCK. This function does not + /// move the cursor. + Expected<bool> isRemarkBlock(); + /// Return true if the parser reached the end of the stream. + bool atEndOfStream() { return Stream.AtEndOfStream(); } + /// Jump to the end of the stream, skipping everything. + void skipToEnd() { return Stream.skipToEnd(); } +}; + /// Parses and holds the state of the latest parsed remark. struct BitstreamRemarkParser : public RemarkParser { /// The buffer to parse. diff --git a/llvm/lib/Support/CMakeLists.txt b/llvm/lib/Support/CMakeLists.txt index 2528e8bd1142..7da972f372c5 100644 --- a/llvm/lib/Support/CMakeLists.txt +++ b/llvm/lib/Support/CMakeLists.txt @@ -135,6 +135,7 @@ if (UNIX AND "${CMAKE_SYSTEM_NAME}" MATCHES "AIX") endif() add_subdirectory(BLAKE3) +add_subdirectory(LSP) add_llvm_component_library(LLVMSupport ABIBreak.cpp diff --git a/llvm/lib/Support/LSP/CMakeLists.txt b/llvm/lib/Support/LSP/CMakeLists.txt new file mode 100644 index 000000000000..6bc9d636fbdf --- /dev/null +++ b/llvm/lib/Support/LSP/CMakeLists.txt @@ -0,0 +1,8 @@ +add_llvm_component_library(LLVMSupportLSP + Protocol.cpp + Transport.cpp + Logging.cpp + + LINK_COMPONENTS + Support +) diff --git a/llvm/lib/Support/LSP/Logging.cpp b/llvm/lib/Support/LSP/Logging.cpp new file mode 100644 index 000000000000..b36621ae1c6c --- /dev/null +++ b/llvm/lib/Support/LSP/Logging.cpp @@ -0,0 +1,41 @@ +//===- Logging.cpp --------------------------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#include "llvm/Support/LSP/Logging.h" +#include "llvm/Support/Chrono.h" +#include "llvm/Support/raw_ostream.h" + +using namespace llvm; +using namespace llvm::lsp; + +void Logger::setLogLevel(Level LogLevel) { get().LogLevel = LogLevel; } + +Logger &Logger::get() { + static Logger Logger; + return Logger; +} + +void Logger::log(Level LogLevel, const char *Fmt, + const llvm::formatv_object_base &Message) { + Logger &Logger = get(); + + // Ignore messages with log levels below the current setting in the logger. + if (LogLevel < Logger.LogLevel) + return; + + // An indicator character for each log level. + const char *LogLevelIndicators = "DIE"; + + // Format the message and print to errs. + llvm::sys::TimePoint<> Timestamp = std::chrono::system_clock::now(); + std::lock_guard<std::mutex> LogGuard(Logger.Mutex); + llvm::errs() << llvm::formatv( + "{0}[{1:%H:%M:%S.%L}] {2}\n", + LogLevelIndicators[static_cast<unsigned>(LogLevel)], Timestamp, Message); + llvm::errs().flush(); +} diff --git a/llvm/lib/Support/LSP/Protocol.cpp b/llvm/lib/Support/LSP/Protocol.cpp new file mode 100644 index 000000000000..f22126345a43 --- /dev/null +++ b/llvm/lib/Support/LSP/Protocol.cpp @@ -0,0 +1,1043 @@ +//===--- Protocol.cpp - Language Server Protocol Implementation -----------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file contains the serialization code for the LSP structs. +// +//===----------------------------------------------------------------------===// + +#include "llvm/Support/LSP/Protocol.h" +#include "llvm/ADT/StringExtras.h" +#include "llvm/ADT/StringSet.h" +#include "llvm/Support/ErrorHandling.h" +#include "llvm/Support/JSON.h" +#include "llvm/Support/MemoryBuffer.h" +#include "llvm/Support/Path.h" +#include "llvm/Support/raw_ostream.h" + +using namespace llvm; +using namespace llvm::lsp; + +// Helper that doesn't treat `null` and absent fields as failures. +template <typename T> +static bool mapOptOrNull(const llvm::json::Value &Params, + llvm::StringLiteral Prop, T &Out, + llvm::json::Path Path) { + const llvm::json::Object *O = Params.getAsObject(); + assert(O); + + // Field is missing or null. + auto *V = O->get(Prop); + if (!V || V->getAsNull()) + return true; + return fromJSON(*V, Out, Path.field(Prop)); +} + +//===----------------------------------------------------------------------===// +// LSPError +//===----------------------------------------------------------------------===// + +char LSPError::ID; + +//===----------------------------------------------------------------------===// +// URIForFile +//===----------------------------------------------------------------------===// + +static bool isWindowsPath(StringRef Path) { + return Path.size() > 1 && llvm::isAlpha(Path[0]) && Path[1] == ':'; +} + +static bool isNetworkPath(StringRef Path) { + return Path.size() > 2 && Path[0] == Path[1] && + llvm::sys::path::is_separator(Path[0]); +} + +static bool shouldEscapeInURI(unsigned char C) { + // Unreserved characters. + if ((C >= 'a' && C <= 'z') || (C >= 'A' && C <= 'Z') || + (C >= '0' && C <= '9')) + return false; + + switch (C) { + case '-': + case '_': + case '.': + case '~': + // '/' is only reserved when parsing. + case '/': + // ':' is only reserved for relative URI paths, which we doesn't produce. + case ':': + return false; + } + return true; +} + +/// Encodes a string according to percent-encoding. +/// - Unreserved characters are not escaped. +/// - Reserved characters always escaped with exceptions like '/'. +/// - All other characters are escaped. +static void percentEncode(StringRef Content, std::string &Out) { + for (unsigned char C : Content) { + if (shouldEscapeInURI(C)) { + Out.push_back('%'); + Out.push_back(llvm::hexdigit(C / 16)); + Out.push_back(llvm::hexdigit(C % 16)); + } else { + Out.push_back(C); + } + } +} + +/// Decodes a string according to percent-encoding. +static std::string percentDecode(StringRef Content) { + std::string Result; + for (auto I = Content.begin(), E = Content.end(); I != E; ++I) { + if (*I != '%') { + Result += *I; + continue; + } + if (*I == '%' && I + 2 < Content.end() && llvm::isHexDigit(*(I + 1)) && + llvm::isHexDigit(*(I + 2))) { + Result.push_back(llvm::hexFromNibbles(*(I + 1), *(I + 2))); + I += 2; + } else { + Result.push_back(*I); + } + } + return Result; +} + +/// Return the set containing the supported URI schemes. +static StringSet<> &getSupportedSchemes() { + static StringSet<> Schemes({"file", "test"}); + return Schemes; +} + +/// Returns true if the given scheme is structurally valid, i.e. it does not +/// contain any invalid scheme characters. This does not check that the scheme +/// is actually supported. +static bool isStructurallyValidScheme(StringRef Scheme) { + if (Scheme.empty()) + return false; + if (!llvm::isAlpha(Scheme[0])) + return false; + return llvm::all_of(llvm::drop_begin(Scheme), [](char C) { + return llvm::isAlnum(C) || C == '+' || C == '.' || C == '-'; + }); +} + +static llvm::Expected<std::string> uriFromAbsolutePath(StringRef AbsolutePath, + StringRef Scheme) { + std::string Body; + StringRef Authority; + StringRef Root = llvm::sys::path::root_name(AbsolutePath); + if (isNetworkPath(Root)) { + // Windows UNC paths e.g. \\server\share => file://server/share + Authority = Root.drop_front(2); + AbsolutePath.consume_front(Root); + } else if (isWindowsPath(Root)) { + // Windows paths e.g. X:\path => file:///X:/path + Body = "/"; + } + Body += llvm::sys::path::convert_to_slash(AbsolutePath); + + std::string Uri = Scheme.str() + ":"; + if (Authority.empty() && Body.empty()) + return Uri; + + // If authority if empty, we only print body if it starts with "/"; otherwise, + // the URI is invalid. + if (!Authority.empty() || StringRef(Body).starts_with("/")) { + Uri.append("//"); + percentEncode(Authority, Uri); + } + percentEncode(Body, Uri); + return Uri; +} + +static llvm::Expected<std::string> getAbsolutePath(StringRef Authority, + StringRef Body) { + if (!Body.starts_with("/")) + return llvm::createStringError( + llvm::inconvertibleErrorCode(), + "File scheme: expect body to be an absolute path starting " + "with '/': " + + Body); + SmallString<128> Path; + if (!Authority.empty()) { + // Windows UNC paths e.g. file://server/share => \\server\share + ("//" + Authority).toVector(Path); + } else if (isWindowsPath(Body.substr(1))) { + // Windows paths e.g. file:///X:/path => X:\path + Body.consume_front("/"); + } + Path.append(Body); + llvm::sys::path::native(Path); + return std::string(Path); +} + +static llvm::Expected<std::string> parseFilePathFromURI(StringRef OrigUri) { + StringRef Uri = OrigUri; + + // Decode the scheme of the URI. + size_t Pos = Uri.find(':'); + if (Pos == StringRef::npos) + return llvm::createStringError(llvm::inconvertibleErrorCode(), + "Scheme must be provided in URI: " + + OrigUri); + StringRef SchemeStr = Uri.substr(0, Pos); + std::string UriScheme = percentDecode(SchemeStr); + if (!isStructurallyValidScheme(UriScheme)) + return llvm::createStringError(llvm::inconvertibleErrorCode(), + "Invalid scheme: " + SchemeStr + + " (decoded: " + UriScheme + ")"); + Uri = Uri.substr(Pos + 1); + + // Decode the authority of the URI. + std::string UriAuthority; + if (Uri.consume_front("//")) { + Pos = Uri.find('/'); + UriAuthority = percentDecode(Uri.substr(0, Pos)); + Uri = Uri.substr(Pos); + } + + // Decode the body of the URI. + std::string UriBody = percentDecode(Uri); + + // Compute the absolute path for this uri. + if (!getSupportedSchemes().contains(UriScheme)) { + return llvm::createStringError(llvm::inconvertibleErrorCode(), + "unsupported URI scheme `" + UriScheme + + "' for workspace files"); + } + return getAbsolutePath(UriAuthority, UriBody); +} + +llvm::Expected<URIForFile> URIForFile::fromURI(StringRef Uri) { + llvm::Expected<std::string> FilePath = parseFilePathFromURI(Uri); + if (!FilePath) + return FilePath.takeError(); + return URIForFile(std::move(*FilePath), Uri.str()); +} + +llvm::Expected<URIForFile> URIForFile::fromFile(StringRef AbsoluteFilepath, + StringRef Scheme) { + llvm::Expected<std::string> Uri = + uriFromAbsolutePath(AbsoluteFilepath, Scheme); + if (!Uri) + return Uri.takeError(); + return fromURI(*Uri); +} + +StringRef URIForFile::scheme() const { return uri().split(':').first; } + +void URIForFile::registerSupportedScheme(StringRef Scheme) { + getSupportedSchemes().insert(Scheme); +} + +bool llvm::lsp::fromJSON(const llvm::json::Value &Value, URIForFile &Result, + llvm::json::Path Path) { + if (std::optional<StringRef> Str = Value.getAsString()) { + llvm::Expected<URIForFile> ExpectedUri = URIForFile::fromURI(*Str); + if (!ExpectedUri) { + Path.report("unresolvable URI"); + consumeError(ExpectedUri.takeError()); + return false; + } + Result = std::move(*ExpectedUri); + return true; + } + return false; +} + +llvm::json::Value llvm::lsp::toJSON(const URIForFile &Value) { + return Value.uri(); +} + +raw_ostream &llvm::lsp::operator<<(raw_ostream &Os, const URIForFile &Value) { + return Os << Value.uri(); +} + +//===----------------------------------------------------------------------===// +// ClientCapabilities +//===----------------------------------------------------------------------===// + +bool llvm::lsp::fromJSON(const llvm::json::Value &Value, + ClientCapabilities &Result, llvm::json::Path Path) { + const llvm::json::Object *O = Value.getAsObject(); + if (!O) { + Path.report("expected object"); + return false; + } + if (const llvm::json::Object *TextDocument = O->getObject("textDocument")) { + if (const llvm::json::Object *DocumentSymbol = + TextDocument->getObject("documentSymbol")) { + if (std::optional<bool> HierarchicalSupport = + DocumentSymbol->getBoolean("hierarchicalDocumentSymbolSupport")) + Result.hierarchicalDocumentSymbol = *HierarchicalSupport; + } + if (auto *CodeAction = TextDocument->getObject("codeAction")) { + if (CodeAction->getObject("codeActionLiteralSupport")) + Result.codeActionStructure = true; + } + } + if (auto *Window = O->getObject("window")) { + if (std::optional<bool> WorkDoneProgressSupport = + Window->getBoolean("workDoneProgress")) + Result.workDoneProgress = *WorkDoneProgressSupport; + } + return true; +} + +//===----------------------------------------------------------------------===// +// ClientInfo +//===----------------------------------------------------------------------===// + +bool llvm::lsp::fromJSON(const llvm::json::Value &Value, ClientInfo &Result, + llvm::json::Path Path) { + llvm::json::ObjectMapper O(Value, Path); + if (!O || !O.map("name", Result.name)) + return false; + + // Don't fail if we can't parse version. + O.map("version", Result.version); + return true; +} + +//===----------------------------------------------------------------------===// +// InitializeParams +//===----------------------------------------------------------------------===// + +bool llvm::lsp::fromJSON(const llvm::json::Value &Value, TraceLevel &Result, + llvm::json::Path Path) { + if (std::optional<StringRef> Str = Value.getAsString()) { + if (*Str == "off") { + Result = TraceLevel::Off; + return true; + } + if (*Str == "messages") { + Result = TraceLevel::Messages; + return true; + } + if (*Str == "verbose") { + Result = TraceLevel::Verbose; + return true; + } + } + return false; +} + +bool llvm::lsp::fromJSON(const llvm::json::Value &Value, + InitializeParams &Result, llvm::json::Path Path) { + llvm::json::ObjectMapper O(Value, Path); + if (!O) + return false; + // We deliberately don't fail if we can't parse individual fields. + O.map("capabilities", Result.capabilities); + O.map("trace", Result.trace); + mapOptOrNull(Value, "clientInfo", Result.clientInfo, Path); + + return true; +} + +//===----------------------------------------------------------------------===// +// TextDocumentItem +//===----------------------------------------------------------------------===// + +bool llvm::lsp::fromJSON(const llvm::json::Value &Value, + TextDocumentItem &Result, llvm::json::Path Path) { + llvm::json::ObjectMapper O(Value, Path); + return O && O.map("uri", Result.uri) && + O.map("languageId", Result.languageId) && O.map("text", Result.text) && + O.map("version", Result.version); +} + +//===----------------------------------------------------------------------===// +// TextDocumentIdentifier +//===----------------------------------------------------------------------===// + +llvm::json::Value llvm::lsp::toJSON(const TextDocumentIdentifier &Value) { + return llvm::json::Object{{"uri", Value.uri}}; +} + +bool llvm::lsp::fromJSON(const llvm::json::Value &Value, + TextDocumentIdentifier &Result, + llvm::json::Path Path) { + llvm::json::ObjectMapper O(Value, Path); + return O && O.map("uri", Result.uri); +} + +//===----------------------------------------------------------------------===// +// VersionedTextDocumentIdentifier +//===----------------------------------------------------------------------===// + +llvm::json::Value +llvm::lsp::toJSON(const VersionedTextDocumentIdentifier &Value) { + return llvm::json::Object{ + {"uri", Value.uri}, + {"version", Value.version}, + }; +} + +bool llvm::lsp::fromJSON(const llvm::json::Value &Value, + VersionedTextDocumentIdentifier &Result, + llvm::json::Path Path) { + llvm::json::ObjectMapper O(Value, Path); + return O && O.map("uri", Result.uri) && O.map("version", Result.version); +} + +//===----------------------------------------------------------------------===// +// Position +//===----------------------------------------------------------------------===// + +bool llvm::lsp::fromJSON(const llvm::json::Value &Value, Position &Result, + llvm::json::Path Path) { + llvm::json::ObjectMapper O(Value, Path); + return O && O.map("line", Result.line) && + O.map("character", Result.character); +} + +llvm::json::Value llvm::lsp::toJSON(const Position &Value) { + return llvm::json::Object{ + {"line", Value.line}, + {"character", Value.character}, + }; +} + +raw_ostream &llvm::lsp::operator<<(raw_ostream &Os, const Position &Value) { + return Os << Value.line << ':' << Value.character; +} + +//===----------------------------------------------------------------------===// +// Range +//===----------------------------------------------------------------------===// + +bool llvm::lsp::fromJSON(const llvm::json::Value &Value, Range &Result, + llvm::json::Path Path) { + llvm::json::ObjectMapper O(Value, Path); + return O && O.map("start", Result.start) && O.map("end", Result.end); +} + +llvm::json::Value llvm::lsp::toJSON(const Range &Value) { + return llvm::json::Object{ + {"start", Value.start}, + {"end", Value.end}, + }; +} + +raw_ostream &llvm::lsp::operator<<(raw_ostream &Os, const Range &Value) { + return Os << Value.start << '-' << Value.end; +} + +//===----------------------------------------------------------------------===// +// Location +//===----------------------------------------------------------------------===// + +bool llvm::lsp::fromJSON(const llvm::json::Value &Value, Location &Result, + llvm::json::Path Path) { + llvm::json::ObjectMapper O(Value, Path); + return O && O.map("uri", Result.uri) && O.map("range", Result.range); +} + +llvm::json::Value llvm::lsp::toJSON(const Location &Value) { + return llvm::json::Object{ + {"uri", Value.uri}, + {"range", Value.range}, + }; +} + +raw_ostream &llvm::lsp::operator<<(raw_ostream &Os, const Location &Value) { + return Os << Value.range << '@' << Value.uri; +} + +//===----------------------------------------------------------------------===// +// TextDocumentPositionParams +//===----------------------------------------------------------------------===// + +bool llvm::lsp::fromJSON(const llvm::json::Value &Value, + TextDocumentPositionParams &Result, + llvm::json::Path Path) { + llvm::json::ObjectMapper O(Value, Path); + return O && O.map("textDocument", Result.textDocument) && + O.map("position", Result.position); +} + +//===----------------------------------------------------------------------===// +// ReferenceParams +//===----------------------------------------------------------------------===// + +bool llvm::lsp::fromJSON(const llvm::json::Value &Value, + ReferenceContext &Result, llvm::json::Path Path) { + llvm::json::ObjectMapper O(Value, Path); + return O && O.mapOptional("includeDeclaration", Result.includeDeclaration); +} + +bool llvm::lsp::fromJSON(const llvm::json::Value &Value, + ReferenceParams &Result, llvm::json::Path Path) { + TextDocumentPositionParams &Base = Result; + llvm::json::ObjectMapper O(Value, Path); + return fromJSON(Value, Base, Path) && O && + O.mapOptional("context", Result.context); +} + +//===----------------------------------------------------------------------===// +// DidOpenTextDocumentParams +//===----------------------------------------------------------------------===// + +bool llvm::lsp::fromJSON(const llvm::json::Value &Value, + DidOpenTextDocumentParams &Result, + llvm::json::Path Path) { + llvm::json::ObjectMapper O(Value, Path); + return O && O.map("textDocument", Result.textDocument); +} + +//===----------------------------------------------------------------------===// +// DidCloseTextDocumentParams +//===----------------------------------------------------------------------===// + +bool llvm::lsp::fromJSON(const llvm::json::Value &Value, + DidCloseTextDocumentParams &Result, + llvm::json::Path Path) { + llvm::json::ObjectMapper O(Value, Path); + return O && O.map("textDocument", Result.textDocument); +} + +//===----------------------------------------------------------------------===// +// DidChangeTextDocumentParams +//===----------------------------------------------------------------------===// + +LogicalResult +TextDocumentContentChangeEvent::applyTo(std::string &Contents) const { + // If there is no range, the full document changed. + if (!range) { + Contents = text; + return success(); + } + + // Try to map the replacement range to the content. + llvm::SourceMgr TmpScrMgr; + TmpScrMgr.AddNewSourceBuffer(llvm::MemoryBuffer::getMemBuffer(Contents), + SMLoc()); + SMRange RangeLoc = range->getAsSMRange(TmpScrMgr); + if (!RangeLoc.isValid()) + return failure(); + + Contents.replace(RangeLoc.Start.getPointer() - Contents.data(), + RangeLoc.End.getPointer() - RangeLoc.Start.getPointer(), + text); + return success(); +} + +LogicalResult TextDocumentContentChangeEvent::applyTo( + ArrayRef<TextDocumentContentChangeEvent> Changes, std::string &Contents) { + for (const auto &Change : Changes) + if (failed(Change.applyTo(Contents))) + return failure(); + return success(); +} + +bool llvm::lsp::fromJSON(const llvm::json::Value &Value, + TextDocumentContentChangeEvent &Result, + llvm::json::Path Path) { + llvm::json::ObjectMapper O(Value, Path); + return O && O.map("range", Result.range) && + O.map("rangeLength", Result.rangeLength) && O.map("text", Result.text); +} + +bool llvm::lsp::fromJSON(const llvm::json::Value &Value, + DidChangeTextDocumentParams &Result, + llvm::json::Path Path) { + llvm::json::ObjectMapper O(Value, Path); + return O && O.map("textDocument", Result.textDocument) && + O.map("contentChanges", Result.contentChanges); +} + +//===----------------------------------------------------------------------===// +// MarkupContent +//===----------------------------------------------------------------------===// + +static llvm::StringRef toTextKind(MarkupKind Kind) { + switch (Kind) { + case MarkupKind::PlainText: + return "plaintext"; + case MarkupKind::Markdown: + return "markdown"; + } + llvm_unreachable("Invalid MarkupKind"); +} + +raw_ostream &llvm::lsp::operator<<(raw_ostream &Os, MarkupKind Kind) { + return Os << toTextKind(Kind); +} + +llvm::json::Value llvm::lsp::toJSON(const MarkupContent &Mc) { + if (Mc.value.empty()) + return nullptr; + + return llvm::json::Object{ + {"kind", toTextKind(Mc.kind)}, + {"value", Mc.value}, + }; +} + +//===----------------------------------------------------------------------===// +// Hover +//===----------------------------------------------------------------------===// + +llvm::json::Value llvm::lsp::toJSON(const Hover &Hover) { + llvm::json::Object Result{{"contents", toJSON(Hover.contents)}}; + if (Hover.range) + Result["range"] = toJSON(*Hover.range); + return std::move(Result); +} + +//===----------------------------------------------------------------------===// +// DocumentSymbol +//===----------------------------------------------------------------------===// + +llvm::json::Value llvm::lsp::toJSON(const DocumentSymbol &Symbol) { + llvm::json::Object Result{{"name", Symbol.name}, + {"kind", static_cast<int>(Symbol.kind)}, + {"range", Symbol.range}, + {"selectionRange", Symbol.selectionRange}}; + + if (!Symbol.detail.empty()) + Result["detail"] = Symbol.detail; + if (!Symbol.children.empty()) + Result["children"] = Symbol.children; + return std::move(Result); +} + +//===----------------------------------------------------------------------===// +// DocumentSymbolParams +//===----------------------------------------------------------------------===// + +bool llvm::lsp::fromJSON(const llvm::json::Value &Value, + DocumentSymbolParams &Result, llvm::json::Path Path) { + llvm::json::ObjectMapper O(Value, Path); + return O && O.map("textDocument", Result.textDocument); +} + +//===----------------------------------------------------------------------===// +// DiagnosticRelatedInformation +//===----------------------------------------------------------------------===// + +bool llvm::lsp::fromJSON(const llvm::json::Value &Value, + DiagnosticRelatedInformation &Result, + llvm::json::Path Path) { + llvm::json::ObjectMapper O(Value, Path); + return O && O.map("location", Result.location) && + O.map("message", Result.message); +} + +llvm::json::Value llvm::lsp::toJSON(const DiagnosticRelatedInformation &Info) { + return llvm::json::Object{ + {"location", Info.location}, + {"message", Info.message}, + }; +} + +//===----------------------------------------------------------------------===// +// Diagnostic +//===----------------------------------------------------------------------===// + +llvm::json::Value llvm::lsp::toJSON(DiagnosticTag Tag) { + return static_cast<int>(Tag); +} + +bool llvm::lsp::fromJSON(const llvm::json::Value &Value, DiagnosticTag &Result, + llvm::json::Path Path) { + if (std::optional<int64_t> I = Value.getAsInteger()) { + Result = (DiagnosticTag)*I; + return true; + } + + return false; +} + +llvm::json::Value llvm::lsp::toJSON(const Diagnostic &Diag) { + llvm::json::Object Result{ + {"range", Diag.range}, + {"severity", (int)Diag.severity}, + {"message", Diag.message}, + }; + if (Diag.category) + Result["category"] = *Diag.category; + if (!Diag.source.empty()) + Result["source"] = Diag.source; + if (Diag.relatedInformation) + Result["relatedInformation"] = *Diag.relatedInformation; + if (!Diag.tags.empty()) + Result["tags"] = Diag.tags; + return std::move(Result); +} + +bool llvm::lsp::fromJSON(const llvm::json::Value &Value, Diagnostic &Result, + llvm::json::Path Path) { + llvm::json::ObjectMapper O(Value, Path); + if (!O) + return false; + int Severity = 0; + if (!mapOptOrNull(Value, "severity", Severity, Path)) + return false; + Result.severity = (DiagnosticSeverity)Severity; + + return O.map("range", Result.range) && O.map("message", Result.message) && + mapOptOrNull(Value, "category", Result.category, Path) && + mapOptOrNull(Value, "source", Result.source, Path) && + mapOptOrNull(Value, "relatedInformation", Result.relatedInformation, + Path) && + mapOptOrNull(Value, "tags", Result.tags, Path); +} + +//===----------------------------------------------------------------------===// +// PublishDiagnosticsParams +//===----------------------------------------------------------------------===// + +llvm::json::Value llvm::lsp::toJSON(const PublishDiagnosticsParams &Params) { + return llvm::json::Object{ + {"uri", Params.uri}, + {"diagnostics", Params.diagnostics}, + {"version", Params.version}, + }; +} + +//===----------------------------------------------------------------------===// +// TextEdit +//===----------------------------------------------------------------------===// + +bool llvm::lsp::fromJSON(const llvm::json::Value &Value, TextEdit &Result, + llvm::json::Path Path) { + llvm::json::ObjectMapper O(Value, Path); + return O && O.map("range", Result.range) && O.map("newText", Result.newText); +} + +llvm::json::Value llvm::lsp::toJSON(const TextEdit &Value) { + return llvm::json::Object{ + {"range", Value.range}, + {"newText", Value.newText}, + }; +} + +raw_ostream &llvm::lsp::operator<<(raw_ostream &Os, const TextEdit &Value) { + Os << Value.range << " => \""; + llvm::printEscapedString(Value.newText, Os); + return Os << '"'; +} + +//===----------------------------------------------------------------------===// +// CompletionItemKind +//===----------------------------------------------------------------------===// + +bool llvm::lsp::fromJSON(const llvm::json::Value &Value, + CompletionItemKind &Result, llvm::json::Path Path) { + if (std::optional<int64_t> IntValue = Value.getAsInteger()) { + if (*IntValue < static_cast<int>(CompletionItemKind::Text) || + *IntValue > static_cast<int>(CompletionItemKind::TypeParameter)) + return false; + Result = static_cast<CompletionItemKind>(*IntValue); + return true; + } + return false; +} + +CompletionItemKind llvm::lsp::adjustKindToCapability( + CompletionItemKind Kind, + CompletionItemKindBitset &SupportedCompletionItemKinds) { + size_t KindVal = static_cast<size_t>(Kind); + if (KindVal >= kCompletionItemKindMin && + KindVal <= SupportedCompletionItemKinds.size() && + SupportedCompletionItemKinds[KindVal]) + return Kind; + + // Provide some fall backs for common kinds that are close enough. + switch (Kind) { + case CompletionItemKind::Folder: + return CompletionItemKind::File; + case CompletionItemKind::EnumMember: + return CompletionItemKind::Enum; + case CompletionItemKind::Struct: + return CompletionItemKind::Class; + default: + return CompletionItemKind::Text; + } +} + +bool llvm::lsp::fromJSON(const llvm::json::Value &Value, + CompletionItemKindBitset &Result, + llvm::json::Path Path) { + if (const llvm::json::Array *ArrayValue = Value.getAsArray()) { + for (size_t I = 0, E = ArrayValue->size(); I < E; ++I) { + CompletionItemKind KindOut; + if (fromJSON((*ArrayValue)[I], KindOut, Path.index(I))) + Result.set(size_t(KindOut)); + } + return true; + } + return false; +} + +//===----------------------------------------------------------------------===// +// CompletionItem +//===----------------------------------------------------------------------===// + +llvm::json::Value llvm::lsp::toJSON(const CompletionItem &Value) { + assert(!Value.label.empty() && "completion item label is required"); + llvm::json::Object Result{{"label", Value.label}}; + if (Value.kind != CompletionItemKind::Missing) + Result["kind"] = static_cast<int>(Value.kind); + if (!Value.detail.empty()) + Result["detail"] = Value.detail; + if (Value.documentation) + Result["documentation"] = Value.documentation; + if (!Value.sortText.empty()) + Result["sortText"] = Value.sortText; + if (!Value.filterText.empty()) + Result["filterText"] = Value.filterText; + if (!Value.insertText.empty()) + Result["insertText"] = Value.insertText; + if (Value.insertTextFormat != InsertTextFormat::Missing) + Result["insertTextFormat"] = static_cast<int>(Value.insertTextFormat); + if (Value.textEdit) + Result["textEdit"] = *Value.textEdit; + if (!Value.additionalTextEdits.empty()) { + Result["additionalTextEdits"] = + llvm::json::Array(Value.additionalTextEdits); + } + if (Value.deprecated) + Result["deprecated"] = Value.deprecated; + return std::move(Result); +} + +raw_ostream &llvm::lsp::operator<<(raw_ostream &Os, + const CompletionItem &Value) { + return Os << Value.label << " - " << toJSON(Value); +} + +bool llvm::lsp::operator<(const CompletionItem &Lhs, + const CompletionItem &Rhs) { + return (Lhs.sortText.empty() ? Lhs.label : Lhs.sortText) < + (Rhs.sortText.empty() ? Rhs.label : Rhs.sortText); +} + +//===----------------------------------------------------------------------===// +// CompletionList +//===----------------------------------------------------------------------===// + +llvm::json::Value llvm::lsp::toJSON(const CompletionList &Value) { + return llvm::json::Object{ + {"isIncomplete", Value.isIncomplete}, + {"items", llvm::json::Array(Value.items)}, + }; +} + +//===----------------------------------------------------------------------===// +// CompletionContext +//===----------------------------------------------------------------------===// + +bool llvm::lsp::fromJSON(const llvm::json::Value &Value, + CompletionContext &Result, llvm::json::Path Path) { + llvm::json::ObjectMapper O(Value, Path); + int TriggerKind; + if (!O || !O.map("triggerKind", TriggerKind) || + !mapOptOrNull(Value, "triggerCharacter", Result.triggerCharacter, Path)) + return false; + Result.triggerKind = static_cast<CompletionTriggerKind>(TriggerKind); + return true; +} + +//===----------------------------------------------------------------------===// +// CompletionParams +//===----------------------------------------------------------------------===// + +bool llvm::lsp::fromJSON(const llvm::json::Value &Value, + CompletionParams &Result, llvm::json::Path Path) { + if (!fromJSON(Value, static_cast<TextDocumentPositionParams &>(Result), Path)) + return false; + if (const llvm::json::Value *Context = Value.getAsObject()->get("context")) + return fromJSON(*Context, Result.context, Path.field("context")); + return true; +} + +//===----------------------------------------------------------------------===// +// ParameterInformation +//===----------------------------------------------------------------------===// + +llvm::json::Value llvm::lsp::toJSON(const ParameterInformation &Value) { + assert((Value.labelOffsets || !Value.labelString.empty()) && + "parameter information label is required"); + llvm::json::Object Result; + if (Value.labelOffsets) + Result["label"] = llvm::json::Array( + {Value.labelOffsets->first, Value.labelOffsets->second}); + else + Result["label"] = Value.labelString; + if (!Value.documentation.empty()) + Result["documentation"] = Value.documentation; + return std::move(Result); +} + +//===----------------------------------------------------------------------===// +// SignatureInformation +//===----------------------------------------------------------------------===// + +llvm::json::Value llvm::lsp::toJSON(const SignatureInformation &Value) { + assert(!Value.label.empty() && "signature information label is required"); + llvm::json::Object Result{ + {"label", Value.label}, + {"parameters", llvm::json::Array(Value.parameters)}, + }; + if (!Value.documentation.empty()) + Result["documentation"] = Value.documentation; + return std::move(Result); +} + +raw_ostream &llvm::lsp::operator<<(raw_ostream &Os, + const SignatureInformation &Value) { + return Os << Value.label << " - " << toJSON(Value); +} + +//===----------------------------------------------------------------------===// +// SignatureHelp +//===----------------------------------------------------------------------===// + +llvm::json::Value llvm::lsp::toJSON(const SignatureHelp &Value) { + assert(Value.activeSignature >= 0 && + "Unexpected negative value for number of active signatures."); + assert(Value.activeParameter >= 0 && + "Unexpected negative value for active parameter index"); + return llvm::json::Object{ + {"activeSignature", Value.activeSignature}, + {"activeParameter", Value.activeParameter}, + {"signatures", llvm::json::Array(Value.signatures)}, + }; +} + +//===----------------------------------------------------------------------===// +// DocumentLinkParams +//===----------------------------------------------------------------------===// + +bool llvm::lsp::fromJSON(const llvm::json::Value &Value, + DocumentLinkParams &Result, llvm::json::Path Path) { + llvm::json::ObjectMapper O(Value, Path); + return O && O.map("textDocument", Result.textDocument); +} + +//===----------------------------------------------------------------------===// +// DocumentLink +//===----------------------------------------------------------------------===// + +llvm::json::Value llvm::lsp::toJSON(const DocumentLink &Value) { + return llvm::json::Object{ + {"range", Value.range}, + {"target", Value.target}, + }; +} + +//===----------------------------------------------------------------------===// +// InlayHintsParams +//===----------------------------------------------------------------------===// + +bool llvm::lsp::fromJSON(const llvm::json::Value &Value, + InlayHintsParams &Result, llvm::json::Path Path) { + llvm::json::ObjectMapper O(Value, Path); + return O && O.map("textDocument", Result.textDocument) && + O.map("range", Result.range); +} + +//===----------------------------------------------------------------------===// +// InlayHint +//===----------------------------------------------------------------------===// + +llvm::json::Value llvm::lsp::toJSON(const InlayHint &Value) { + return llvm::json::Object{{"position", Value.position}, + {"kind", (int)Value.kind}, + {"label", Value.label}, + {"paddingLeft", Value.paddingLeft}, + {"paddingRight", Value.paddingRight}}; +} +bool llvm::lsp::operator==(const InlayHint &Lhs, const InlayHint &Rhs) { + return std::tie(Lhs.position, Lhs.kind, Lhs.label) == + std::tie(Rhs.position, Rhs.kind, Rhs.label); +} +bool llvm::lsp::operator<(const InlayHint &Lhs, const InlayHint &Rhs) { + return std::tie(Lhs.position, Lhs.kind, Lhs.label) < + std::tie(Rhs.position, Rhs.kind, Rhs.label); +} + +llvm::raw_ostream &llvm::lsp::operator<<(llvm::raw_ostream &Os, + InlayHintKind Value) { + switch (Value) { + case InlayHintKind::Parameter: + return Os << "parameter"; + case InlayHintKind::Type: + return Os << "type"; + } + llvm_unreachable("Unknown InlayHintKind"); +} + +//===----------------------------------------------------------------------===// +// CodeActionContext +//===----------------------------------------------------------------------===// + +bool llvm::lsp::fromJSON(const llvm::json::Value &Value, + CodeActionContext &Result, llvm::json::Path Path) { + llvm::json::ObjectMapper O(Value, Path); + if (!O || !O.map("diagnostics", Result.diagnostics)) + return false; + O.map("only", Result.only); + return true; +} + +//===----------------------------------------------------------------------===// +// CodeActionParams +//===----------------------------------------------------------------------===// + +bool llvm::lsp::fromJSON(const llvm::json::Value &Value, + CodeActionParams &Result, llvm::json::Path Path) { + llvm::json::ObjectMapper O(Value, Path); + return O && O.map("textDocument", Result.textDocument) && + O.map("range", Result.range) && O.map("context", Result.context); +} + +//===----------------------------------------------------------------------===// +// WorkspaceEdit +//===----------------------------------------------------------------------===// + +bool llvm::lsp::fromJSON(const llvm::json::Value &Value, WorkspaceEdit &Result, + llvm::json::Path Path) { + llvm::json::ObjectMapper O(Value, Path); + return O && O.map("changes", Result.changes); +} + +llvm::json::Value llvm::lsp::toJSON(const WorkspaceEdit &Value) { + llvm::json::Object FileChanges; + for (auto &Change : Value.changes) + FileChanges[Change.first] = llvm::json::Array(Change.second); + return llvm::json::Object{{"changes", std::move(FileChanges)}}; +} + +//===----------------------------------------------------------------------===// +// CodeAction +//===----------------------------------------------------------------------===// + +const llvm::StringLiteral CodeAction::kQuickFix = "quickfix"; +const llvm::StringLiteral CodeAction::kRefactor = "refactor"; +const llvm::StringLiteral CodeAction::kInfo = "info"; + +llvm::json::Value llvm::lsp::toJSON(const CodeAction &Value) { + llvm::json::Object CodeAction{{"title", Value.title}}; + if (Value.kind) + CodeAction["kind"] = *Value.kind; + if (Value.diagnostics) + CodeAction["diagnostics"] = llvm::json::Array(*Value.diagnostics); + if (Value.isPreferred) + CodeAction["isPreferred"] = true; + if (Value.edit) + CodeAction["edit"] = *Value.edit; + return std::move(CodeAction); +} diff --git a/llvm/lib/Support/LSP/Transport.cpp b/llvm/lib/Support/LSP/Transport.cpp new file mode 100644 index 000000000000..e71f17701636 --- /dev/null +++ b/llvm/lib/Support/LSP/Transport.cpp @@ -0,0 +1,369 @@ +//===--- JSONTransport.cpp - sending and receiving LSP messages over JSON -===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#include "llvm/Support/LSP/Transport.h" +#include "llvm/ADT/SmallString.h" +#include "llvm/Support/Error.h" +#include "llvm/Support/LSP/Logging.h" +#include "llvm/Support/LSP/Protocol.h" +#include <atomic> +#include <optional> +#include <system_error> +#include <utility> + +using namespace llvm; +using namespace llvm::lsp; + +//===----------------------------------------------------------------------===// +// Reply +//===----------------------------------------------------------------------===// + +namespace { +/// Function object to reply to an LSP call. +/// Each instance must be called exactly once, otherwise: +/// - if there was no reply, an error reply is sent +/// - if there were multiple replies, only the first is sent +class Reply { +public: + Reply(const llvm::json::Value &Id, StringRef Method, JSONTransport &Transport, + std::mutex &TransportOutputMutex); + Reply(Reply &&Other); + Reply &operator=(Reply &&) = delete; + Reply(const Reply &) = delete; + Reply &operator=(const Reply &) = delete; + + void operator()(llvm::Expected<llvm::json::Value> Reply); + +private: + std::string Method; + std::atomic<bool> Replied = {false}; + llvm::json::Value Id; + JSONTransport *Transport; + std::mutex &TransportOutputMutex; +}; +} // namespace + +Reply::Reply(const llvm::json::Value &Id, llvm::StringRef Method, + JSONTransport &Transport, std::mutex &TransportOutputMutex) + : Method(Method), Id(Id), Transport(&Transport), + TransportOutputMutex(TransportOutputMutex) {} + +Reply::Reply(Reply &&Other) + : Method(Other.Method), Replied(Other.Replied.load()), + Id(std::move(Other.Id)), Transport(Other.Transport), + TransportOutputMutex(Other.TransportOutputMutex) { + Other.Transport = nullptr; +} + +void Reply::operator()(llvm::Expected<llvm::json::Value> Reply) { + if (Replied.exchange(true)) { + Logger::error("Replied twice to message {0}({1})", Method, Id); + assert(false && "must reply to each call only once!"); + return; + } + assert(Transport && "expected valid transport to reply to"); + + std::lock_guard<std::mutex> TransportLock(TransportOutputMutex); + if (Reply) { + Logger::info("--> reply:{0}({1})", Method, Id); + Transport->reply(std::move(Id), std::move(Reply)); + } else { + llvm::Error Error = Reply.takeError(); + Logger::info("--> reply:{0}({1}): {2}", Method, Id, Error); + Transport->reply(std::move(Id), std::move(Error)); + } +} + +//===----------------------------------------------------------------------===// +// MessageHandler +//===----------------------------------------------------------------------===// + +bool MessageHandler::onNotify(llvm::StringRef Method, llvm::json::Value Value) { + Logger::info("--> {0}", Method); + + if (Method == "exit") + return false; + if (Method == "$cancel") { + // TODO: Add support for cancelling requests. + } else { + auto It = NotificationHandlers.find(Method); + if (It != NotificationHandlers.end()) + It->second(std::move(Value)); + } + return true; +} + +bool MessageHandler::onCall(llvm::StringRef Method, llvm::json::Value Params, + llvm::json::Value Id) { + Logger::info("--> {0}({1})", Method, Id); + + Reply Reply(Id, Method, Transport, TransportOutputMutex); + + auto It = MethodHandlers.find(Method); + if (It != MethodHandlers.end()) { + It->second(std::move(Params), std::move(Reply)); + } else { + Reply(llvm::make_error<LSPError>("method not found: " + Method.str(), + ErrorCode::MethodNotFound)); + } + return true; +} + +bool MessageHandler::onReply(llvm::json::Value Id, + llvm::Expected<llvm::json::Value> Result) { + // Find the response handler in the mapping. If it exists, move it out of the + // mapping and erase it. + ResponseHandlerTy ResponseHandler; + { + std::lock_guard<std::mutex> responseHandlersLock(ResponseHandlerTy); + auto It = ResponseHandlers.find(debugString(Id)); + if (It != ResponseHandlers.end()) { + ResponseHandler = std::move(It->second); + ResponseHandlers.erase(It); + } + } + + // If we found a response handler, invoke it. Otherwise, log an error. + if (ResponseHandler.second) { + Logger::info("--> reply:{0}({1})", ResponseHandler.first, Id); + ResponseHandler.second(std::move(Id), std::move(Result)); + } else { + Logger::error( + "received a reply with ID {0}, but there was no such outgoing request", + Id); + if (!Result) + llvm::consumeError(Result.takeError()); + } + return true; +} + +//===----------------------------------------------------------------------===// +// JSONTransport +//===----------------------------------------------------------------------===// + +/// Encode the given error as a JSON object. +static llvm::json::Object encodeError(llvm::Error Error) { + std::string Message; + ErrorCode Code = ErrorCode::UnknownErrorCode; + auto HandlerFn = [&](const LSPError &LspError) -> llvm::Error { + Message = LspError.message; + Code = LspError.code; + return llvm::Error::success(); + }; + if (llvm::Error Unhandled = llvm::handleErrors(std::move(Error), HandlerFn)) + Message = llvm::toString(std::move(Unhandled)); + + return llvm::json::Object{ + {"message", std::move(Message)}, + {"code", int64_t(Code)}, + }; +} + +/// Decode the given JSON object into an error. +llvm::Error decodeError(const llvm::json::Object &O) { + StringRef Msg = O.getString("message").value_or("Unspecified error"); + if (std::optional<int64_t> Code = O.getInteger("code")) + return llvm::make_error<LSPError>(Msg.str(), ErrorCode(*Code)); + return llvm::make_error<llvm::StringError>(llvm::inconvertibleErrorCode(), + Msg.str()); +} + +void JSONTransport::notify(StringRef Method, llvm::json::Value Params) { + sendMessage(llvm::json::Object{ + {"jsonrpc", "2.0"}, + {"method", Method}, + {"params", std::move(Params)}, + }); +} +void JSONTransport::call(StringRef Method, llvm::json::Value Params, + llvm::json::Value Id) { + sendMessage(llvm::json::Object{ + {"jsonrpc", "2.0"}, + {"id", std::move(Id)}, + {"method", Method}, + {"params", std::move(Params)}, + }); +} +void JSONTransport::reply(llvm::json::Value Id, + llvm::Expected<llvm::json::Value> Result) { + if (Result) { + return sendMessage(llvm::json::Object{ + {"jsonrpc", "2.0"}, + {"id", std::move(Id)}, + {"result", std::move(*Result)}, + }); + } + + sendMessage(llvm::json::Object{ + {"jsonrpc", "2.0"}, + {"id", std::move(Id)}, + {"error", encodeError(Result.takeError())}, + }); +} + +llvm::Error JSONTransport::run(MessageHandler &Handler) { + std::string Json; + while (!In->isEndOfInput()) { + if (In->hasError()) { + return llvm::errorCodeToError( + std::error_code(errno, std::system_category())); + } + + if (succeeded(In->readMessage(Json))) { + if (llvm::Expected<llvm::json::Value> Doc = llvm::json::parse(Json)) { + if (!handleMessage(std::move(*Doc), Handler)) + return llvm::Error::success(); + } else { + Logger::error("JSON parse error: {0}", llvm::toString(Doc.takeError())); + } + } + } + return llvm::errorCodeToError(std::make_error_code(std::errc::io_error)); +} + +void JSONTransport::sendMessage(llvm::json::Value Msg) { + OutputBuffer.clear(); + llvm::raw_svector_ostream os(OutputBuffer); + os << llvm::formatv(PrettyOutput ? "{0:2}\n" : "{0}", Msg); + Out << "Content-Length: " << OutputBuffer.size() << "\r\n\r\n" + << OutputBuffer; + Out.flush(); + Logger::debug(">>> {0}\n", OutputBuffer); +} + +bool JSONTransport::handleMessage(llvm::json::Value Msg, + MessageHandler &Handler) { + // Message must be an object with "jsonrpc":"2.0". + llvm::json::Object *Object = Msg.getAsObject(); + if (!Object || + Object->getString("jsonrpc") != std::optional<StringRef>("2.0")) + return false; + + // `id` may be any JSON value. If absent, this is a notification. + std::optional<llvm::json::Value> Id; + if (llvm::json::Value *I = Object->get("id")) + Id = std::move(*I); + std::optional<StringRef> Method = Object->getString("method"); + + // This is a response. + if (!Method) { + if (!Id) + return false; + if (auto *Err = Object->getObject("error")) + return Handler.onReply(std::move(*Id), decodeError(*Err)); + // result should be given, use null if not. + llvm::json::Value Result = nullptr; + if (llvm::json::Value *R = Object->get("result")) + Result = std::move(*R); + return Handler.onReply(std::move(*Id), std::move(Result)); + } + + // Params should be given, use null if not. + llvm::json::Value Params = nullptr; + if (llvm::json::Value *P = Object->get("params")) + Params = std::move(*P); + + if (Id) + return Handler.onCall(*Method, std::move(Params), std::move(*Id)); + return Handler.onNotify(*Method, std::move(Params)); +} + +/// Tries to read a line up to and including \n. +/// If failing, feof(), ferror(), or shutdownRequested() will be set. +LogicalResult readLine(std::FILE *In, SmallVectorImpl<char> &Out) { + // Big enough to hold any reasonable header line. May not fit content lines + // in delimited mode, but performance doesn't matter for that mode. + static constexpr int BufSize = 128; + size_t Size = 0; + Out.clear(); + for (;;) { + Out.resize_for_overwrite(Size + BufSize); + if (!std::fgets(&Out[Size], BufSize, In)) + return failure(); + + clearerr(In); + + // If the line contained null bytes, anything after it (including \n) will + // be ignored. Fortunately this is not a legal header or JSON. + size_t Read = std::strlen(&Out[Size]); + if (Read > 0 && Out[Size + Read - 1] == '\n') { + Out.resize(Size + Read); + return success(); + } + Size += Read; + } +} + +// Returns std::nullopt when: +// - ferror(), feof(), or shutdownRequested() are set. +// - Content-Length is missing or empty (protocol error) +LogicalResult +JSONTransportInputOverFile::readStandardMessage(std::string &Json) { + // A Language Server Protocol message starts with a set of HTTP headers, + // delimited by \r\n, and terminated by an empty line (\r\n). + unsigned long long ContentLength = 0; + llvm::SmallString<128> Line; + while (true) { + if (feof(In) || hasError() || failed(readLine(In, Line))) + return failure(); + + // Content-Length is a mandatory header, and the only one we handle. + StringRef LineRef = Line; + if (LineRef.consume_front("Content-Length: ")) { + llvm::getAsUnsignedInteger(LineRef.trim(), 0, ContentLength); + } else if (!LineRef.trim().empty()) { + // It's another header, ignore it. + continue; + } else { + // An empty line indicates the end of headers. Go ahead and read the JSON. + break; + } + } + + // The fuzzer likes crashing us by sending "Content-Length: 9999999999999999" + if (ContentLength == 0 || ContentLength > 1 << 30) + return failure(); + + Json.resize(ContentLength); + for (size_t Pos = 0, Read; Pos < ContentLength; Pos += Read) { + Read = std::fread(&Json[Pos], 1, ContentLength - Pos, In); + if (Read == 0) + return failure(); + + // If we're done, the error was transient. If we're not done, either it was + // transient or we'll see it again on retry. + clearerr(In); + Pos += Read; + } + return success(); +} + +/// For lit tests we support a simplified syntax: +/// - messages are delimited by '// -----' on a line by itself +/// - lines starting with // are ignored. +/// This is a testing path, so favor simplicity over performance here. +/// When returning failure: feof(), ferror(), or shutdownRequested() will be +/// set. +LogicalResult +JSONTransportInputOverFile::readDelimitedMessage(std::string &Json) { + Json.clear(); + llvm::SmallString<128> Line; + while (succeeded(readLine(In, Line))) { + StringRef LineRef = Line.str().trim(); + if (LineRef.starts_with("//")) { + // Found a delimiter for the message. + if (LineRef == "// -----") + break; + continue; + } + + Json += Line; + } + + return failure(ferror(In)); +} diff --git a/llvm/lib/Target/AArch64/AArch64AsmPrinter.cpp b/llvm/lib/Target/AArch64/AArch64AsmPrinter.cpp index fa050526b722..c31a090bba77 100644 --- a/llvm/lib/Target/AArch64/AArch64AsmPrinter.cpp +++ b/llvm/lib/Target/AArch64/AArch64AsmPrinter.cpp @@ -307,6 +307,7 @@ private: /// Emit instruction to set float register to zero. void emitFMov0(const MachineInstr &MI); + void emitFMov0AsFMov(const MachineInstr &MI, Register DestReg); using MInstToMCSymbol = std::map<const MachineInstr *, MCSymbol *>; @@ -1829,45 +1830,77 @@ void AArch64AsmPrinter::emitMOVK(Register Dest, uint64_t Imm, unsigned Shift) { void AArch64AsmPrinter::emitFMov0(const MachineInstr &MI) { Register DestReg = MI.getOperand(0).getReg(); - if (STI->hasZeroCycleZeroingFPR64() && - !STI->hasZeroCycleZeroingFPWorkaround() && STI->isNeonAvailable()) { - // Convert H/S register to corresponding D register - if (AArch64::H0 <= DestReg && DestReg <= AArch64::H31) - DestReg = AArch64::D0 + (DestReg - AArch64::H0); - else if (AArch64::S0 <= DestReg && DestReg <= AArch64::S31) - DestReg = AArch64::D0 + (DestReg - AArch64::S0); - else - assert(AArch64::D0 <= DestReg && DestReg <= AArch64::D31); + if (!STI->hasZeroCycleZeroingFPWorkaround() && STI->isNeonAvailable()) { + if (STI->hasZeroCycleZeroingFPR64()) { + // Convert H/S register to corresponding D register + const AArch64RegisterInfo *TRI = STI->getRegisterInfo(); + if (AArch64::FPR16RegClass.contains(DestReg)) + DestReg = TRI->getMatchingSuperReg(DestReg, AArch64::hsub, + &AArch64::FPR64RegClass); + else if (AArch64::FPR32RegClass.contains(DestReg)) + DestReg = TRI->getMatchingSuperReg(DestReg, AArch64::ssub, + &AArch64::FPR64RegClass); + else + assert(AArch64::FPR64RegClass.contains(DestReg)); + + MCInst MOVI; + MOVI.setOpcode(AArch64::MOVID); + MOVI.addOperand(MCOperand::createReg(DestReg)); + MOVI.addOperand(MCOperand::createImm(0)); + EmitToStreamer(*OutStreamer, MOVI); + } else if (STI->hasZeroCycleZeroingFPR128()) { + // Convert H/S/D register to corresponding Q register + const AArch64RegisterInfo *TRI = STI->getRegisterInfo(); + if (AArch64::FPR16RegClass.contains(DestReg)) { + DestReg = TRI->getMatchingSuperReg(DestReg, AArch64::hsub, + &AArch64::FPR128RegClass); + } else if (AArch64::FPR32RegClass.contains(DestReg)) { + DestReg = TRI->getMatchingSuperReg(DestReg, AArch64::ssub, + &AArch64::FPR128RegClass); + } else { + assert(AArch64::FPR64RegClass.contains(DestReg)); + DestReg = TRI->getMatchingSuperReg(DestReg, AArch64::dsub, + &AArch64::FPR128RegClass); + } - MCInst MOVI; - MOVI.setOpcode(AArch64::MOVID); - MOVI.addOperand(MCOperand::createReg(DestReg)); - MOVI.addOperand(MCOperand::createImm(0)); - EmitToStreamer(*OutStreamer, MOVI); - } else { - MCInst FMov; - switch (MI.getOpcode()) { - default: llvm_unreachable("Unexpected opcode"); - case AArch64::FMOVH0: - FMov.setOpcode(STI->hasFullFP16() ? AArch64::FMOVWHr : AArch64::FMOVWSr); - if (!STI->hasFullFP16()) - DestReg = (AArch64::S0 + (DestReg - AArch64::H0)); - FMov.addOperand(MCOperand::createReg(DestReg)); - FMov.addOperand(MCOperand::createReg(AArch64::WZR)); - break; - case AArch64::FMOVS0: - FMov.setOpcode(AArch64::FMOVWSr); - FMov.addOperand(MCOperand::createReg(DestReg)); - FMov.addOperand(MCOperand::createReg(AArch64::WZR)); - break; - case AArch64::FMOVD0: - FMov.setOpcode(AArch64::FMOVXDr); - FMov.addOperand(MCOperand::createReg(DestReg)); - FMov.addOperand(MCOperand::createReg(AArch64::XZR)); - break; + MCInst MOVI; + MOVI.setOpcode(AArch64::MOVIv2d_ns); + MOVI.addOperand(MCOperand::createReg(DestReg)); + MOVI.addOperand(MCOperand::createImm(0)); + EmitToStreamer(*OutStreamer, MOVI); + } else { + emitFMov0AsFMov(MI, DestReg); } - EmitToStreamer(*OutStreamer, FMov); + } else { + emitFMov0AsFMov(MI, DestReg); + } +} + +void AArch64AsmPrinter::emitFMov0AsFMov(const MachineInstr &MI, + Register DestReg) { + MCInst FMov; + switch (MI.getOpcode()) { + default: + llvm_unreachable("Unexpected opcode"); + case AArch64::FMOVH0: + FMov.setOpcode(STI->hasFullFP16() ? AArch64::FMOVWHr : AArch64::FMOVWSr); + if (!STI->hasFullFP16()) + DestReg = (AArch64::S0 + (DestReg - AArch64::H0)); + FMov.addOperand(MCOperand::createReg(DestReg)); + FMov.addOperand(MCOperand::createReg(AArch64::WZR)); + break; + case AArch64::FMOVS0: + FMov.setOpcode(AArch64::FMOVWSr); + FMov.addOperand(MCOperand::createReg(DestReg)); + FMov.addOperand(MCOperand::createReg(AArch64::WZR)); + break; + case AArch64::FMOVD0: + FMov.setOpcode(AArch64::FMOVXDr); + FMov.addOperand(MCOperand::createReg(DestReg)); + FMov.addOperand(MCOperand::createReg(AArch64::XZR)); + break; } + EmitToStreamer(*OutStreamer, FMov); } Register AArch64AsmPrinter::emitPtrauthDiscriminator(uint16_t Disc, diff --git a/llvm/lib/Target/AArch64/AArch64Combine.td b/llvm/lib/Target/AArch64/AArch64Combine.td index 5f499e5e9700..076a6235eef0 100644 --- a/llvm/lib/Target/AArch64/AArch64Combine.td +++ b/llvm/lib/Target/AArch64/AArch64Combine.td @@ -369,5 +369,5 @@ def AArch64PostLegalizerCombiner commute_constant_to_rhs, extract_vec_elt_combines, push_freeze_to_prevent_poison_from_propagating, combine_mul_cmlt, combine_use_vector_truncate, - extmultomull, truncsat_combines]> { + extmultomull, truncsat_combines, lshr_of_trunc_of_lshr]> { } diff --git a/llvm/lib/Target/AArch64/AArch64Features.td b/llvm/lib/Target/AArch64/AArch64Features.td index 6904e0907264..46f5f0c1ca9d 100644 --- a/llvm/lib/Target/AArch64/AArch64Features.td +++ b/llvm/lib/Target/AArch64/AArch64Features.td @@ -636,6 +636,9 @@ def FeatureZCZeroingGPR64 : SubtargetFeature<"zcz-gpr64", "HasZeroCycleZeroingGP def FeatureZCZeroingGPR32 : SubtargetFeature<"zcz-gpr32", "HasZeroCycleZeroingGPR32", "true", "Has zero-cycle zeroing instructions for GPR32 registers">; +def FeatureZCZeroingFPR128 : SubtargetFeature<"zcz-fpr128", "HasZeroCycleZeroingFPR128", "true", + "Has zero-cycle zeroing instructions for FPR128 registers">; + // It is generally beneficial to rewrite "fmov s0, wzr" to "movi d0, #0". // as movi is more efficient across all cores. Newer cores can eliminate // fmovs early and there is no difference with movi, but this not true for diff --git a/llvm/lib/Target/AArch64/AArch64FrameLowering.cpp b/llvm/lib/Target/AArch64/AArch64FrameLowering.cpp index 87a09b72933d..175b5e04d82f 100644 --- a/llvm/lib/Target/AArch64/AArch64FrameLowering.cpp +++ b/llvm/lib/Target/AArch64/AArch64FrameLowering.cpp @@ -211,6 +211,7 @@ #include "AArch64FrameLowering.h" #include "AArch64InstrInfo.h" #include "AArch64MachineFunctionInfo.h" +#include "AArch64PrologueEpilogue.h" #include "AArch64RegisterInfo.h" #include "AArch64Subtarget.h" #include "MCTargetDesc/AArch64AddressingModes.h" @@ -218,7 +219,6 @@ #include "Utils/AArch64SMEAttributes.h" #include "llvm/ADT/ScopeExit.h" #include "llvm/ADT/SmallVector.h" -#include "llvm/ADT/Statistic.h" #include "llvm/Analysis/ValueTracking.h" #include "llvm/CodeGen/CFIInstBuilder.h" #include "llvm/CodeGen/LivePhysRegs.h" @@ -293,8 +293,6 @@ static cl::opt<bool> DisableMultiVectorSpillFill( cl::desc("Disable use of LD/ST pairs for SME2 or SVE2p1"), cl::init(false), cl::Hidden); -STATISTIC(NumRedZoneFunctions, "Number of functions using red zone"); - /// Returns how much of the incoming argument stack area (in bytes) we should /// clean up in an epilogue. For the C calling convention this will be 0, for /// guaranteed tail call conventions it can be positive (a normal return or a @@ -328,23 +326,20 @@ static int64_t getArgumentStackToRestore(MachineFunction &MF, return ArgumentPopSize; } -static bool produceCompactUnwindFrame(MachineFunction &MF); -static bool needsWinCFI(const MachineFunction &MF); -static StackOffset getSVEStackSize(const MachineFunction &MF); -static Register findScratchNonCalleeSaveRegister(MachineBasicBlock *MBB, - bool HasCall = false); -static bool requiresSaveVG(const MachineFunction &MF); +static bool produceCompactUnwindFrame(const AArch64FrameLowering &, + MachineFunction &MF); // Conservatively, returns true if the function is likely to have an SVE vectors // on the stack. This function is safe to be called before callee-saves or // object offsets have been determined. -static bool isLikelyToHaveSVEStack(const MachineFunction &MF) { +static bool isLikelyToHaveSVEStack(const AArch64FrameLowering &AFL, + const MachineFunction &MF) { auto *AFI = MF.getInfo<AArch64FunctionInfo>(); if (AFI->isSVECC()) return true; if (AFI->hasCalculatedStackSizeSVE()) - return bool(getSVEStackSize(MF)); + return bool(AFL.getSVEStackSize(MF)); const MachineFrameInfo &MFI = MF.getFrameInfo(); for (int FI = MFI.getObjectIndexBegin(); FI < MFI.getObjectIndexEnd(); FI++) { @@ -372,7 +367,7 @@ bool AArch64FrameLowering::homogeneousPrologEpilog( return false; // TODO: SVE is not supported yet. - if (isLikelyToHaveSVEStack(MF)) + if (isLikelyToHaveSVEStack(*this, MF)) return false; // Bail on stack adjustment needed on return for simplicity. @@ -409,7 +404,7 @@ bool AArch64FrameLowering::homogeneousPrologEpilog( /// Returns true if CSRs should be paired. bool AArch64FrameLowering::producePairRegisters(MachineFunction &MF) const { - return produceCompactUnwindFrame(MF) || homogeneousPrologEpilog(MF); + return produceCompactUnwindFrame(*this, MF) || homogeneousPrologEpilog(MF); } /// This is the biggest offset to the stack pointer we can encode in aarch64 @@ -451,11 +446,10 @@ AArch64FrameLowering::getStackIDForScalableVectors() const { return TargetStackID::ScalableVector; } -/// Returns the size of the fixed object area (allocated next to sp on entry) -/// On Win64 this may include a var args area and an UnwindHelp object for EH. -static unsigned getFixedObjectSize(const MachineFunction &MF, - const AArch64FunctionInfo *AFI, bool IsWin64, - bool IsFunclet) { +unsigned +AArch64FrameLowering::getFixedObjectSize(const MachineFunction &MF, + const AArch64FunctionInfo *AFI, + bool IsWin64, bool IsFunclet) const { assert(AFI->getTailCallReservedStack() % 16 == 0 && "Tail call reserved stack must be aligned to 16 bytes"); if (!IsWin64 || IsFunclet) { @@ -494,7 +488,8 @@ static unsigned getFixedObjectSize(const MachineFunction &MF, } /// Returns the size of the entire SVE stackframe (calleesaves + spills). -static StackOffset getSVEStackSize(const MachineFunction &MF) { +StackOffset +AArch64FrameLowering::getSVEStackSize(const MachineFunction &MF) const { const AArch64FunctionInfo *AFI = MF.getInfo<AArch64FunctionInfo>(); return StackOffset::getScalable((int64_t)AFI->getStackSizeSVE()); } @@ -683,70 +678,6 @@ MachineBasicBlock::iterator AArch64FrameLowering::eliminateCallFramePseudoInstr( return MBB.erase(I); } -void AArch64FrameLowering::emitCalleeSavedGPRLocations( - MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI) const { - MachineFunction &MF = *MBB.getParent(); - MachineFrameInfo &MFI = MF.getFrameInfo(); - - const std::vector<CalleeSavedInfo> &CSI = MFI.getCalleeSavedInfo(); - if (CSI.empty()) - return; - - CFIInstBuilder CFIBuilder(MBB, MBBI, MachineInstr::FrameSetup); - for (const auto &Info : CSI) { - unsigned FrameIdx = Info.getFrameIdx(); - if (MFI.getStackID(FrameIdx) == TargetStackID::ScalableVector) - continue; - - assert(!Info.isSpilledToReg() && "Spilling to registers not implemented"); - int64_t Offset = MFI.getObjectOffset(FrameIdx) - getOffsetOfLocalArea(); - CFIBuilder.buildOffset(Info.getReg(), Offset); - } -} - -void AArch64FrameLowering::emitCalleeSavedSVELocations( - MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI) const { - MachineFunction &MF = *MBB.getParent(); - MachineFrameInfo &MFI = MF.getFrameInfo(); - - // Add callee saved registers to move list. - const std::vector<CalleeSavedInfo> &CSI = MFI.getCalleeSavedInfo(); - if (CSI.empty()) - return; - - const TargetSubtargetInfo &STI = MF.getSubtarget(); - const TargetRegisterInfo &TRI = *STI.getRegisterInfo(); - AArch64FunctionInfo &AFI = *MF.getInfo<AArch64FunctionInfo>(); - CFIInstBuilder CFIBuilder(MBB, MBBI, MachineInstr::FrameSetup); - - std::optional<int64_t> IncomingVGOffsetFromDefCFA; - if (requiresSaveVG(MF)) { - auto IncomingVG = *find_if( - reverse(CSI), [](auto &Info) { return Info.getReg() == AArch64::VG; }); - IncomingVGOffsetFromDefCFA = - MFI.getObjectOffset(IncomingVG.getFrameIdx()) - getOffsetOfLocalArea(); - } - - for (const auto &Info : CSI) { - if (MFI.getStackID(Info.getFrameIdx()) != TargetStackID::ScalableVector) - continue; - - // Not all unwinders may know about SVE registers, so assume the lowest - // common denominator. - assert(!Info.isSpilledToReg() && "Spilling to registers not implemented"); - MCRegister Reg = Info.getReg(); - if (!static_cast<const AArch64RegisterInfo &>(TRI).regNeedsCFI(Reg, Reg)) - continue; - - StackOffset Offset = - StackOffset::getScalable(MFI.getObjectOffset(Info.getFrameIdx())) - - StackOffset::getFixed(AFI.getCalleeSavedStackSize(MFI)); - - CFIBuilder.insertCFIInst( - createCFAOffset(TRI, Reg, Offset, IncomingVGOffsetFromDefCFA)); - } -} - void AArch64FrameLowering::resetCFIToInitialState( MachineBasicBlock &MBB) const { @@ -1088,8 +1019,8 @@ void AArch64FrameLowering::emitZeroCallUsedRegs(BitVector RegsToZero, } } -static bool windowsRequiresStackProbe(const MachineFunction &MF, - uint64_t StackSizeInBytes) { +bool AArch64FrameLowering::windowsRequiresStackProbe( + const MachineFunction &MF, uint64_t StackSizeInBytes) const { const AArch64Subtarget &Subtarget = MF.getSubtarget<AArch64Subtarget>(); const AArch64FunctionInfo &MFI = *MF.getInfo<AArch64FunctionInfo>(); // TODO: When implementing stack protectors, take that into account @@ -1108,19 +1039,9 @@ static void getLiveRegsForEntryMBB(LivePhysRegs &LiveRegs, LiveRegs.addReg(CSRegs[i]); } -// Find a scratch register that we can use at the start of the prologue to -// re-align the stack pointer. We avoid using callee-save registers since they -// may appear to be free when this is called from canUseAsPrologue (during -// shrink wrapping), but then no longer be free when this is called from -// emitPrologue. -// -// FIXME: This is a bit conservative, since in the above case we could use one -// of the callee-save registers as a scratch temp to re-align the stack pointer, -// but we would then have to make sure that we were in fact saving at least one -// callee-save register in the prologue, which is additional complexity that -// doesn't seem worth the benefit. -static Register findScratchNonCalleeSaveRegister(MachineBasicBlock *MBB, - bool HasCall) { +Register +AArch64FrameLowering::findScratchNonCalleeSaveRegister(MachineBasicBlock *MBB, + bool HasCall) const { MachineFunction *MF = MBB->getParent(); // If MBB is an entry block, use X9 as the scratch register @@ -1193,13 +1114,14 @@ bool AArch64FrameLowering::canUseAsPrologue( return true; } -static bool needsWinCFI(const MachineFunction &MF) { +bool AArch64FrameLowering::needsWinCFI(const MachineFunction &MF) const { const Function &F = MF.getFunction(); return MF.getTarget().getMCAsmInfo()->usesWindowsCFI() && F.needsUnwindTableEntry(); } -static bool shouldSignReturnAddressEverywhere(const MachineFunction &MF) { +bool AArch64FrameLowering::shouldSignReturnAddressEverywhere( + const MachineFunction &MF) const { // FIXME: With WinCFI, extra care should be taken to place SEH_PACSignLR // and SEH_EpilogEnd instructions in the correct order. if (MF.getTarget().getMCAsmInfo()->usesWindowsCFI()) @@ -1475,13 +1397,13 @@ static void fixupSEHOpcode(MachineBasicBlock::iterator MBBI, ImmOpnd->setImm(ImmOpnd->getImm() + LocalStackSize); } -bool requiresGetVGCall(MachineFunction &MF) { - AArch64FunctionInfo *AFI = MF.getInfo<AArch64FunctionInfo>(); +bool AArch64FrameLowering::requiresGetVGCall(const MachineFunction &MF) const { + auto *AFI = MF.getInfo<AArch64FunctionInfo>(); return AFI->hasStreamingModeChanges() && !MF.getSubtarget<AArch64Subtarget>().hasSVE(); } -static bool requiresSaveVG(const MachineFunction &MF) { +bool AArch64FrameLowering::requiresSaveVG(const MachineFunction &MF) const { const AArch64FunctionInfo *AFI = MF.getInfo<AArch64FunctionInfo>(); if (!AFI->needsDwarfUnwindInfo(MF) || !AFI->hasStreamingModeChanges()) return false; @@ -1499,8 +1421,8 @@ static bool matchLibcall(const TargetLowering &TLI, const MachineOperand &MO, StringRef(TLI.getLibcallName(LC)) == MO.getSymbolName(); } -bool isVGInstruction(MachineBasicBlock::iterator MBBI, - const TargetLowering &TLI) { +bool AArch64FrameLowering::isVGInstruction(MachineBasicBlock::iterator MBBI, + const TargetLowering &TLI) const { unsigned Opc = MBBI->getOpcode(); if (Opc == AArch64::CNTD_XPiI) return true; @@ -1514,15 +1436,12 @@ bool isVGInstruction(MachineBasicBlock::iterator MBBI, return Opc == TargetOpcode::COPY; } -// Convert callee-save register save/restore instruction to do stack pointer -// decrement/increment to allocate/deallocate the callee-save stack area by -// converting store/load to use pre/post increment version. -static MachineBasicBlock::iterator convertCalleeSaveRestoreToSPPrePostIncDec( +MachineBasicBlock::iterator +AArch64FrameLowering::convertCalleeSaveRestoreToSPPrePostIncDec( MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI, const DebugLoc &DL, const TargetInstrInfo *TII, int CSStackSizeInc, bool NeedsWinCFI, bool *HasWinCFI, bool EmitCFI, - MachineInstr::MIFlag FrameFlag = MachineInstr::FrameSetup, - int CFAOffset = 0) { + MachineInstr::MIFlag FrameFlag, int CFAOffset) const { unsigned NewOpc; // If the function contains streaming mode changes, we expect instructions @@ -1643,12 +1562,9 @@ static MachineBasicBlock::iterator convertCalleeSaveRestoreToSPPrePostIncDec( return std::prev(MBB.erase(MBBI)); } -// Fixup callee-save register save/restore instructions to take into account -// combined SP bump by adding the local stack size to the stack offsets. -static void fixupCalleeSaveRestoreStackOffset(MachineInstr &MI, - uint64_t LocalStackSize, - bool NeedsWinCFI, - bool *HasWinCFI) { +void AArch64FrameLowering::fixupCalleeSaveRestoreStackOffset( + MachineInstr &MI, uint64_t LocalStackSize, bool NeedsWinCFI, + bool *HasWinCFI) const { if (AArch64InstrInfo::isSEHInstruction(MI)) return; @@ -1703,7 +1619,8 @@ static unsigned getStackHazardSize(const MachineFunction &MF) { } // Convenience function to determine whether I is an SVE callee save. -static bool IsSVECalleeSave(MachineBasicBlock::iterator I) { +bool AArch64FrameLowering::isSVECalleeSave( + MachineBasicBlock::iterator I) const { switch (I->getOpcode()) { default: return false; @@ -1725,42 +1642,6 @@ static bool IsSVECalleeSave(MachineBasicBlock::iterator I) { } } -static void emitShadowCallStackPrologue(const TargetInstrInfo &TII, - MachineFunction &MF, - MachineBasicBlock &MBB, - MachineBasicBlock::iterator MBBI, - const DebugLoc &DL, bool NeedsWinCFI, - bool NeedsUnwindInfo) { - // Shadow call stack prolog: str x30, [x18], #8 - BuildMI(MBB, MBBI, DL, TII.get(AArch64::STRXpost)) - .addReg(AArch64::X18, RegState::Define) - .addReg(AArch64::LR) - .addReg(AArch64::X18) - .addImm(8) - .setMIFlag(MachineInstr::FrameSetup); - - // This instruction also makes x18 live-in to the entry block. - MBB.addLiveIn(AArch64::X18); - - if (NeedsWinCFI) - BuildMI(MBB, MBBI, DL, TII.get(AArch64::SEH_Nop)) - .setMIFlag(MachineInstr::FrameSetup); - - if (NeedsUnwindInfo) { - // Emit a CFI instruction that causes 8 to be subtracted from the value of - // x18 when unwinding past this frame. - static const char CFIInst[] = { - dwarf::DW_CFA_val_expression, - 18, // register - 2, // length - static_cast<char>(unsigned(dwarf::DW_OP_breg18)), - static_cast<char>(-8) & 0x7f, // addend (sleb128) - }; - CFIInstBuilder(MBB, MBBI, MachineInstr::FrameSetup) - .buildEscape(StringRef(CFIInst, sizeof(CFIInst))); - } -} - static void emitShadowCallStackEpilogue(const TargetInstrInfo &TII, MachineFunction &MF, MachineBasicBlock &MBB, @@ -1783,36 +1664,6 @@ static void emitShadowCallStackEpilogue(const TargetInstrInfo &TII, .buildRestore(AArch64::X18); } -// Define the current CFA rule to use the provided FP. -static void emitDefineCFAWithFP(MachineFunction &MF, MachineBasicBlock &MBB, - MachineBasicBlock::iterator MBBI, - unsigned FixedObject) { - const AArch64Subtarget &STI = MF.getSubtarget<AArch64Subtarget>(); - const AArch64RegisterInfo *TRI = STI.getRegisterInfo(); - AArch64FunctionInfo *AFI = MF.getInfo<AArch64FunctionInfo>(); - - const int OffsetToFirstCalleeSaveFromFP = - AFI->getCalleeSaveBaseToFrameRecordOffset() - - AFI->getCalleeSavedStackSize(); - Register FramePtr = TRI->getFrameRegister(MF); - CFIInstBuilder(MBB, MBBI, MachineInstr::FrameSetup) - .buildDefCFA(FramePtr, FixedObject - OffsetToFirstCalleeSaveFromFP); -} - -#ifndef NDEBUG -/// Collect live registers from the end of \p MI's parent up to (including) \p -/// MI in \p LiveRegs. -static void getLivePhysRegsUpTo(MachineInstr &MI, const TargetRegisterInfo &TRI, - LivePhysRegs &LiveRegs) { - - MachineBasicBlock &MBB = *MI.getParent(); - LiveRegs.addLiveOuts(MBB); - for (const MachineInstr &MI : - reverse(make_range(MI.getIterator(), MBB.instr_end()))) - LiveRegs.stepBackward(MI); -} -#endif - void AArch64FrameLowering::emitPacRetPlusLeafHardening( MachineFunction &MF) const { const AArch64Subtarget &Subtarget = MF.getSubtarget<AArch64Subtarget>(); @@ -1848,616 +1699,8 @@ void AArch64FrameLowering::emitPacRetPlusLeafHardening( void AArch64FrameLowering::emitPrologue(MachineFunction &MF, MachineBasicBlock &MBB) const { - MachineBasicBlock::iterator MBBI = MBB.begin(); - const MachineFrameInfo &MFI = MF.getFrameInfo(); - const Function &F = MF.getFunction(); - const AArch64Subtarget &Subtarget = MF.getSubtarget<AArch64Subtarget>(); - const AArch64RegisterInfo *RegInfo = Subtarget.getRegisterInfo(); - const TargetInstrInfo *TII = Subtarget.getInstrInfo(); - - AArch64FunctionInfo *AFI = MF.getInfo<AArch64FunctionInfo>(); - bool EmitCFI = AFI->needsDwarfUnwindInfo(MF); - bool EmitAsyncCFI = AFI->needsAsyncDwarfUnwindInfo(MF); - bool HasFP = hasFP(MF); - bool NeedsWinCFI = needsWinCFI(MF); - bool HasWinCFI = false; - auto Cleanup = make_scope_exit([&]() { MF.setHasWinCFI(HasWinCFI); }); - - MachineBasicBlock::iterator End = MBB.end(); -#ifndef NDEBUG - const TargetRegisterInfo *TRI = MF.getSubtarget().getRegisterInfo(); - // Collect live register from the end of MBB up to the start of the existing - // frame setup instructions. - MachineBasicBlock::iterator NonFrameStart = MBB.begin(); - while (NonFrameStart != End && - NonFrameStart->getFlag(MachineInstr::FrameSetup)) - ++NonFrameStart; - - LivePhysRegs LiveRegs(*TRI); - if (NonFrameStart != MBB.end()) { - getLivePhysRegsUpTo(*NonFrameStart, *TRI, LiveRegs); - // Ignore registers used for stack management for now. - LiveRegs.removeReg(AArch64::SP); - LiveRegs.removeReg(AArch64::X19); - LiveRegs.removeReg(AArch64::FP); - LiveRegs.removeReg(AArch64::LR); - - // X0 will be clobbered by a call to __arm_get_current_vg in the prologue. - // This is necessary to spill VG if required where SVE is unavailable, but - // X0 is preserved around this call. - if (requiresGetVGCall(MF)) - LiveRegs.removeReg(AArch64::X0); - } - - auto VerifyClobberOnExit = make_scope_exit([&]() { - if (NonFrameStart == MBB.end()) - return; - // Check if any of the newly instructions clobber any of the live registers. - for (MachineInstr &MI : - make_range(MBB.instr_begin(), NonFrameStart->getIterator())) { - for (auto &Op : MI.operands()) - if (Op.isReg() && Op.isDef()) - assert(!LiveRegs.contains(Op.getReg()) && - "live register clobbered by inserted prologue instructions"); - } - }); -#endif - - bool IsFunclet = MBB.isEHFuncletEntry(); - - // At this point, we're going to decide whether or not the function uses a - // redzone. In most cases, the function doesn't have a redzone so let's - // assume that's false and set it to true in the case that there's a redzone. - AFI->setHasRedZone(false); - - // Debug location must be unknown since the first debug location is used - // to determine the end of the prologue. - DebugLoc DL; - - const auto &MFnI = *MF.getInfo<AArch64FunctionInfo>(); - if (MFnI.shouldSignReturnAddress(MF)) { - // If pac-ret+leaf is in effect, PAUTH_PROLOGUE pseudo instructions - // are inserted by emitPacRetPlusLeafHardening(). - if (!shouldSignReturnAddressEverywhere(MF)) { - BuildMI(MBB, MBBI, DL, TII->get(AArch64::PAUTH_PROLOGUE)) - .setMIFlag(MachineInstr::FrameSetup); - } - // AArch64PointerAuth pass will insert SEH_PACSignLR - HasWinCFI |= NeedsWinCFI; - } - - if (MFnI.needsShadowCallStackPrologueEpilogue(MF)) { - emitShadowCallStackPrologue(*TII, MF, MBB, MBBI, DL, NeedsWinCFI, - MFnI.needsDwarfUnwindInfo(MF)); - HasWinCFI |= NeedsWinCFI; - } - - if (EmitCFI && MFnI.isMTETagged()) { - BuildMI(MBB, MBBI, DL, TII->get(AArch64::EMITMTETAGGED)) - .setMIFlag(MachineInstr::FrameSetup); - } - - // We signal the presence of a Swift extended frame to external tools by - // storing FP with 0b0001 in bits 63:60. In normal userland operation a simple - // ORR is sufficient, it is assumed a Swift kernel would initialize the TBI - // bits so that is still true. - if (HasFP && AFI->hasSwiftAsyncContext()) { - switch (MF.getTarget().Options.SwiftAsyncFramePointer) { - case SwiftAsyncFramePointerMode::DeploymentBased: - if (Subtarget.swiftAsyncContextIsDynamicallySet()) { - // The special symbol below is absolute and has a *value* that can be - // combined with the frame pointer to signal an extended frame. - BuildMI(MBB, MBBI, DL, TII->get(AArch64::LOADgot), AArch64::X16) - .addExternalSymbol("swift_async_extendedFramePointerFlags", - AArch64II::MO_GOT); - if (NeedsWinCFI) { - BuildMI(MBB, MBBI, DL, TII->get(AArch64::SEH_Nop)) - .setMIFlags(MachineInstr::FrameSetup); - HasWinCFI = true; - } - BuildMI(MBB, MBBI, DL, TII->get(AArch64::ORRXrs), AArch64::FP) - .addUse(AArch64::FP) - .addUse(AArch64::X16) - .addImm(Subtarget.isTargetILP32() ? 32 : 0); - if (NeedsWinCFI) { - BuildMI(MBB, MBBI, DL, TII->get(AArch64::SEH_Nop)) - .setMIFlags(MachineInstr::FrameSetup); - HasWinCFI = true; - } - break; - } - [[fallthrough]]; - - case SwiftAsyncFramePointerMode::Always: - // ORR x29, x29, #0x1000_0000_0000_0000 - BuildMI(MBB, MBBI, DL, TII->get(AArch64::ORRXri), AArch64::FP) - .addUse(AArch64::FP) - .addImm(0x1100) - .setMIFlag(MachineInstr::FrameSetup); - if (NeedsWinCFI) { - BuildMI(MBB, MBBI, DL, TII->get(AArch64::SEH_Nop)) - .setMIFlags(MachineInstr::FrameSetup); - HasWinCFI = true; - } - break; - - case SwiftAsyncFramePointerMode::Never: - break; - } - } - - // All calls are tail calls in GHC calling conv, and functions have no - // prologue/epilogue. - if (MF.getFunction().getCallingConv() == CallingConv::GHC) - return; - - // Set tagged base pointer to the requested stack slot. - // Ideally it should match SP value after prologue. - std::optional<int> TBPI = AFI->getTaggedBasePointerIndex(); - if (TBPI) - AFI->setTaggedBasePointerOffset(-MFI.getObjectOffset(*TBPI)); - else - AFI->setTaggedBasePointerOffset(MFI.getStackSize()); - - const StackOffset &SVEStackSize = getSVEStackSize(MF); - - // getStackSize() includes all the locals in its size calculation. We don't - // include these locals when computing the stack size of a funclet, as they - // are allocated in the parent's stack frame and accessed via the frame - // pointer from the funclet. We only save the callee saved registers in the - // funclet, which are really the callee saved registers of the parent - // function, including the funclet. - int64_t NumBytes = - IsFunclet ? getWinEHFuncletFrameSize(MF) : MFI.getStackSize(); - if (!AFI->hasStackFrame() && !windowsRequiresStackProbe(MF, NumBytes)) { - assert(!HasFP && "unexpected function without stack frame but with FP"); - assert(!SVEStackSize && - "unexpected function without stack frame but with SVE objects"); - // All of the stack allocation is for locals. - AFI->setLocalStackSize(NumBytes); - if (!NumBytes) { - if (NeedsWinCFI && HasWinCFI) { - BuildMI(MBB, MBBI, DL, TII->get(AArch64::SEH_PrologEnd)) - .setMIFlag(MachineInstr::FrameSetup); - } - return; - } - // REDZONE: If the stack size is less than 128 bytes, we don't need - // to actually allocate. - if (canUseRedZone(MF)) { - AFI->setHasRedZone(true); - ++NumRedZoneFunctions; - } else { - emitFrameOffset(MBB, MBBI, DL, AArch64::SP, AArch64::SP, - StackOffset::getFixed(-NumBytes), TII, - MachineInstr::FrameSetup, false, NeedsWinCFI, &HasWinCFI); - if (EmitCFI) { - // Label used to tie together the PROLOG_LABEL and the MachineMoves. - MCSymbol *FrameLabel = MF.getContext().createTempSymbol(); - // Encode the stack size of the leaf function. - CFIInstBuilder(MBB, MBBI, MachineInstr::FrameSetup) - .buildDefCFAOffset(NumBytes, FrameLabel); - } - } - - if (NeedsWinCFI) { - HasWinCFI = true; - BuildMI(MBB, MBBI, DL, TII->get(AArch64::SEH_PrologEnd)) - .setMIFlag(MachineInstr::FrameSetup); - } - - return; - } - - bool IsWin64 = Subtarget.isCallingConvWin64(F.getCallingConv(), F.isVarArg()); - unsigned FixedObject = getFixedObjectSize(MF, AFI, IsWin64, IsFunclet); - - // Windows unwind can't represent the required stack adjustments if we have - // both SVE callee-saves and dynamic stack allocations, and the frame - // pointer is before the SVE spills. The allocation of the frame pointer - // must be the last instruction in the prologue so the unwinder can restore - // the stack pointer correctly. (And there isn't any unwind opcode for - // `addvl sp, x29, -17`.) - // - // Because of this, we do spills in the opposite order on Windows: first SVE, - // then GPRs. The main side-effect of this is that it makes accessing - // parameters passed on the stack more expensive. - // - // We could consider rearranging the spills for simpler cases. - bool FPAfterSVECalleeSaves = - Subtarget.isTargetWindows() && AFI->getSVECalleeSavedStackSize(); - - if (FPAfterSVECalleeSaves && AFI->hasStackHazardSlotIndex()) - reportFatalUsageError("SME hazard padding is not supported on Windows"); - - auto PrologueSaveSize = AFI->getCalleeSavedStackSize() + FixedObject; - // All of the remaining stack allocations are for locals. - AFI->setLocalStackSize(NumBytes - PrologueSaveSize); - bool CombineSPBump = shouldCombineCSRLocalStackBump(MF, NumBytes); - bool HomPrologEpilog = homogeneousPrologEpilog(MF); - if (FPAfterSVECalleeSaves) { - // If we're doing SVE saves first, we need to immediately allocate space - // for fixed objects, then space for the SVE callee saves. - // - // Windows unwind requires that the scalable size is a multiple of 16; - // that's handled when the callee-saved size is computed. - auto SaveSize = - StackOffset::getScalable(AFI->getSVECalleeSavedStackSize()) + - StackOffset::getFixed(FixedObject); - allocateStackSpace(MBB, MBBI, 0, SaveSize, NeedsWinCFI, &HasWinCFI, - /*EmitCFI=*/false, StackOffset{}, - /*FollowupAllocs=*/true); - NumBytes -= FixedObject; - - // Now allocate space for the GPR callee saves. - while (MBBI != End && IsSVECalleeSave(MBBI)) - ++MBBI; - MBBI = convertCalleeSaveRestoreToSPPrePostIncDec( - MBB, MBBI, DL, TII, -AFI->getCalleeSavedStackSize(), NeedsWinCFI, - &HasWinCFI, EmitAsyncCFI); - NumBytes -= AFI->getCalleeSavedStackSize(); - } else if (CombineSPBump) { - assert(!SVEStackSize && "Cannot combine SP bump with SVE"); - emitFrameOffset(MBB, MBBI, DL, AArch64::SP, AArch64::SP, - StackOffset::getFixed(-NumBytes), TII, - MachineInstr::FrameSetup, false, NeedsWinCFI, &HasWinCFI, - EmitAsyncCFI); - NumBytes = 0; - } else if (HomPrologEpilog) { - // Stack has been already adjusted. - NumBytes -= PrologueSaveSize; - } else if (PrologueSaveSize != 0) { - MBBI = convertCalleeSaveRestoreToSPPrePostIncDec( - MBB, MBBI, DL, TII, -PrologueSaveSize, NeedsWinCFI, &HasWinCFI, - EmitAsyncCFI); - NumBytes -= PrologueSaveSize; - } - assert(NumBytes >= 0 && "Negative stack allocation size!?"); - - // Move past the saves of the callee-saved registers, fixing up the offsets - // and pre-inc if we decided to combine the callee-save and local stack - // pointer bump above. - auto &TLI = *MF.getSubtarget().getTargetLowering(); - while (MBBI != End && MBBI->getFlag(MachineInstr::FrameSetup) && - !IsSVECalleeSave(MBBI)) { - if (CombineSPBump && - // Only fix-up frame-setup load/store instructions. - (!requiresSaveVG(MF) || !isVGInstruction(MBBI, TLI))) - fixupCalleeSaveRestoreStackOffset(*MBBI, AFI->getLocalStackSize(), - NeedsWinCFI, &HasWinCFI); - ++MBBI; - } - - // For funclets the FP belongs to the containing function. - if (!IsFunclet && HasFP) { - // Only set up FP if we actually need to. - int64_t FPOffset = AFI->getCalleeSaveBaseToFrameRecordOffset(); - - if (CombineSPBump) - FPOffset += AFI->getLocalStackSize(); - - if (AFI->hasSwiftAsyncContext()) { - // Before we update the live FP we have to ensure there's a valid (or - // null) asynchronous context in its slot just before FP in the frame - // record, so store it now. - const auto &Attrs = MF.getFunction().getAttributes(); - bool HaveInitialContext = Attrs.hasAttrSomewhere(Attribute::SwiftAsync); - if (HaveInitialContext) - MBB.addLiveIn(AArch64::X22); - Register Reg = HaveInitialContext ? AArch64::X22 : AArch64::XZR; - BuildMI(MBB, MBBI, DL, TII->get(AArch64::StoreSwiftAsyncContext)) - .addUse(Reg) - .addUse(AArch64::SP) - .addImm(FPOffset - 8) - .setMIFlags(MachineInstr::FrameSetup); - if (NeedsWinCFI) { - // WinCFI and arm64e, where StoreSwiftAsyncContext is expanded - // to multiple instructions, should be mutually-exclusive. - assert(Subtarget.getTargetTriple().getArchName() != "arm64e"); - BuildMI(MBB, MBBI, DL, TII->get(AArch64::SEH_Nop)) - .setMIFlags(MachineInstr::FrameSetup); - HasWinCFI = true; - } - } - - if (HomPrologEpilog) { - auto Prolog = MBBI; - --Prolog; - assert(Prolog->getOpcode() == AArch64::HOM_Prolog); - Prolog->addOperand(MachineOperand::CreateImm(FPOffset)); - } else { - // Issue sub fp, sp, FPOffset or - // mov fp,sp when FPOffset is zero. - // Note: All stores of callee-saved registers are marked as "FrameSetup". - // This code marks the instruction(s) that set the FP also. - emitFrameOffset(MBB, MBBI, DL, AArch64::FP, AArch64::SP, - StackOffset::getFixed(FPOffset), TII, - MachineInstr::FrameSetup, false, NeedsWinCFI, &HasWinCFI); - if (NeedsWinCFI && HasWinCFI) { - BuildMI(MBB, MBBI, DL, TII->get(AArch64::SEH_PrologEnd)) - .setMIFlag(MachineInstr::FrameSetup); - // After setting up the FP, the rest of the prolog doesn't need to be - // included in the SEH unwind info. - NeedsWinCFI = false; - } - } - if (EmitAsyncCFI) - emitDefineCFAWithFP(MF, MBB, MBBI, FixedObject); - } - - // Now emit the moves for whatever callee saved regs we have (including FP, - // LR if those are saved). Frame instructions for SVE register are emitted - // later, after the instruction which actually save SVE regs. - if (EmitAsyncCFI) - emitCalleeSavedGPRLocations(MBB, MBBI); - - // Alignment is required for the parent frame, not the funclet - const bool NeedsRealignment = - NumBytes && !IsFunclet && RegInfo->hasStackRealignment(MF); - const int64_t RealignmentPadding = - (NeedsRealignment && MFI.getMaxAlign() > Align(16)) - ? MFI.getMaxAlign().value() - 16 - : 0; - - if (windowsRequiresStackProbe(MF, NumBytes + RealignmentPadding)) { - if (AFI->getSVECalleeSavedStackSize()) - report_fatal_error( - "SVE callee saves not yet supported with stack probing"); - - // Find an available register to spill the value of X15 to, if X15 is being - // used already for nest. - unsigned X15Scratch = AArch64::NoRegister; - const AArch64Subtarget &STI = MF.getSubtarget<AArch64Subtarget>(); - if (llvm::any_of(MBB.liveins(), - [&STI](const MachineBasicBlock::RegisterMaskPair &LiveIn) { - return STI.getRegisterInfo()->isSuperOrSubRegisterEq( - AArch64::X15, LiveIn.PhysReg); - })) { - X15Scratch = findScratchNonCalleeSaveRegister(&MBB, true); - assert(X15Scratch != AArch64::NoRegister && - (X15Scratch < AArch64::X15 || X15Scratch > AArch64::X17)); -#ifndef NDEBUG - LiveRegs.removeReg(AArch64::X15); // ignore X15 since we restore it -#endif - BuildMI(MBB, MBBI, DL, TII->get(AArch64::ORRXrr), X15Scratch) - .addReg(AArch64::XZR) - .addReg(AArch64::X15, RegState::Undef) - .addReg(AArch64::X15, RegState::Implicit) - .setMIFlag(MachineInstr::FrameSetup); - } - - uint64_t NumWords = (NumBytes + RealignmentPadding) >> 4; - if (NeedsWinCFI) { - HasWinCFI = true; - // alloc_l can hold at most 256MB, so assume that NumBytes doesn't - // exceed this amount. We need to move at most 2^24 - 1 into x15. - // This is at most two instructions, MOVZ followed by MOVK. - // TODO: Fix to use multiple stack alloc unwind codes for stacks - // exceeding 256MB in size. - if (NumBytes >= (1 << 28)) - report_fatal_error("Stack size cannot exceed 256MB for stack " - "unwinding purposes"); - - uint32_t LowNumWords = NumWords & 0xFFFF; - BuildMI(MBB, MBBI, DL, TII->get(AArch64::MOVZXi), AArch64::X15) - .addImm(LowNumWords) - .addImm(AArch64_AM::getShifterImm(AArch64_AM::LSL, 0)) - .setMIFlag(MachineInstr::FrameSetup); - BuildMI(MBB, MBBI, DL, TII->get(AArch64::SEH_Nop)) - .setMIFlag(MachineInstr::FrameSetup); - if ((NumWords & 0xFFFF0000) != 0) { - BuildMI(MBB, MBBI, DL, TII->get(AArch64::MOVKXi), AArch64::X15) - .addReg(AArch64::X15) - .addImm((NumWords & 0xFFFF0000) >> 16) // High half - .addImm(AArch64_AM::getShifterImm(AArch64_AM::LSL, 16)) - .setMIFlag(MachineInstr::FrameSetup); - BuildMI(MBB, MBBI, DL, TII->get(AArch64::SEH_Nop)) - .setMIFlag(MachineInstr::FrameSetup); - } - } else { - BuildMI(MBB, MBBI, DL, TII->get(AArch64::MOVi64imm), AArch64::X15) - .addImm(NumWords) - .setMIFlags(MachineInstr::FrameSetup); - } - - const char *ChkStk = Subtarget.getChkStkName(); - switch (MF.getTarget().getCodeModel()) { - case CodeModel::Tiny: - case CodeModel::Small: - case CodeModel::Medium: - case CodeModel::Kernel: - BuildMI(MBB, MBBI, DL, TII->get(AArch64::BL)) - .addExternalSymbol(ChkStk) - .addReg(AArch64::X15, RegState::Implicit) - .addReg(AArch64::X16, RegState::Implicit | RegState::Define | RegState::Dead) - .addReg(AArch64::X17, RegState::Implicit | RegState::Define | RegState::Dead) - .addReg(AArch64::NZCV, RegState::Implicit | RegState::Define | RegState::Dead) - .setMIFlags(MachineInstr::FrameSetup); - if (NeedsWinCFI) { - HasWinCFI = true; - BuildMI(MBB, MBBI, DL, TII->get(AArch64::SEH_Nop)) - .setMIFlag(MachineInstr::FrameSetup); - } - break; - case CodeModel::Large: - BuildMI(MBB, MBBI, DL, TII->get(AArch64::MOVaddrEXT)) - .addReg(AArch64::X16, RegState::Define) - .addExternalSymbol(ChkStk) - .addExternalSymbol(ChkStk) - .setMIFlags(MachineInstr::FrameSetup); - if (NeedsWinCFI) { - HasWinCFI = true; - BuildMI(MBB, MBBI, DL, TII->get(AArch64::SEH_Nop)) - .setMIFlag(MachineInstr::FrameSetup); - } - - BuildMI(MBB, MBBI, DL, TII->get(getBLRCallOpcode(MF))) - .addReg(AArch64::X16, RegState::Kill) - .addReg(AArch64::X15, RegState::Implicit | RegState::Define) - .addReg(AArch64::X16, RegState::Implicit | RegState::Define | RegState::Dead) - .addReg(AArch64::X17, RegState::Implicit | RegState::Define | RegState::Dead) - .addReg(AArch64::NZCV, RegState::Implicit | RegState::Define | RegState::Dead) - .setMIFlags(MachineInstr::FrameSetup); - if (NeedsWinCFI) { - HasWinCFI = true; - BuildMI(MBB, MBBI, DL, TII->get(AArch64::SEH_Nop)) - .setMIFlag(MachineInstr::FrameSetup); - } - break; - } - - BuildMI(MBB, MBBI, DL, TII->get(AArch64::SUBXrx64), AArch64::SP) - .addReg(AArch64::SP, RegState::Kill) - .addReg(AArch64::X15, RegState::Kill) - .addImm(AArch64_AM::getArithExtendImm(AArch64_AM::UXTX, 4)) - .setMIFlags(MachineInstr::FrameSetup); - if (NeedsWinCFI) { - HasWinCFI = true; - BuildMI(MBB, MBBI, DL, TII->get(AArch64::SEH_StackAlloc)) - .addImm(NumBytes) - .setMIFlag(MachineInstr::FrameSetup); - } - NumBytes = 0; - - if (RealignmentPadding > 0) { - if (RealignmentPadding >= 4096) { - BuildMI(MBB, MBBI, DL, TII->get(AArch64::MOVi64imm)) - .addReg(AArch64::X16, RegState::Define) - .addImm(RealignmentPadding) - .setMIFlags(MachineInstr::FrameSetup); - BuildMI(MBB, MBBI, DL, TII->get(AArch64::ADDXrx64), AArch64::X15) - .addReg(AArch64::SP) - .addReg(AArch64::X16, RegState::Kill) - .addImm(AArch64_AM::getArithExtendImm(AArch64_AM::UXTX, 0)) - .setMIFlag(MachineInstr::FrameSetup); - } else { - BuildMI(MBB, MBBI, DL, TII->get(AArch64::ADDXri), AArch64::X15) - .addReg(AArch64::SP) - .addImm(RealignmentPadding) - .addImm(0) - .setMIFlag(MachineInstr::FrameSetup); - } - - uint64_t AndMask = ~(MFI.getMaxAlign().value() - 1); - BuildMI(MBB, MBBI, DL, TII->get(AArch64::ANDXri), AArch64::SP) - .addReg(AArch64::X15, RegState::Kill) - .addImm(AArch64_AM::encodeLogicalImmediate(AndMask, 64)); - AFI->setStackRealigned(true); - - // No need for SEH instructions here; if we're realigning the stack, - // we've set a frame pointer and already finished the SEH prologue. - assert(!NeedsWinCFI); - } - if (X15Scratch != AArch64::NoRegister) { - BuildMI(MBB, MBBI, DL, TII->get(AArch64::ORRXrr), AArch64::X15) - .addReg(AArch64::XZR) - .addReg(X15Scratch, RegState::Undef) - .addReg(X15Scratch, RegState::Implicit) - .setMIFlag(MachineInstr::FrameSetup); - } - } - - StackOffset SVECalleeSavesSize = {}, SVELocalsSize = SVEStackSize; - MachineBasicBlock::iterator CalleeSavesEnd = MBBI; - - StackOffset CFAOffset = - StackOffset::getFixed((int64_t)MFI.getStackSize() - NumBytes); - - // Process the SVE callee-saves to determine what space needs to be - // allocated. - if (int64_t CalleeSavedSize = AFI->getSVECalleeSavedStackSize()) { - LLVM_DEBUG(dbgs() << "SVECalleeSavedStackSize = " << CalleeSavedSize - << "\n"); - SVECalleeSavesSize = StackOffset::getScalable(CalleeSavedSize); - SVELocalsSize = SVEStackSize - SVECalleeSavesSize; - // Find callee save instructions in frame. - // Note: With FPAfterSVECalleeSaves the callee saves have already been - // allocated. - if (!FPAfterSVECalleeSaves) { - MachineBasicBlock::iterator CalleeSavesBegin = MBBI; - assert(IsSVECalleeSave(CalleeSavesBegin) && "Unexpected instruction"); - while (IsSVECalleeSave(MBBI) && MBBI != MBB.getFirstTerminator()) - ++MBBI; - CalleeSavesEnd = MBBI; - - StackOffset LocalsSize = SVELocalsSize + StackOffset::getFixed(NumBytes); - // Allocate space for the callee saves (if any). - allocateStackSpace(MBB, CalleeSavesBegin, 0, SVECalleeSavesSize, false, - nullptr, EmitAsyncCFI && !HasFP, CFAOffset, - MFI.hasVarSizedObjects() || LocalsSize); - } - } - CFAOffset += SVECalleeSavesSize; - - if (EmitAsyncCFI) - emitCalleeSavedSVELocations(MBB, CalleeSavesEnd); - - // Allocate space for the rest of the frame including SVE locals. Align the - // stack as necessary. - assert(!(canUseRedZone(MF) && NeedsRealignment) && - "Cannot use redzone with stack realignment"); - if (!canUseRedZone(MF)) { - // FIXME: in the case of dynamic re-alignment, NumBytes doesn't have - // the correct value here, as NumBytes also includes padding bytes, - // which shouldn't be counted here. - allocateStackSpace(MBB, CalleeSavesEnd, RealignmentPadding, - SVELocalsSize + StackOffset::getFixed(NumBytes), - NeedsWinCFI, &HasWinCFI, EmitAsyncCFI && !HasFP, - CFAOffset, MFI.hasVarSizedObjects()); - } - - // If we need a base pointer, set it up here. It's whatever the value of the - // stack pointer is at this point. Any variable size objects will be allocated - // after this, so we can still use the base pointer to reference locals. - // - // FIXME: Clarify FrameSetup flags here. - // Note: Use emitFrameOffset() like above for FP if the FrameSetup flag is - // needed. - // For funclets the BP belongs to the containing function. - if (!IsFunclet && RegInfo->hasBasePointer(MF)) { - TII->copyPhysReg(MBB, MBBI, DL, RegInfo->getBaseRegister(), AArch64::SP, - false); - if (NeedsWinCFI) { - HasWinCFI = true; - BuildMI(MBB, MBBI, DL, TII->get(AArch64::SEH_Nop)) - .setMIFlag(MachineInstr::FrameSetup); - } - } - - // The very last FrameSetup instruction indicates the end of prologue. Emit a - // SEH opcode indicating the prologue end. - if (NeedsWinCFI && HasWinCFI) { - BuildMI(MBB, MBBI, DL, TII->get(AArch64::SEH_PrologEnd)) - .setMIFlag(MachineInstr::FrameSetup); - } - - // SEH funclets are passed the frame pointer in X1. If the parent - // function uses the base register, then the base register is used - // directly, and is not retrieved from X1. - if (IsFunclet && F.hasPersonalityFn()) { - EHPersonality Per = classifyEHPersonality(F.getPersonalityFn()); - if (isAsynchronousEHPersonality(Per)) { - BuildMI(MBB, MBBI, DL, TII->get(TargetOpcode::COPY), AArch64::FP) - .addReg(AArch64::X1) - .setMIFlag(MachineInstr::FrameSetup); - MBB.addLiveIn(AArch64::X1); - } - } - - if (EmitCFI && !EmitAsyncCFI) { - if (HasFP) { - emitDefineCFAWithFP(MF, MBB, MBBI, FixedObject); - } else { - StackOffset TotalSize = - SVEStackSize + StackOffset::getFixed((int64_t)MFI.getStackSize()); - CFIInstBuilder CFIBuilder(MBB, MBBI, MachineInstr::FrameSetup); - CFIBuilder.insertCFIInst( - createDefCFA(*RegInfo, /*FrameReg=*/AArch64::SP, /*Reg=*/AArch64::SP, - TotalSize, /*LastAdjustmentWasScalable=*/false)); - } - emitCalleeSavedGPRLocations(MBB, MBBI); - emitCalleeSavedSVELocations(MBB, MBBI); - } + AArch64PrologueEmitter PrologueEmitter(MF, MBB, *this); + PrologueEmitter.emitPrologue(); } static bool isFuncletReturnInstr(const MachineInstr &MI) { @@ -2607,7 +1850,7 @@ void AArch64FrameLowering::emitEpilogue(MachineFunction &MF, while (FirstGPRRestoreI != Begin) { --FirstGPRRestoreI; if (!FirstGPRRestoreI->getFlag(MachineInstr::FrameDestroy) || - (!FPAfterSVECalleeSaves && IsSVECalleeSave(FirstGPRRestoreI))) { + (!FPAfterSVECalleeSaves && isSVECalleeSave(FirstGPRRestoreI))) { ++FirstGPRRestoreI; break; } else if (CombineSPBump) @@ -2689,11 +1932,11 @@ void AArch64FrameLowering::emitEpilogue(MachineFunction &MF, RestoreBegin = std::prev(RestoreEnd); while (RestoreBegin != MBB.begin() && - IsSVECalleeSave(std::prev(RestoreBegin))) + isSVECalleeSave(std::prev(RestoreBegin))) --RestoreBegin; - assert(IsSVECalleeSave(RestoreBegin) && - IsSVECalleeSave(std::prev(RestoreEnd)) && "Unexpected instruction"); + assert(isSVECalleeSave(RestoreBegin) && + isSVECalleeSave(std::prev(RestoreEnd)) && "Unexpected instruction"); StackOffset CalleeSavedSizeAsOffset = StackOffset::getScalable(CalleeSavedSize); @@ -2927,8 +2170,8 @@ AArch64FrameLowering::getNonLocalFrameIndexReference(const MachineFunction &MF, return StackOffset::getFixed(getSEHFrameIndexOffset(MF, FI)); } -static StackOffset getFPOffset(const MachineFunction &MF, - int64_t ObjectOffset) { +StackOffset AArch64FrameLowering::getFPOffset(const MachineFunction &MF, + int64_t ObjectOffset) const { const auto *AFI = MF.getInfo<AArch64FunctionInfo>(); const auto &Subtarget = MF.getSubtarget<AArch64Subtarget>(); const Function &F = MF.getFunction(); @@ -2941,8 +2184,8 @@ static StackOffset getFPOffset(const MachineFunction &MF, return StackOffset::getFixed(ObjectOffset + FixedObject + FPAdjust); } -static StackOffset getStackOffset(const MachineFunction &MF, - int64_t ObjectOffset) { +StackOffset AArch64FrameLowering::getStackOffset(const MachineFunction &MF, + int64_t ObjectOffset) const { const auto &MFI = MF.getFrameInfo(); return StackOffset::getFixed(ObjectOffset + (int64_t)MFI.getStackSize()); } @@ -3140,7 +2383,8 @@ static unsigned getPrologueDeath(MachineFunction &MF, unsigned Reg) { return getKillRegState(!IsLiveIn); } -static bool produceCompactUnwindFrame(MachineFunction &MF) { +static bool produceCompactUnwindFrame(const AArch64FrameLowering &AFL, + MachineFunction &MF) { const AArch64Subtarget &Subtarget = MF.getSubtarget<AArch64Subtarget>(); AttributeList Attrs = MF.getFunction().getAttributes(); AArch64FunctionInfo *AFI = MF.getInfo<AArch64FunctionInfo>(); @@ -3148,7 +2392,7 @@ static bool produceCompactUnwindFrame(MachineFunction &MF) { !(Subtarget.getTargetLowering()->supportSwiftError() && Attrs.hasAttrSomewhere(Attribute::SwiftError)) && MF.getFunction().getCallingConv() != CallingConv::SwiftTail && - !requiresSaveVG(MF) && !AFI->isSVECC(); + !AFL.requiresSaveVG(MF) && !AFI->isSVECC(); } static bool invalidateWindowsRegisterPairing(unsigned Reg1, unsigned Reg2, @@ -3245,16 +2489,18 @@ bool enableMultiVectorSpillFill(const AArch64Subtarget &Subtarget, (!IsLocallyStreaming && Subtarget.isStreaming())); } -static void computeCalleeSaveRegisterPairs( - MachineFunction &MF, ArrayRef<CalleeSavedInfo> CSI, - const TargetRegisterInfo *TRI, SmallVectorImpl<RegPairInfo> &RegPairs, - bool NeedsFrameRecord) { +void computeCalleeSaveRegisterPairs(const AArch64FrameLowering &AFL, + MachineFunction &MF, + ArrayRef<CalleeSavedInfo> CSI, + const TargetRegisterInfo *TRI, + SmallVectorImpl<RegPairInfo> &RegPairs, + bool NeedsFrameRecord) { if (CSI.empty()) return; bool IsWindows = isTargetWindows(MF); - bool NeedsWinCFI = needsWinCFI(MF); + bool NeedsWinCFI = AFL.needsWinCFI(MF); AArch64FunctionInfo *AFI = MF.getInfo<AArch64FunctionInfo>(); unsigned StackHazardSize = getStackHazardSize(MF); MachineFrameInfo &MFI = MF.getFrameInfo(); @@ -3263,9 +2509,10 @@ static void computeCalleeSaveRegisterPairs( (void)CC; // MachO's compact unwind format relies on all registers being stored in // pairs. - assert((!produceCompactUnwindFrame(MF) || CC == CallingConv::PreserveMost || - CC == CallingConv::PreserveAll || CC == CallingConv::CXX_FAST_TLS || - CC == CallingConv::Win64 || (Count & 1) == 0) && + assert((!produceCompactUnwindFrame(AFL, MF) || + CC == CallingConv::PreserveMost || CC == CallingConv::PreserveAll || + CC == CallingConv::CXX_FAST_TLS || CC == CallingConv::Win64 || + (Count & 1) == 0) && "Odd number of callee-saved regs to spill!"); int ByteOffset = AFI->getCalleeSavedStackSize(); int StackFillDir = -1; @@ -3381,9 +2628,9 @@ static void computeCalleeSaveRegisterPairs( // MachO's compact unwind format relies on all registers being stored in // adjacent register pairs. - assert((!produceCompactUnwindFrame(MF) || CC == CallingConv::PreserveMost || - CC == CallingConv::PreserveAll || CC == CallingConv::CXX_FAST_TLS || - CC == CallingConv::Win64 || + assert((!produceCompactUnwindFrame(AFL, MF) || + CC == CallingConv::PreserveMost || CC == CallingConv::PreserveAll || + CC == CallingConv::CXX_FAST_TLS || CC == CallingConv::Win64 || (RPI.isPaired() && ((RPI.Reg1 == AArch64::LR && RPI.Reg2 == AArch64::FP) || RPI.Reg1 + 1 == RPI.Reg2))) && @@ -3496,7 +2743,7 @@ bool AArch64FrameLowering::spillCalleeSavedRegisters( DebugLoc DL; SmallVector<RegPairInfo, 8> RegPairs; - computeCalleeSaveRegisterPairs(MF, CSI, TRI, RegPairs, hasFP(MF)); + computeCalleeSaveRegisterPairs(*this, MF, CSI, TRI, RegPairs, hasFP(MF)); MachineRegisterInfo &MRI = MF.getRegInfo(); // Refresh the reserved regs in case there are any potential changes since the @@ -3708,7 +2955,7 @@ bool AArch64FrameLowering::restoreCalleeSavedRegisters( if (MBBI != MBB.end()) DL = MBBI->getDebugLoc(); - computeCalleeSaveRegisterPairs(MF, CSI, TRI, RegPairs, hasFP(MF)); + computeCalleeSaveRegisterPairs(*this, MF, CSI, TRI, RegPairs, hasFP(MF)); if (homogeneousPrologEpilog(MF, &MBB)) { auto MIB = BuildMI(MBB, MBBI, DL, TII.get(AArch64::HOM_Epilog)) .setMIFlag(MachineInstr::FrameDestroy); @@ -4142,7 +3389,7 @@ void AArch64FrameLowering::determineCalleeSaves(MachineFunction &MF, if (producePairRegisters(MF)) { if (UnspilledCSGPRPaired == AArch64::NoRegister) { // Failed to make a pair for compact unwind format, revert spilling. - if (produceCompactUnwindFrame(MF)) { + if (produceCompactUnwindFrame(*this, MF)) { SavedRegs.reset(UnspilledCSGPR); ExtraCSSpill = AArch64::NoRegister; } diff --git a/llvm/lib/Target/AArch64/AArch64FrameLowering.h b/llvm/lib/Target/AArch64/AArch64FrameLowering.h index 555a93359c27..a9d65441a4e3 100644 --- a/llvm/lib/Target/AArch64/AArch64FrameLowering.h +++ b/llvm/lib/Target/AArch64/AArch64FrameLowering.h @@ -19,6 +19,10 @@ namespace llvm { +class TargetLowering; +class AArch64FunctionInfo; +class AArch64PrologueEmitter; + class AArch64FrameLowering : public TargetFrameLowering { public: explicit AArch64FrameLowering() @@ -130,12 +134,19 @@ public: return StackId != TargetStackID::ScalableVector; } + friend class AArch64PrologueEmitter; void orderFrameObjects(const MachineFunction &MF, SmallVectorImpl<int> &ObjectsToAllocate) const override; bool isFPReserved(const MachineFunction &MF) const; + bool needsWinCFI(const MachineFunction &MF) const; + + bool requiresSaveVG(const MachineFunction &MF) const; + + StackOffset getSVEStackSize(const MachineFunction &MF) const; + protected: bool hasFPImpl(const MachineFunction &MF) const override; @@ -159,10 +170,6 @@ private: int &MaxCSFrameIndex) const; bool shouldCombineCSRLocalStackBumpInEpilogue(MachineBasicBlock &MBB, uint64_t StackBumpBytes) const; - void emitCalleeSavedGPRLocations(MachineBasicBlock &MBB, - MachineBasicBlock::iterator MBBI) const; - void emitCalleeSavedSVELocations(MachineBasicBlock &MBB, - MachineBasicBlock::iterator MBBI) const; void emitCalleeSavedGPRRestores(MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI) const; void emitCalleeSavedSVERestores(MachineBasicBlock &MBB, @@ -196,6 +203,61 @@ private: void emitRemarks(const MachineFunction &MF, MachineOptimizationRemarkEmitter *ORE) const override; + + bool windowsRequiresStackProbe(const MachineFunction &MF, + uint64_t StackSizeInBytes) const; + + bool shouldSignReturnAddressEverywhere(const MachineFunction &MF) const; + + StackOffset getFPOffset(const MachineFunction &MF, + int64_t ObjectOffset) const; + + StackOffset getStackOffset(const MachineFunction &MF, + int64_t ObjectOffset) const; + + // Find a scratch register that we can use at the start of the prologue to + // re-align the stack pointer. We avoid using callee-save registers since + // they may appear to be free when this is called from canUseAsPrologue + // (during shrink wrapping), but then no longer be free when this is called + // from emitPrologue. + // + // FIXME: This is a bit conservative, since in the above case we could use one + // of the callee-save registers as a scratch temp to re-align the stack + // pointer, but we would then have to make sure that we were in fact saving at + // least one callee-save register in the prologue, which is additional + // complexity that doesn't seem worth the benefit. + Register findScratchNonCalleeSaveRegister(MachineBasicBlock *MBB, + bool HasCall = false) const; + + // Convert callee-save register save/restore instruction to do stack pointer + // decrement/increment to allocate/deallocate the callee-save stack area by + // converting store/load to use pre/post increment version. + MachineBasicBlock::iterator convertCalleeSaveRestoreToSPPrePostIncDec( + MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI, + const DebugLoc &DL, const TargetInstrInfo *TII, int CSStackSizeInc, + bool NeedsWinCFI, bool *HasWinCFI, bool EmitCFI, + MachineInstr::MIFlag FrameFlag = MachineInstr::FrameSetup, + int CFAOffset = 0) const; + + // Fixup callee-save register save/restore instructions to take into account + // combined SP bump by adding the local stack size to the stack offsets. + void fixupCalleeSaveRestoreStackOffset(MachineInstr &MI, + uint64_t LocalStackSize, + bool NeedsWinCFI, + bool *HasWinCFI) const; + + bool isSVECalleeSave(MachineBasicBlock::iterator I) const; + + /// Returns the size of the fixed object area (allocated next to sp on entry) + /// On Win64 this may include a var args area and an UnwindHelp object for EH. + unsigned getFixedObjectSize(const MachineFunction &MF, + const AArch64FunctionInfo *AFI, bool IsWin64, + bool IsFunclet) const; + + bool isVGInstruction(MachineBasicBlock::iterator MBBI, + const TargetLowering &TLI) const; + + bool requiresGetVGCall(const MachineFunction &MF) const; }; } // End llvm namespace diff --git a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp index 351235dd5bcd..a194147d0939 100644 --- a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp +++ b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp @@ -8489,13 +8489,22 @@ SDValue AArch64TargetLowering::LowerFormalArguments( if (Subtarget->hasCustomCallingConv()) Subtarget->getRegisterInfo()->UpdateCustomCalleeSavedRegs(MF); - if (getTM().useNewSMEABILowering() && !Attrs.hasAgnosticZAInterface()) { + if (getTM().useNewSMEABILowering()) { if (Subtarget->isTargetWindows() || hasInlineStackProbe(MF)) { SDValue Size; if (Attrs.hasZAState()) { SDValue SVL = DAG.getNode(AArch64ISD::RDSVL, DL, MVT::i64, DAG.getConstant(1, DL, MVT::i32)); Size = DAG.getNode(ISD::MUL, DL, MVT::i64, SVL, SVL); + } else if (Attrs.hasAgnosticZAInterface()) { + RTLIB::Libcall LC = RTLIB::SMEABI_SME_STATE_SIZE; + SDValue Callee = DAG.getExternalSymbol( + getLibcallName(LC), getPointerTy(DAG.getDataLayout())); + auto *RetTy = EVT(MVT::i64).getTypeForEVT(*DAG.getContext()); + TargetLowering::CallLoweringInfo CLI(DAG); + CLI.setDebugLoc(DL).setChain(Chain).setLibCallee( + getLibcallCallingConv(LC), RetTy, Callee, {}); + std::tie(Size, Chain) = LowerCallTo(CLI); } if (Size) { SDValue Buffer = DAG.getNode( @@ -8561,7 +8570,7 @@ SDValue AArch64TargetLowering::LowerFormalArguments( Register BufferPtr = MF.getRegInfo().createVirtualRegister(&AArch64::GPR64RegClass); FuncInfo->setSMESaveBufferAddr(BufferPtr); - Chain = DAG.getCopyToReg(Chain, DL, BufferPtr, Buffer); + Chain = DAG.getCopyToReg(Buffer.getValue(1), DL, BufferPtr, Buffer); } } @@ -9300,17 +9309,17 @@ AArch64TargetLowering::LowerCall(CallLoweringInfo &CLI, // Determine whether we need any streaming mode changes. SMECallAttrs CallAttrs = getSMECallAttrs(MF.getFunction(), *this, CLI); + + std::optional<unsigned> ZAMarkerNode; bool UseNewSMEABILowering = getTM().useNewSMEABILowering(); - bool IsAgnosticZAFunction = CallAttrs.caller().hasAgnosticZAInterface(); - auto ZAMarkerNode = [&]() -> std::optional<unsigned> { - // TODO: Handle agnostic ZA functions. - if (!UseNewSMEABILowering || IsAgnosticZAFunction) - return std::nullopt; - if (!CallAttrs.caller().hasZAState() && !CallAttrs.caller().hasZT0State()) - return std::nullopt; - return CallAttrs.requiresLazySave() ? AArch64ISD::REQUIRES_ZA_SAVE - : AArch64ISD::INOUT_ZA_USE; - }(); + if (UseNewSMEABILowering) { + if (CallAttrs.requiresLazySave() || + CallAttrs.requiresPreservingAllZAState()) + ZAMarkerNode = AArch64ISD::REQUIRES_ZA_SAVE; + else if (CallAttrs.caller().hasZAState() || + CallAttrs.caller().hasZT0State()) + ZAMarkerNode = AArch64ISD::INOUT_ZA_USE; + } if (IsTailCall) { // Check if it's really possible to do a tail call. @@ -9385,7 +9394,8 @@ AArch64TargetLowering::LowerCall(CallLoweringInfo &CLI, }; bool RequiresLazySave = !UseNewSMEABILowering && CallAttrs.requiresLazySave(); - bool RequiresSaveAllZA = CallAttrs.requiresPreservingAllZAState(); + bool RequiresSaveAllZA = + !UseNewSMEABILowering && CallAttrs.requiresPreservingAllZAState(); if (RequiresLazySave) { TPIDR2Object &TPIDR2 = FuncInfo->getTPIDR2Obj(); SDValue TPIDR2ObjAddr = DAG.getFrameIndex( @@ -20185,13 +20195,21 @@ static bool isPredicateCCSettingOp(SDValue N) { (N.getOpcode() == ISD::GET_ACTIVE_LANE_MASK) || (N.getOpcode() == ISD::INTRINSIC_WO_CHAIN && (N.getConstantOperandVal(0) == Intrinsic::aarch64_sve_whilege || + N.getConstantOperandVal(0) == Intrinsic::aarch64_sve_whilege_x2 || N.getConstantOperandVal(0) == Intrinsic::aarch64_sve_whilegt || + N.getConstantOperandVal(0) == Intrinsic::aarch64_sve_whilegt_x2 || N.getConstantOperandVal(0) == Intrinsic::aarch64_sve_whilehi || + N.getConstantOperandVal(0) == Intrinsic::aarch64_sve_whilehi_x2 || N.getConstantOperandVal(0) == Intrinsic::aarch64_sve_whilehs || + N.getConstantOperandVal(0) == Intrinsic::aarch64_sve_whilehs_x2 || N.getConstantOperandVal(0) == Intrinsic::aarch64_sve_whilele || + N.getConstantOperandVal(0) == Intrinsic::aarch64_sve_whilele_x2 || N.getConstantOperandVal(0) == Intrinsic::aarch64_sve_whilelo || + N.getConstantOperandVal(0) == Intrinsic::aarch64_sve_whilelo_x2 || N.getConstantOperandVal(0) == Intrinsic::aarch64_sve_whilels || - N.getConstantOperandVal(0) == Intrinsic::aarch64_sve_whilelt))) + N.getConstantOperandVal(0) == Intrinsic::aarch64_sve_whilels_x2 || + N.getConstantOperandVal(0) == Intrinsic::aarch64_sve_whilelt || + N.getConstantOperandVal(0) == Intrinsic::aarch64_sve_whilelt_x2))) return true; return false; @@ -20217,7 +20235,7 @@ performFirstTrueTestVectorCombine(SDNode *N, // Restricted the DAG combine to only cases where we're extracting from a // flag-setting operation. - if (!isPredicateCCSettingOp(N0)) + if (!isPredicateCCSettingOp(N0) || N0.getResNo() != 0) return SDValue(); // Extracts of lane 0 for SVE can be expressed as PTEST(Op, FIRST) ? 1 : 0 @@ -22296,10 +22314,14 @@ static SDValue getPTest(SelectionDAG &DAG, EVT VT, SDValue Pg, SDValue Op, Op = DAG.getNode(AArch64ISD::REINTERPRET_CAST, DL, MVT::nxv16i1, Op); } + unsigned PTest = AArch64ISD::PTEST; + if (Cond == AArch64CC::ANY_ACTIVE) + PTest = AArch64ISD::PTEST_ANY; + else if (Cond == AArch64CC::FIRST_ACTIVE) + PTest = AArch64ISD::PTEST_FIRST; + // Set condition code (CC) flags. - SDValue Test = DAG.getNode( - Cond == AArch64CC::ANY_ACTIVE ? AArch64ISD::PTEST_ANY : AArch64ISD::PTEST, - DL, MVT::i32, Pg, Op); + SDValue Test = DAG.getNode(PTest, DL, MVT::i32, Pg, Op); // Convert CC to integer based on requested condition. // NOTE: Cond is inverted to promote CSEL's removal when it feeds a compare. @@ -22659,7 +22681,7 @@ static SDValue performIntrinsicCombine(SDNode *N, return tryCombineCRC32(0xffff, N, DAG); case Intrinsic::aarch64_sve_saddv: // There is no i64 version of SADDV because the sign is irrelevant. - if (N->getOperand(2)->getValueType(0).getVectorElementType() == MVT::i64) + if (N->getOperand(2).getValueType().getVectorElementType() == MVT::i64) return combineSVEReductionInt(N, AArch64ISD::UADDV_PRED, DAG); else return combineSVEReductionInt(N, AArch64ISD::SADDV_PRED, DAG); diff --git a/llvm/lib/Target/AArch64/AArch64InstrInfo.cpp b/llvm/lib/Target/AArch64/AArch64InstrInfo.cpp index 8843ce3b9478..bf3d47ac4360 100644 --- a/llvm/lib/Target/AArch64/AArch64InstrInfo.cpp +++ b/llvm/lib/Target/AArch64/AArch64InstrInfo.cpp @@ -1299,6 +1299,7 @@ bool AArch64InstrInfo::analyzeCompare(const MachineInstr &MI, Register &SrcReg, break; case AArch64::PTEST_PP: case AArch64::PTEST_PP_ANY: + case AArch64::PTEST_PP_FIRST: SrcReg = MI.getOperand(0).getReg(); SrcReg2 = MI.getOperand(1).getReg(); if (MI.getOperand(2).getSubReg()) @@ -1612,6 +1613,16 @@ bool AArch64InstrInfo::optimizePTestInstr( const MachineRegisterInfo *MRI) const { auto *Mask = MRI->getUniqueVRegDef(MaskReg); auto *Pred = MRI->getUniqueVRegDef(PredReg); + + if (Pred->isCopy() && PTest->getOpcode() == AArch64::PTEST_PP_FIRST) { + // Instructions which return a multi-vector (e.g. WHILECC_x2) require copies + // before the branch to extract each subregister. + auto Op = Pred->getOperand(1); + if (Op.isReg() && Op.getReg().isVirtual() && + Op.getSubReg() == AArch64::psub0) + Pred = MRI->getUniqueVRegDef(Op.getReg()); + } + unsigned PredOpcode = Pred->getOpcode(); auto NewOp = canRemovePTestInstr(PTest, Mask, Pred, MRI); if (!NewOp) @@ -1691,7 +1702,8 @@ bool AArch64InstrInfo::optimizeCompareInstr( } if (CmpInstr.getOpcode() == AArch64::PTEST_PP || - CmpInstr.getOpcode() == AArch64::PTEST_PP_ANY) + CmpInstr.getOpcode() == AArch64::PTEST_PP_ANY || + CmpInstr.getOpcode() == AArch64::PTEST_PP_FIRST) return optimizePTestInstr(&CmpInstr, SrcReg, SrcReg2, MRI); if (SrcReg2 != 0) @@ -5469,8 +5481,12 @@ void AArch64InstrInfo::copyPhysReg(MachineBasicBlock &MBB, // Copies between GPR64 and FPR64. if (AArch64::FPR64RegClass.contains(DestReg) && AArch64::GPR64RegClass.contains(SrcReg)) { - BuildMI(MBB, I, DL, get(AArch64::FMOVXDr), DestReg) - .addReg(SrcReg, getKillRegState(KillSrc)); + if (AArch64::XZR == SrcReg) { + BuildMI(MBB, I, DL, get(AArch64::FMOVD0), DestReg); + } else { + BuildMI(MBB, I, DL, get(AArch64::FMOVXDr), DestReg) + .addReg(SrcReg, getKillRegState(KillSrc)); + } return; } if (AArch64::GPR64RegClass.contains(DestReg) && @@ -5482,8 +5498,12 @@ void AArch64InstrInfo::copyPhysReg(MachineBasicBlock &MBB, // Copies between GPR32 and FPR32. if (AArch64::FPR32RegClass.contains(DestReg) && AArch64::GPR32RegClass.contains(SrcReg)) { - BuildMI(MBB, I, DL, get(AArch64::FMOVWSr), DestReg) - .addReg(SrcReg, getKillRegState(KillSrc)); + if (AArch64::WZR == SrcReg) { + BuildMI(MBB, I, DL, get(AArch64::FMOVS0), DestReg); + } else { + BuildMI(MBB, I, DL, get(AArch64::FMOVWSr), DestReg) + .addReg(SrcReg, getKillRegState(KillSrc)); + } return; } if (AArch64::GPR32RegClass.contains(DestReg) && @@ -11256,7 +11276,6 @@ AArch64InstrInfo::analyzeLoopForPipelining(MachineBasicBlock *LoopBB) const { /// verifyInstruction - Perform target specific instruction verification. bool AArch64InstrInfo::verifyInstruction(const MachineInstr &MI, StringRef &ErrInfo) const { - // Verify that immediate offsets on load/store instructions are within range. // Stack objects with an FI operand are excluded as they can be fixed up // during PEI. @@ -11272,6 +11291,30 @@ bool AArch64InstrInfo::verifyInstruction(const MachineInstr &MI, } } } + + const MCInstrDesc &MCID = MI.getDesc(); + for (unsigned Op = 0; Op < MCID.getNumOperands(); Op++) { + const MachineOperand &MO = MI.getOperand(Op); + switch (MCID.operands()[Op].OperandType) { + case AArch64::OPERAND_IMPLICIT_IMM_0: + if (!MO.isImm() || MO.getImm() != 0) { + ErrInfo = "OPERAND_IMPLICIT_IMM_0 should be 0"; + return false; + } + break; + case AArch64::OPERAND_SHIFT_MSL: + if (!MO.isImm() || + AArch64_AM::getShiftType(MO.getImm()) != AArch64_AM::MSL || + (AArch64_AM::getShiftValue(MO.getImm()) != 8 && + AArch64_AM::getShiftValue(MO.getImm()) != 16)) { + ErrInfo = "OPERAND_SHIFT_MSL should be msl shift of 8 or 16"; + return false; + } + break; + default: + break; + } + } return true; } diff --git a/llvm/lib/Target/AArch64/AArch64MachineFunctionInfo.cpp b/llvm/lib/Target/AArch64/AArch64MachineFunctionInfo.cpp index b4197a04840b..a81f5b3d436a 100644 --- a/llvm/lib/Target/AArch64/AArch64MachineFunctionInfo.cpp +++ b/llvm/lib/Target/AArch64/AArch64MachineFunctionInfo.cpp @@ -28,7 +28,10 @@ yaml::AArch64FunctionInfo::AArch64FunctionInfo( : HasRedZone(MFI.hasRedZone()), StackSizeSVE(MFI.hasCalculatedStackSizeSVE() ? std::optional<uint64_t>(MFI.getStackSizeSVE()) - : std::nullopt) {} + : std::nullopt), + HasStackFrame(MFI.hasStackFrame() + ? std::optional<bool>(MFI.hasStackFrame()) + : std::nullopt) {} void yaml::AArch64FunctionInfo::mappingImpl(yaml::IO &YamlIO) { MappingTraits<AArch64FunctionInfo>::mapping(YamlIO, *this); @@ -40,6 +43,8 @@ void AArch64FunctionInfo::initializeBaseYamlFields( HasRedZone = YamlMFI.HasRedZone; if (YamlMFI.StackSizeSVE) setStackSizeSVE(*YamlMFI.StackSizeSVE); + if (YamlMFI.HasStackFrame) + setHasStackFrame(*YamlMFI.HasStackFrame); } static std::pair<bool, bool> GetSignReturnAddress(const Function &F) { diff --git a/llvm/lib/Target/AArch64/AArch64MachineFunctionInfo.h b/llvm/lib/Target/AArch64/AArch64MachineFunctionInfo.h index 5dca1861a6d5..98fd018bf33a 100644 --- a/llvm/lib/Target/AArch64/AArch64MachineFunctionInfo.h +++ b/llvm/lib/Target/AArch64/AArch64MachineFunctionInfo.h @@ -261,7 +261,9 @@ public: EarlyAllocSMESaveBuffer = Ptr; } - Register getEarlyAllocSMESaveBuffer() { return EarlyAllocSMESaveBuffer; } + Register getEarlyAllocSMESaveBuffer() const { + return EarlyAllocSMESaveBuffer; + } // Old SME ABI lowering state getters/setters: Register getSMESaveBufferAddr() const { return SMESaveBufferAddr; }; @@ -598,6 +600,7 @@ namespace yaml { struct AArch64FunctionInfo final : public yaml::MachineFunctionInfo { std::optional<bool> HasRedZone; std::optional<uint64_t> StackSizeSVE; + std::optional<bool> HasStackFrame; AArch64FunctionInfo() = default; AArch64FunctionInfo(const llvm::AArch64FunctionInfo &MFI); @@ -610,6 +613,7 @@ template <> struct MappingTraits<AArch64FunctionInfo> { static void mapping(IO &YamlIO, AArch64FunctionInfo &MFI) { YamlIO.mapOptional("hasRedZone", MFI.HasRedZone); YamlIO.mapOptional("stackSizeSVE", MFI.StackSizeSVE); + YamlIO.mapOptional("hasStackFrame", MFI.HasStackFrame); } }; diff --git a/llvm/lib/Target/AArch64/AArch64Processors.td b/llvm/lib/Target/AArch64/AArch64Processors.td index d5f4e91ae518..81f5d075729d 100644 --- a/llvm/lib/Target/AArch64/AArch64Processors.td +++ b/llvm/lib/Target/AArch64/AArch64Processors.td @@ -344,6 +344,8 @@ def TuneAppleA7 : SubtargetFeature<"apple-a7", "ARMProcFamily", "AppleA7", FeatureZCRegMoveFPR128, FeatureZCZeroingGPR32, FeatureZCZeroingGPR64, + FeatureNoZCZeroingFPR64, + FeatureZCZeroingFPR128, FeatureZCZeroingFPWorkaround]>; def TuneAppleA10 : SubtargetFeature<"apple-a10", "ARMProcFamily", "AppleA10", @@ -358,7 +360,9 @@ def TuneAppleA10 : SubtargetFeature<"apple-a10", "ARMProcFamily", "AppleA10", FeatureZCRegMoveGPR64, FeatureZCRegMoveFPR128, FeatureZCZeroingGPR32, - FeatureZCZeroingGPR64]>; + FeatureZCZeroingGPR64, + FeatureNoZCZeroingFPR64, + FeatureZCZeroingFPR128]>; def TuneAppleA11 : SubtargetFeature<"apple-a11", "ARMProcFamily", "AppleA11", "Apple A11", [ @@ -372,7 +376,9 @@ def TuneAppleA11 : SubtargetFeature<"apple-a11", "ARMProcFamily", "AppleA11", FeatureZCRegMoveGPR64, FeatureZCRegMoveFPR128, FeatureZCZeroingGPR32, - FeatureZCZeroingGPR64]>; + FeatureZCZeroingGPR64, + FeatureNoZCZeroingFPR64, + FeatureZCZeroingFPR128]>; def TuneAppleA12 : SubtargetFeature<"apple-a12", "ARMProcFamily", "AppleA12", "Apple A12", [ @@ -386,7 +392,9 @@ def TuneAppleA12 : SubtargetFeature<"apple-a12", "ARMProcFamily", "AppleA12", FeatureZCRegMoveGPR64, FeatureZCRegMoveFPR128, FeatureZCZeroingGPR32, - FeatureZCZeroingGPR64]>; + FeatureZCZeroingGPR64, + FeatureNoZCZeroingFPR64, + FeatureZCZeroingFPR128]>; def TuneAppleA13 : SubtargetFeature<"apple-a13", "ARMProcFamily", "AppleA13", "Apple A13", [ @@ -400,7 +408,9 @@ def TuneAppleA13 : SubtargetFeature<"apple-a13", "ARMProcFamily", "AppleA13", FeatureZCRegMoveGPR64, FeatureZCRegMoveFPR128, FeatureZCZeroingGPR32, - FeatureZCZeroingGPR64]>; + FeatureZCZeroingGPR64, + FeatureNoZCZeroingFPR64, + FeatureZCZeroingFPR128]>; def TuneAppleA14 : SubtargetFeature<"apple-a14", "ARMProcFamily", "AppleA14", "Apple A14", [ @@ -419,7 +429,9 @@ def TuneAppleA14 : SubtargetFeature<"apple-a14", "ARMProcFamily", "AppleA14", FeatureZCRegMoveGPR64, FeatureZCRegMoveFPR128, FeatureZCZeroingGPR32, - FeatureZCZeroingGPR64]>; + FeatureZCZeroingGPR64, + FeatureNoZCZeroingFPR64, + FeatureZCZeroingFPR128]>; def TuneAppleA15 : SubtargetFeature<"apple-a15", "ARMProcFamily", "AppleA15", "Apple A15", [ @@ -438,7 +450,9 @@ def TuneAppleA15 : SubtargetFeature<"apple-a15", "ARMProcFamily", "AppleA15", FeatureZCRegMoveGPR64, FeatureZCRegMoveFPR128, FeatureZCZeroingGPR32, - FeatureZCZeroingGPR64]>; + FeatureZCZeroingGPR64, + FeatureNoZCZeroingFPR64, + FeatureZCZeroingFPR128]>; def TuneAppleA16 : SubtargetFeature<"apple-a16", "ARMProcFamily", "AppleA16", "Apple A16", [ @@ -457,7 +471,9 @@ def TuneAppleA16 : SubtargetFeature<"apple-a16", "ARMProcFamily", "AppleA16", FeatureZCRegMoveGPR64, FeatureZCRegMoveFPR128, FeatureZCZeroingGPR32, - FeatureZCZeroingGPR64]>; + FeatureZCZeroingGPR64, + FeatureNoZCZeroingFPR64, + FeatureZCZeroingFPR128]>; def TuneAppleA17 : SubtargetFeature<"apple-a17", "ARMProcFamily", "AppleA17", "Apple A17", [ @@ -476,7 +492,9 @@ def TuneAppleA17 : SubtargetFeature<"apple-a17", "ARMProcFamily", "AppleA17", FeatureZCRegMoveGPR64, FeatureZCRegMoveFPR128, FeatureZCZeroingGPR32, - FeatureZCZeroingGPR64]>; + FeatureZCZeroingGPR64, + FeatureNoZCZeroingFPR64, + FeatureZCZeroingFPR128]>; def TuneAppleM4 : SubtargetFeature<"apple-m4", "ARMProcFamily", "AppleM4", "Apple M4", [ @@ -494,7 +512,9 @@ def TuneAppleM4 : SubtargetFeature<"apple-m4", "ARMProcFamily", "AppleM4", FeatureZCRegMoveGPR64, FeatureZCRegMoveFPR128, FeatureZCZeroingGPR32, - FeatureZCZeroingGPR64]>; + FeatureZCZeroingGPR64, + FeatureNoZCZeroingFPR64, + FeatureZCZeroingFPR128]>; def TuneExynosM3 : SubtargetFeature<"exynosm3", "ARMProcFamily", "ExynosM3", "Samsung Exynos-M3 processors", diff --git a/llvm/lib/Target/AArch64/AArch64PrologueEpilogue.cpp b/llvm/lib/Target/AArch64/AArch64PrologueEpilogue.cpp new file mode 100644 index 000000000000..af424987b8dd --- /dev/null +++ b/llvm/lib/Target/AArch64/AArch64PrologueEpilogue.cpp @@ -0,0 +1,794 @@ +//===----------------------------------------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#include "AArch64PrologueEpilogue.h" +#include "AArch64FrameLowering.h" +#include "AArch64MachineFunctionInfo.h" +#include "AArch64Subtarget.h" +#include "MCTargetDesc/AArch64AddressingModes.h" +#include "llvm/ADT/Statistic.h" +#include "llvm/BinaryFormat/Dwarf.h" +#include "llvm/CodeGen/CFIInstBuilder.h" +#include "llvm/MC/MCContext.h" + +#define DEBUG_TYPE "frame-info" + +STATISTIC(NumRedZoneFunctions, "Number of functions using red zone"); + +namespace llvm { + +AArch64PrologueEmitter::AArch64PrologueEmitter(MachineFunction &MF, + MachineBasicBlock &MBB, + const AArch64FrameLowering &AFL) + : MF(MF), MBB(MBB), F(MF.getFunction()), MFI(MF.getFrameInfo()), + Subtarget(MF.getSubtarget<AArch64Subtarget>()), AFL(AFL), + RegInfo(*Subtarget.getRegisterInfo()) { + TII = Subtarget.getInstrInfo(); + AFI = MF.getInfo<AArch64FunctionInfo>(); + + EmitCFI = AFI->needsDwarfUnwindInfo(MF); + EmitAsyncCFI = AFI->needsAsyncDwarfUnwindInfo(MF); + HasFP = AFL.hasFP(MF); + NeedsWinCFI = AFL.needsWinCFI(MF); + IsFunclet = MBB.isEHFuncletEntry(); + HomPrologEpilog = AFL.homogeneousPrologEpilog(MF); + +#ifndef NDEBUG + collectBlockLiveins(); +#endif +} + +#ifndef NDEBUG +/// Collect live registers from the end of \p MI's parent up to (including) \p +/// MI in \p LiveRegs. +static void getLivePhysRegsUpTo(MachineInstr &MI, const TargetRegisterInfo &TRI, + LivePhysRegs &LiveRegs) { + + MachineBasicBlock &MBB = *MI.getParent(); + LiveRegs.addLiveOuts(MBB); + for (const MachineInstr &MI : + reverse(make_range(MI.getIterator(), MBB.instr_end()))) + LiveRegs.stepBackward(MI); +} + +void AArch64PrologueEmitter::collectBlockLiveins() { + // Collect live register from the end of MBB up to the start of the existing + // frame setup instructions. + PrologueEndI = MBB.begin(); + while (PrologueEndI != MBB.end() && + PrologueEndI->getFlag(MachineInstr::FrameSetup)) + ++PrologueEndI; + + if (PrologueEndI != MBB.end()) { + getLivePhysRegsUpTo(*PrologueEndI, RegInfo, LiveRegs); + // Ignore registers used for stack management for now. + LiveRegs.removeReg(AArch64::SP); + LiveRegs.removeReg(AArch64::X19); + LiveRegs.removeReg(AArch64::FP); + LiveRegs.removeReg(AArch64::LR); + + // X0 will be clobbered by a call to __arm_get_current_vg in the prologue. + // This is necessary to spill VG if required where SVE is unavailable, but + // X0 is preserved around this call. + if (AFL.requiresGetVGCall(MF)) + LiveRegs.removeReg(AArch64::X0); + } +} + +void AArch64PrologueEmitter::verifyPrologueClobbers() const { + if (PrologueEndI == MBB.end()) + return; + // Check if any of the newly instructions clobber any of the live registers. + for (MachineInstr &MI : + make_range(MBB.instr_begin(), PrologueEndI->getIterator())) { + for (auto &Op : MI.operands()) + if (Op.isReg() && Op.isDef()) + assert(!LiveRegs.contains(Op.getReg()) && + "live register clobbered by inserted prologue instructions"); + } +} +#endif + +void AArch64PrologueEmitter::determineLocalsStackSize( + uint64_t StackSize, uint64_t PrologueSaveSize) { + AFI->setLocalStackSize(StackSize - PrologueSaveSize); + CombineSPBump = AFL.shouldCombineCSRLocalStackBump(MF, StackSize); +} + +void AArch64PrologueEmitter::emitPrologue() { + const MachineBasicBlock::iterator PrologueBeginI = MBB.begin(); + const MachineBasicBlock::iterator EndI = MBB.end(); + + // At this point, we're going to decide whether or not the function uses a + // redzone. In most cases, the function doesn't have a redzone so let's + // assume that's false and set it to true in the case that there's a redzone. + AFI->setHasRedZone(false); + + // Debug location must be unknown since the first debug location is used + // to determine the end of the prologue. + DebugLoc DL; + + if (AFI->shouldSignReturnAddress(MF)) { + // If pac-ret+leaf is in effect, PAUTH_PROLOGUE pseudo instructions + // are inserted by emitPacRetPlusLeafHardening(). + if (!AFL.shouldSignReturnAddressEverywhere(MF)) { + BuildMI(MBB, PrologueBeginI, DL, TII->get(AArch64::PAUTH_PROLOGUE)) + .setMIFlag(MachineInstr::FrameSetup); + } + // AArch64PointerAuth pass will insert SEH_PACSignLR + HasWinCFI |= NeedsWinCFI; + } + + if (AFI->needsShadowCallStackPrologueEpilogue(MF)) { + emitShadowCallStackPrologue(PrologueBeginI, DL); + HasWinCFI |= NeedsWinCFI; + } + + if (EmitCFI && AFI->isMTETagged()) + BuildMI(MBB, PrologueBeginI, DL, TII->get(AArch64::EMITMTETAGGED)) + .setMIFlag(MachineInstr::FrameSetup); + + // We signal the presence of a Swift extended frame to external tools by + // storing FP with 0b0001 in bits 63:60. In normal userland operation a simple + // ORR is sufficient, it is assumed a Swift kernel would initialize the TBI + // bits so that is still true. + if (HasFP && AFI->hasSwiftAsyncContext()) + emitSwiftAsyncContextFramePointer(PrologueBeginI, DL); + + // All calls are tail calls in GHC calling conv, and functions have no + // prologue/epilogue. + if (MF.getFunction().getCallingConv() == CallingConv::GHC) + return; + + // Set tagged base pointer to the requested stack slot. Ideally it should + // match SP value after prologue. + if (std::optional<int> TBPI = AFI->getTaggedBasePointerIndex()) + AFI->setTaggedBasePointerOffset(-MFI.getObjectOffset(*TBPI)); + else + AFI->setTaggedBasePointerOffset(MFI.getStackSize()); + + // getStackSize() includes all the locals in its size calculation. We don't + // include these locals when computing the stack size of a funclet, as they + // are allocated in the parent's stack frame and accessed via the frame + // pointer from the funclet. We only save the callee saved registers in the + // funclet, which are really the callee saved registers of the parent + // function, including the funclet. + int64_t NumBytes = + IsFunclet ? AFL.getWinEHFuncletFrameSize(MF) : MFI.getStackSize(); + if (!AFI->hasStackFrame() && !AFL.windowsRequiresStackProbe(MF, NumBytes)) + return emitEmptyStackFramePrologue(NumBytes, PrologueBeginI, DL); + + bool IsWin64 = Subtarget.isCallingConvWin64(F.getCallingConv(), F.isVarArg()); + unsigned FixedObject = AFL.getFixedObjectSize(MF, AFI, IsWin64, IsFunclet); + + // Windows unwind can't represent the required stack adjustments if we have + // both SVE callee-saves and dynamic stack allocations, and the frame + // pointer is before the SVE spills. The allocation of the frame pointer + // must be the last instruction in the prologue so the unwinder can restore + // the stack pointer correctly. (And there isn't any unwind opcode for + // `addvl sp, x29, -17`.) + // + // Because of this, we do spills in the opposite order on Windows: first SVE, + // then GPRs. The main side-effect of this is that it makes accessing + // parameters passed on the stack more expensive. + // + // We could consider rearranging the spills for simpler cases. + bool FPAfterSVECalleeSaves = + Subtarget.isTargetWindows() && AFI->getSVECalleeSavedStackSize(); + + if (FPAfterSVECalleeSaves && AFI->hasStackHazardSlotIndex()) + reportFatalUsageError("SME hazard padding is not supported on Windows"); + + auto PrologueSaveSize = AFI->getCalleeSavedStackSize() + FixedObject; + // All of the remaining stack allocations are for locals. + determineLocalsStackSize(NumBytes, PrologueSaveSize); + + MachineBasicBlock::iterator FirstGPRSaveI = PrologueBeginI; + if (FPAfterSVECalleeSaves) { + // If we're doing SVE saves first, we need to immediately allocate space + // for fixed objects, then space for the SVE callee saves. + // + // Windows unwind requires that the scalable size is a multiple of 16; + // that's handled when the callee-saved size is computed. + auto SaveSize = + StackOffset::getScalable(AFI->getSVECalleeSavedStackSize()) + + StackOffset::getFixed(FixedObject); + AFL.allocateStackSpace(MBB, PrologueBeginI, 0, SaveSize, NeedsWinCFI, + &HasWinCFI, + /*EmitCFI=*/false, StackOffset{}, + /*FollowupAllocs=*/true); + NumBytes -= FixedObject; + + // Now allocate space for the GPR callee saves. + MachineBasicBlock::iterator MBBI = PrologueBeginI; + while (MBBI != EndI && AFL.isSVECalleeSave(MBBI)) + ++MBBI; + FirstGPRSaveI = AFL.convertCalleeSaveRestoreToSPPrePostIncDec( + MBB, MBBI, DL, TII, -AFI->getCalleeSavedStackSize(), NeedsWinCFI, + &HasWinCFI, EmitAsyncCFI); + NumBytes -= AFI->getCalleeSavedStackSize(); + } else if (CombineSPBump) { + assert(!AFL.getSVEStackSize(MF) && "Cannot combine SP bump with SVE"); + emitFrameOffset(MBB, PrologueBeginI, DL, AArch64::SP, AArch64::SP, + StackOffset::getFixed(-NumBytes), TII, + MachineInstr::FrameSetup, false, NeedsWinCFI, &HasWinCFI, + EmitAsyncCFI); + NumBytes = 0; + } else if (HomPrologEpilog) { + // Stack has been already adjusted. + NumBytes -= PrologueSaveSize; + } else if (PrologueSaveSize != 0) { + FirstGPRSaveI = AFL.convertCalleeSaveRestoreToSPPrePostIncDec( + MBB, PrologueBeginI, DL, TII, -PrologueSaveSize, NeedsWinCFI, + &HasWinCFI, EmitAsyncCFI); + NumBytes -= PrologueSaveSize; + } + assert(NumBytes >= 0 && "Negative stack allocation size!?"); + + // Move past the saves of the callee-saved registers, fixing up the offsets + // and pre-inc if we decided to combine the callee-save and local stack + // pointer bump above. + auto &TLI = *MF.getSubtarget().getTargetLowering(); + + MachineBasicBlock::iterator AfterGPRSavesI = FirstGPRSaveI; + while (AfterGPRSavesI != EndI && + AfterGPRSavesI->getFlag(MachineInstr::FrameSetup) && + !AFL.isSVECalleeSave(AfterGPRSavesI)) { + if (CombineSPBump && + // Only fix-up frame-setup load/store instructions. + (!AFL.requiresSaveVG(MF) || !AFL.isVGInstruction(AfterGPRSavesI, TLI))) + AFL.fixupCalleeSaveRestoreStackOffset( + *AfterGPRSavesI, AFI->getLocalStackSize(), NeedsWinCFI, &HasWinCFI); + ++AfterGPRSavesI; + } + + // For funclets the FP belongs to the containing function. Only set up FP if + // we actually need to. + if (!IsFunclet && HasFP) + emitFramePointerSetup(AfterGPRSavesI, DL, FixedObject); + + // Now emit the moves for whatever callee saved regs we have (including FP, + // LR if those are saved). Frame instructions for SVE register are emitted + // later, after the instruction which actually save SVE regs. + if (EmitAsyncCFI) + emitCalleeSavedGPRLocations(AfterGPRSavesI); + + // Alignment is required for the parent frame, not the funclet + const bool NeedsRealignment = + NumBytes && !IsFunclet && RegInfo.hasStackRealignment(MF); + const int64_t RealignmentPadding = + (NeedsRealignment && MFI.getMaxAlign() > Align(16)) + ? MFI.getMaxAlign().value() - 16 + : 0; + + if (AFL.windowsRequiresStackProbe(MF, NumBytes + RealignmentPadding)) + emitWindowsStackProbe(AfterGPRSavesI, DL, NumBytes, RealignmentPadding); + + StackOffset SVEStackSize = AFL.getSVEStackSize(MF); + StackOffset SVECalleeSavesSize = {}, SVELocalsSize = SVEStackSize; + MachineBasicBlock::iterator CalleeSavesEnd = AfterGPRSavesI; + + StackOffset CFAOffset = + StackOffset::getFixed((int64_t)MFI.getStackSize() - NumBytes); + + // Process the SVE callee-saves to determine what space needs to be + // allocated. + MachineBasicBlock::iterator AfterSVESavesI = AfterGPRSavesI; + if (int64_t CalleeSavedSize = AFI->getSVECalleeSavedStackSize()) { + LLVM_DEBUG(dbgs() << "SVECalleeSavedStackSize = " << CalleeSavedSize + << "\n"); + SVECalleeSavesSize = StackOffset::getScalable(CalleeSavedSize); + SVELocalsSize = SVEStackSize - SVECalleeSavesSize; + // Find callee save instructions in frame. + // Note: With FPAfterSVECalleeSaves the callee saves have already been + // allocated. + if (!FPAfterSVECalleeSaves) { + MachineBasicBlock::iterator CalleeSavesBegin = AfterGPRSavesI; + assert(AFL.isSVECalleeSave(CalleeSavesBegin) && "Unexpected instruction"); + while (AFL.isSVECalleeSave(AfterSVESavesI) && + AfterSVESavesI != MBB.getFirstTerminator()) + ++AfterSVESavesI; + CalleeSavesEnd = AfterSVESavesI; + + StackOffset LocalsSize = SVELocalsSize + StackOffset::getFixed(NumBytes); + // Allocate space for the callee saves (if any). + AFL.allocateStackSpace(MBB, CalleeSavesBegin, 0, SVECalleeSavesSize, + false, nullptr, EmitAsyncCFI && !HasFP, CFAOffset, + MFI.hasVarSizedObjects() || LocalsSize); + } + } + CFAOffset += SVECalleeSavesSize; + + if (EmitAsyncCFI) + emitCalleeSavedSVELocations(CalleeSavesEnd); + + // Allocate space for the rest of the frame including SVE locals. Align the + // stack as necessary. + assert(!(AFL.canUseRedZone(MF) && NeedsRealignment) && + "Cannot use redzone with stack realignment"); + if (!AFL.canUseRedZone(MF)) { + // FIXME: in the case of dynamic re-alignment, NumBytes doesn't have + // the correct value here, as NumBytes also includes padding bytes, + // which shouldn't be counted here. + AFL.allocateStackSpace(MBB, CalleeSavesEnd, RealignmentPadding, + SVELocalsSize + StackOffset::getFixed(NumBytes), + NeedsWinCFI, &HasWinCFI, EmitAsyncCFI && !HasFP, + CFAOffset, MFI.hasVarSizedObjects()); + } + + // If we need a base pointer, set it up here. It's whatever the value of the + // stack pointer is at this point. Any variable size objects will be allocated + // after this, so we can still use the base pointer to reference locals. + // + // FIXME: Clarify FrameSetup flags here. + // Note: Use emitFrameOffset() like above for FP if the FrameSetup flag is + // needed. + // For funclets the BP belongs to the containing function. + if (!IsFunclet && RegInfo.hasBasePointer(MF)) { + TII->copyPhysReg(MBB, AfterSVESavesI, DL, RegInfo.getBaseRegister(), + AArch64::SP, false); + if (NeedsWinCFI) { + HasWinCFI = true; + BuildMI(MBB, AfterSVESavesI, DL, TII->get(AArch64::SEH_Nop)) + .setMIFlag(MachineInstr::FrameSetup); + } + } + + // The very last FrameSetup instruction indicates the end of prologue. Emit a + // SEH opcode indicating the prologue end. + if (NeedsWinCFI && HasWinCFI) { + BuildMI(MBB, AfterSVESavesI, DL, TII->get(AArch64::SEH_PrologEnd)) + .setMIFlag(MachineInstr::FrameSetup); + } + + // SEH funclets are passed the frame pointer in X1. If the parent + // function uses the base register, then the base register is used + // directly, and is not retrieved from X1. + if (IsFunclet && F.hasPersonalityFn()) { + EHPersonality Per = classifyEHPersonality(F.getPersonalityFn()); + if (isAsynchronousEHPersonality(Per)) { + BuildMI(MBB, AfterSVESavesI, DL, TII->get(TargetOpcode::COPY), + AArch64::FP) + .addReg(AArch64::X1) + .setMIFlag(MachineInstr::FrameSetup); + MBB.addLiveIn(AArch64::X1); + } + } + + if (EmitCFI && !EmitAsyncCFI) { + if (HasFP) { + emitDefineCFAWithFP(AfterSVESavesI, FixedObject); + } else { + StackOffset TotalSize = + SVEStackSize + StackOffset::getFixed((int64_t)MFI.getStackSize()); + CFIInstBuilder CFIBuilder(MBB, AfterSVESavesI, MachineInstr::FrameSetup); + CFIBuilder.insertCFIInst( + createDefCFA(RegInfo, /*FrameReg=*/AArch64::SP, /*Reg=*/AArch64::SP, + TotalSize, /*LastAdjustmentWasScalable=*/false)); + } + emitCalleeSavedGPRLocations(AfterSVESavesI); + emitCalleeSavedSVELocations(AfterSVESavesI); + } +} + +void AArch64PrologueEmitter::emitShadowCallStackPrologue( + MachineBasicBlock::iterator MBBI, const DebugLoc &DL) const { + // Shadow call stack prolog: str x30, [x18], #8 + BuildMI(MBB, MBBI, DL, TII->get(AArch64::STRXpost)) + .addReg(AArch64::X18, RegState::Define) + .addReg(AArch64::LR) + .addReg(AArch64::X18) + .addImm(8) + .setMIFlag(MachineInstr::FrameSetup); + + // This instruction also makes x18 live-in to the entry block. + MBB.addLiveIn(AArch64::X18); + + if (NeedsWinCFI) + BuildMI(MBB, MBBI, DL, TII->get(AArch64::SEH_Nop)) + .setMIFlag(MachineInstr::FrameSetup); + + if (EmitCFI) { + // Emit a CFI instruction that causes 8 to be subtracted from the value of + // x18 when unwinding past this frame. + static const char CFIInst[] = { + dwarf::DW_CFA_val_expression, + 18, // register + 2, // length + static_cast<char>(unsigned(dwarf::DW_OP_breg18)), + static_cast<char>(-8) & 0x7f, // addend (sleb128) + }; + CFIInstBuilder(MBB, MBBI, MachineInstr::FrameSetup) + .buildEscape(StringRef(CFIInst, sizeof(CFIInst))); + } +} + +void AArch64PrologueEmitter::emitSwiftAsyncContextFramePointer( + MachineBasicBlock::iterator MBBI, const DebugLoc &DL) const { + switch (MF.getTarget().Options.SwiftAsyncFramePointer) { + case SwiftAsyncFramePointerMode::DeploymentBased: + if (Subtarget.swiftAsyncContextIsDynamicallySet()) { + // The special symbol below is absolute and has a *value* that can be + // combined with the frame pointer to signal an extended frame. + BuildMI(MBB, MBBI, DL, TII->get(AArch64::LOADgot), AArch64::X16) + .addExternalSymbol("swift_async_extendedFramePointerFlags", + AArch64II::MO_GOT); + if (NeedsWinCFI) { + BuildMI(MBB, MBBI, DL, TII->get(AArch64::SEH_Nop)) + .setMIFlags(MachineInstr::FrameSetup); + HasWinCFI = true; + } + BuildMI(MBB, MBBI, DL, TII->get(AArch64::ORRXrs), AArch64::FP) + .addUse(AArch64::FP) + .addUse(AArch64::X16) + .addImm(Subtarget.isTargetILP32() ? 32 : 0); + if (NeedsWinCFI) { + BuildMI(MBB, MBBI, DL, TII->get(AArch64::SEH_Nop)) + .setMIFlags(MachineInstr::FrameSetup); + HasWinCFI = true; + } + break; + } + [[fallthrough]]; + + case SwiftAsyncFramePointerMode::Always: + // ORR x29, x29, #0x1000_0000_0000_0000 + BuildMI(MBB, MBBI, DL, TII->get(AArch64::ORRXri), AArch64::FP) + .addUse(AArch64::FP) + .addImm(0x1100) + .setMIFlag(MachineInstr::FrameSetup); + if (NeedsWinCFI) { + BuildMI(MBB, MBBI, DL, TII->get(AArch64::SEH_Nop)) + .setMIFlags(MachineInstr::FrameSetup); + HasWinCFI = true; + } + break; + + case SwiftAsyncFramePointerMode::Never: + break; + } +} + +void AArch64PrologueEmitter::emitEmptyStackFramePrologue( + int64_t NumBytes, MachineBasicBlock::iterator MBBI, + const DebugLoc &DL) const { + assert(!HasFP && "unexpected function without stack frame but with FP"); + assert(!AFL.getSVEStackSize(MF) && + "unexpected function without stack frame but with SVE objects"); + // All of the stack allocation is for locals. + AFI->setLocalStackSize(NumBytes); + if (!NumBytes) { + if (NeedsWinCFI && HasWinCFI) { + BuildMI(MBB, MBBI, DL, TII->get(AArch64::SEH_PrologEnd)) + .setMIFlag(MachineInstr::FrameSetup); + } + return; + } + // REDZONE: If the stack size is less than 128 bytes, we don't need + // to actually allocate. + if (AFL.canUseRedZone(MF)) { + AFI->setHasRedZone(true); + ++NumRedZoneFunctions; + } else { + emitFrameOffset(MBB, MBBI, DL, AArch64::SP, AArch64::SP, + StackOffset::getFixed(-NumBytes), TII, + MachineInstr::FrameSetup, false, NeedsWinCFI, &HasWinCFI); + if (EmitCFI) { + // Label used to tie together the PROLOG_LABEL and the MachineMoves. + MCSymbol *FrameLabel = MF.getContext().createTempSymbol(); + // Encode the stack size of the leaf function. + CFIInstBuilder(MBB, MBBI, MachineInstr::FrameSetup) + .buildDefCFAOffset(NumBytes, FrameLabel); + } + } + + if (NeedsWinCFI) { + HasWinCFI = true; + BuildMI(MBB, MBBI, DL, TII->get(AArch64::SEH_PrologEnd)) + .setMIFlag(MachineInstr::FrameSetup); + } +} + +void AArch64PrologueEmitter::emitFramePointerSetup( + MachineBasicBlock::iterator MBBI, const DebugLoc &DL, + unsigned FixedObject) { + int64_t FPOffset = AFI->getCalleeSaveBaseToFrameRecordOffset(); + if (CombineSPBump) + FPOffset += AFI->getLocalStackSize(); + + if (AFI->hasSwiftAsyncContext()) { + // Before we update the live FP we have to ensure there's a valid (or + // null) asynchronous context in its slot just before FP in the frame + // record, so store it now. + const auto &Attrs = MF.getFunction().getAttributes(); + bool HaveInitialContext = Attrs.hasAttrSomewhere(Attribute::SwiftAsync); + if (HaveInitialContext) + MBB.addLiveIn(AArch64::X22); + Register Reg = HaveInitialContext ? AArch64::X22 : AArch64::XZR; + BuildMI(MBB, MBBI, DL, TII->get(AArch64::StoreSwiftAsyncContext)) + .addUse(Reg) + .addUse(AArch64::SP) + .addImm(FPOffset - 8) + .setMIFlags(MachineInstr::FrameSetup); + if (NeedsWinCFI) { + // WinCFI and arm64e, where StoreSwiftAsyncContext is expanded + // to multiple instructions, should be mutually-exclusive. + assert(Subtarget.getTargetTriple().getArchName() != "arm64e"); + BuildMI(MBB, MBBI, DL, TII->get(AArch64::SEH_Nop)) + .setMIFlags(MachineInstr::FrameSetup); + HasWinCFI = true; + } + } + + if (HomPrologEpilog) { + auto Prolog = MBBI; + --Prolog; + assert(Prolog->getOpcode() == AArch64::HOM_Prolog); + Prolog->addOperand(MachineOperand::CreateImm(FPOffset)); + } else { + // Issue sub fp, sp, FPOffset or + // mov fp,sp when FPOffset is zero. + // Note: All stores of callee-saved registers are marked as "FrameSetup". + // This code marks the instruction(s) that set the FP also. + emitFrameOffset(MBB, MBBI, DL, AArch64::FP, AArch64::SP, + StackOffset::getFixed(FPOffset), TII, + MachineInstr::FrameSetup, false, NeedsWinCFI, &HasWinCFI); + if (NeedsWinCFI && HasWinCFI) { + BuildMI(MBB, MBBI, DL, TII->get(AArch64::SEH_PrologEnd)) + .setMIFlag(MachineInstr::FrameSetup); + // After setting up the FP, the rest of the prolog doesn't need to be + // included in the SEH unwind info. + NeedsWinCFI = false; + } + } + if (EmitAsyncCFI) + emitDefineCFAWithFP(MBBI, FixedObject); +} + +// Define the current CFA rule to use the provided FP. +void AArch64PrologueEmitter::emitDefineCFAWithFP( + MachineBasicBlock::iterator MBBI, unsigned FixedObject) const { + const AArch64RegisterInfo *TRI = Subtarget.getRegisterInfo(); + const int OffsetToFirstCalleeSaveFromFP = + AFI->getCalleeSaveBaseToFrameRecordOffset() - + AFI->getCalleeSavedStackSize(); + Register FramePtr = TRI->getFrameRegister(MF); + CFIInstBuilder(MBB, MBBI, MachineInstr::FrameSetup) + .buildDefCFA(FramePtr, FixedObject - OffsetToFirstCalleeSaveFromFP); +} + +void AArch64PrologueEmitter::emitWindowsStackProbe( + MachineBasicBlock::iterator MBBI, const DebugLoc &DL, int64_t &NumBytes, + int64_t RealignmentPadding) const { + if (AFI->getSVECalleeSavedStackSize()) + report_fatal_error("SVE callee saves not yet supported with stack probing"); + + // Find an available register to spill the value of X15 to, if X15 is being + // used already for nest. + unsigned X15Scratch = AArch64::NoRegister; + const AArch64Subtarget &STI = MF.getSubtarget<AArch64Subtarget>(); + if (llvm::any_of(MBB.liveins(), + [&STI](const MachineBasicBlock::RegisterMaskPair &LiveIn) { + return STI.getRegisterInfo()->isSuperOrSubRegisterEq( + AArch64::X15, LiveIn.PhysReg); + })) { + X15Scratch = AFL.findScratchNonCalleeSaveRegister(&MBB, /*HasCall=*/true); + assert(X15Scratch != AArch64::NoRegister && + (X15Scratch < AArch64::X15 || X15Scratch > AArch64::X17)); +#ifndef NDEBUG + LiveRegs.removeReg(AArch64::X15); // ignore X15 since we restore it +#endif + BuildMI(MBB, MBBI, DL, TII->get(AArch64::ORRXrr), X15Scratch) + .addReg(AArch64::XZR) + .addReg(AArch64::X15, RegState::Undef) + .addReg(AArch64::X15, RegState::Implicit) + .setMIFlag(MachineInstr::FrameSetup); + } + + uint64_t NumWords = (NumBytes + RealignmentPadding) >> 4; + if (NeedsWinCFI) { + HasWinCFI = true; + // alloc_l can hold at most 256MB, so assume that NumBytes doesn't + // exceed this amount. We need to move at most 2^24 - 1 into x15. + // This is at most two instructions, MOVZ followed by MOVK. + // TODO: Fix to use multiple stack alloc unwind codes for stacks + // exceeding 256MB in size. + if (NumBytes >= (1 << 28)) + report_fatal_error("Stack size cannot exceed 256MB for stack " + "unwinding purposes"); + + uint32_t LowNumWords = NumWords & 0xFFFF; + BuildMI(MBB, MBBI, DL, TII->get(AArch64::MOVZXi), AArch64::X15) + .addImm(LowNumWords) + .addImm(AArch64_AM::getShifterImm(AArch64_AM::LSL, 0)) + .setMIFlag(MachineInstr::FrameSetup); + BuildMI(MBB, MBBI, DL, TII->get(AArch64::SEH_Nop)) + .setMIFlag(MachineInstr::FrameSetup); + if ((NumWords & 0xFFFF0000) != 0) { + BuildMI(MBB, MBBI, DL, TII->get(AArch64::MOVKXi), AArch64::X15) + .addReg(AArch64::X15) + .addImm((NumWords & 0xFFFF0000) >> 16) // High half + .addImm(AArch64_AM::getShifterImm(AArch64_AM::LSL, 16)) + .setMIFlag(MachineInstr::FrameSetup); + BuildMI(MBB, MBBI, DL, TII->get(AArch64::SEH_Nop)) + .setMIFlag(MachineInstr::FrameSetup); + } + } else { + BuildMI(MBB, MBBI, DL, TII->get(AArch64::MOVi64imm), AArch64::X15) + .addImm(NumWords) + .setMIFlags(MachineInstr::FrameSetup); + } + + const char *ChkStk = Subtarget.getChkStkName(); + switch (MF.getTarget().getCodeModel()) { + case CodeModel::Tiny: + case CodeModel::Small: + case CodeModel::Medium: + case CodeModel::Kernel: + BuildMI(MBB, MBBI, DL, TII->get(AArch64::BL)) + .addExternalSymbol(ChkStk) + .addReg(AArch64::X15, RegState::Implicit) + .addReg(AArch64::X16, + RegState::Implicit | RegState::Define | RegState::Dead) + .addReg(AArch64::X17, + RegState::Implicit | RegState::Define | RegState::Dead) + .addReg(AArch64::NZCV, + RegState::Implicit | RegState::Define | RegState::Dead) + .setMIFlags(MachineInstr::FrameSetup); + if (NeedsWinCFI) { + HasWinCFI = true; + BuildMI(MBB, MBBI, DL, TII->get(AArch64::SEH_Nop)) + .setMIFlag(MachineInstr::FrameSetup); + } + break; + case CodeModel::Large: + BuildMI(MBB, MBBI, DL, TII->get(AArch64::MOVaddrEXT)) + .addReg(AArch64::X16, RegState::Define) + .addExternalSymbol(ChkStk) + .addExternalSymbol(ChkStk) + .setMIFlags(MachineInstr::FrameSetup); + if (NeedsWinCFI) { + HasWinCFI = true; + BuildMI(MBB, MBBI, DL, TII->get(AArch64::SEH_Nop)) + .setMIFlag(MachineInstr::FrameSetup); + } + + BuildMI(MBB, MBBI, DL, TII->get(getBLRCallOpcode(MF))) + .addReg(AArch64::X16, RegState::Kill) + .addReg(AArch64::X15, RegState::Implicit | RegState::Define) + .addReg(AArch64::X16, + RegState::Implicit | RegState::Define | RegState::Dead) + .addReg(AArch64::X17, + RegState::Implicit | RegState::Define | RegState::Dead) + .addReg(AArch64::NZCV, + RegState::Implicit | RegState::Define | RegState::Dead) + .setMIFlags(MachineInstr::FrameSetup); + if (NeedsWinCFI) { + HasWinCFI = true; + BuildMI(MBB, MBBI, DL, TII->get(AArch64::SEH_Nop)) + .setMIFlag(MachineInstr::FrameSetup); + } + break; + } + + BuildMI(MBB, MBBI, DL, TII->get(AArch64::SUBXrx64), AArch64::SP) + .addReg(AArch64::SP, RegState::Kill) + .addReg(AArch64::X15, RegState::Kill) + .addImm(AArch64_AM::getArithExtendImm(AArch64_AM::UXTX, 4)) + .setMIFlags(MachineInstr::FrameSetup); + if (NeedsWinCFI) { + HasWinCFI = true; + BuildMI(MBB, MBBI, DL, TII->get(AArch64::SEH_StackAlloc)) + .addImm(NumBytes) + .setMIFlag(MachineInstr::FrameSetup); + } + NumBytes = 0; + + if (RealignmentPadding > 0) { + if (RealignmentPadding >= 4096) { + BuildMI(MBB, MBBI, DL, TII->get(AArch64::MOVi64imm)) + .addReg(AArch64::X16, RegState::Define) + .addImm(RealignmentPadding) + .setMIFlags(MachineInstr::FrameSetup); + BuildMI(MBB, MBBI, DL, TII->get(AArch64::ADDXrx64), AArch64::X15) + .addReg(AArch64::SP) + .addReg(AArch64::X16, RegState::Kill) + .addImm(AArch64_AM::getArithExtendImm(AArch64_AM::UXTX, 0)) + .setMIFlag(MachineInstr::FrameSetup); + } else { + BuildMI(MBB, MBBI, DL, TII->get(AArch64::ADDXri), AArch64::X15) + .addReg(AArch64::SP) + .addImm(RealignmentPadding) + .addImm(0) + .setMIFlag(MachineInstr::FrameSetup); + } + + uint64_t AndMask = ~(MFI.getMaxAlign().value() - 1); + BuildMI(MBB, MBBI, DL, TII->get(AArch64::ANDXri), AArch64::SP) + .addReg(AArch64::X15, RegState::Kill) + .addImm(AArch64_AM::encodeLogicalImmediate(AndMask, 64)); + AFI->setStackRealigned(true); + + // No need for SEH instructions here; if we're realigning the stack, + // we've set a frame pointer and already finished the SEH prologue. + assert(!NeedsWinCFI); + } + if (X15Scratch != AArch64::NoRegister) { + BuildMI(MBB, MBBI, DL, TII->get(AArch64::ORRXrr), AArch64::X15) + .addReg(AArch64::XZR) + .addReg(X15Scratch, RegState::Undef) + .addReg(X15Scratch, RegState::Implicit) + .setMIFlag(MachineInstr::FrameSetup); + } +} + +void AArch64PrologueEmitter::emitCalleeSavedGPRLocations( + MachineBasicBlock::iterator MBBI) const { + MachineFunction &MF = *MBB.getParent(); + MachineFrameInfo &MFI = MF.getFrameInfo(); + + const std::vector<CalleeSavedInfo> &CSI = MFI.getCalleeSavedInfo(); + if (CSI.empty()) + return; + + CFIInstBuilder CFIBuilder(MBB, MBBI, MachineInstr::FrameSetup); + for (const auto &Info : CSI) { + unsigned FrameIdx = Info.getFrameIdx(); + if (MFI.getStackID(FrameIdx) == TargetStackID::ScalableVector) + continue; + + assert(!Info.isSpilledToReg() && "Spilling to registers not implemented"); + int64_t Offset = MFI.getObjectOffset(FrameIdx) - AFL.getOffsetOfLocalArea(); + CFIBuilder.buildOffset(Info.getReg(), Offset); + } +} + +void AArch64PrologueEmitter::emitCalleeSavedSVELocations( + MachineBasicBlock::iterator MBBI) const { + MachineFunction &MF = *MBB.getParent(); + MachineFrameInfo &MFI = MF.getFrameInfo(); + + // Add callee saved registers to move list. + const std::vector<CalleeSavedInfo> &CSI = MFI.getCalleeSavedInfo(); + if (CSI.empty()) + return; + + const TargetSubtargetInfo &STI = MF.getSubtarget(); + const TargetRegisterInfo &TRI = *STI.getRegisterInfo(); + AArch64FunctionInfo &AFI = *MF.getInfo<AArch64FunctionInfo>(); + CFIInstBuilder CFIBuilder(MBB, MBBI, MachineInstr::FrameSetup); + + std::optional<int64_t> IncomingVGOffsetFromDefCFA; + if (AFL.requiresSaveVG(MF)) { + auto IncomingVG = *find_if( + reverse(CSI), [](auto &Info) { return Info.getReg() == AArch64::VG; }); + IncomingVGOffsetFromDefCFA = MFI.getObjectOffset(IncomingVG.getFrameIdx()) - + AFL.getOffsetOfLocalArea(); + } + + for (const auto &Info : CSI) { + if (MFI.getStackID(Info.getFrameIdx()) != TargetStackID::ScalableVector) + continue; + + // Not all unwinders may know about SVE registers, so assume the lowest + // common denominator. + assert(!Info.isSpilledToReg() && "Spilling to registers not implemented"); + MCRegister Reg = Info.getReg(); + if (!static_cast<const AArch64RegisterInfo &>(TRI).regNeedsCFI(Reg, Reg)) + continue; + + StackOffset Offset = + StackOffset::getScalable(MFI.getObjectOffset(Info.getFrameIdx())) - + StackOffset::getFixed(AFI.getCalleeSavedStackSize(MFI)); + + CFIBuilder.insertCFIInst( + createCFAOffset(TRI, Reg, Offset, IncomingVGOffsetFromDefCFA)); + } +} + +} // namespace llvm diff --git a/llvm/lib/Target/AArch64/AArch64PrologueEpilogue.h b/llvm/lib/Target/AArch64/AArch64PrologueEpilogue.h new file mode 100644 index 000000000000..94029ede60c7 --- /dev/null +++ b/llvm/lib/Target/AArch64/AArch64PrologueEpilogue.h @@ -0,0 +1,111 @@ +//===----------------------------------------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +/// +/// \file +/// This file contains the declaration of the AArch64PrologueEmitter class, +/// which is is used to emit the prologue on AArch64. +/// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_LIB_TARGET_AARCH64_AARCH64PROLOGUEEPILOGUE_H +#define LLVM_LIB_TARGET_AARCH64_AARCH64PROLOGUEEPILOGUE_H + +#include "AArch64RegisterInfo.h" +#include "llvm/CodeGen/LivePhysRegs.h" +#include "llvm/CodeGen/MachineFunction.h" + +namespace llvm { + +class AArch64Subtarget; +class AArch64FunctionInfo; +class AArch64FrameLowering; + +/// A helper class for emitting the prologue. Substantial new functionality +/// should be factored into a new method. Where possible "emit*" methods should +/// be const, and any flags that change how the prologue is emitted should be +/// set in the constructor. +class AArch64PrologueEmitter { +public: + AArch64PrologueEmitter(MachineFunction &MF, MachineBasicBlock &MBB, + const AArch64FrameLowering &AFL); + + /// Emit the prologue. + void emitPrologue(); + + ~AArch64PrologueEmitter() { + MF.setHasWinCFI(HasWinCFI); +#ifndef NDEBUG + verifyPrologueClobbers(); +#endif + } + +private: + void emitShadowCallStackPrologue(MachineBasicBlock::iterator MBBI, + const DebugLoc &DL) const; + + void emitSwiftAsyncContextFramePointer(MachineBasicBlock::iterator MBBI, + const DebugLoc &DL) const; + + void emitEmptyStackFramePrologue(int64_t NumBytes, + MachineBasicBlock::iterator MBBI, + const DebugLoc &DL) const; + + void emitFramePointerSetup(MachineBasicBlock::iterator MBBI, + const DebugLoc &DL, unsigned FixedObject); + + void emitDefineCFAWithFP(MachineBasicBlock::iterator MBBI, + unsigned FixedObject) const; + + void emitWindowsStackProbe(MachineBasicBlock::iterator MBBI, + const DebugLoc &DL, int64_t &NumBytes, + int64_t RealignmentPadding) const; + + void emitCalleeSavedGPRLocations(MachineBasicBlock::iterator MBBI) const; + void emitCalleeSavedSVELocations(MachineBasicBlock::iterator MBBI) const; + + void determineLocalsStackSize(uint64_t StackSize, uint64_t PrologueSaveSize); + + MachineFunction &MF; + MachineBasicBlock &MBB; + + const Function &F; + const MachineFrameInfo &MFI; + const AArch64Subtarget &Subtarget; + const AArch64FrameLowering &AFL; + const AArch64RegisterInfo &RegInfo; + +#ifndef NDEBUG + mutable LivePhysRegs LiveRegs{RegInfo}; + MachineBasicBlock::iterator PrologueEndI; + + void collectBlockLiveins(); + void verifyPrologueClobbers() const; +#endif + + // Prologue flags. These generally should not change outside of the + // constructor. Two exceptions are "CombineSPBump" which is set in + // determineLocalsStackSize, and "NeedsWinCFI" which is set in + // emitFramePointerSetup. + bool EmitCFI = false; + bool EmitAsyncCFI = false; + bool HasFP = false; + bool IsFunclet = false; + bool CombineSPBump = false; + bool HomPrologEpilog = false; + bool NeedsWinCFI = false; + + // Note: "HasWinCFI" is mutable as it can change in any "emit" function. + mutable bool HasWinCFI = false; + + const TargetInstrInfo *TII = nullptr; + AArch64FunctionInfo *AFI = nullptr; +}; + +} // namespace llvm + +#endif diff --git a/llvm/lib/Target/AArch64/AArch64RegisterInfo.td b/llvm/lib/Target/AArch64/AArch64RegisterInfo.td index 1a7609bfee8a..d8ffea85a1c2 100644 --- a/llvm/lib/Target/AArch64/AArch64RegisterInfo.td +++ b/llvm/lib/Target/AArch64/AArch64RegisterInfo.td @@ -983,7 +983,7 @@ class ZPRRegOp <string Suffix, AsmOperandClass C, ElementSizeEnum Size, // Note: This hardware mode is enabled in AArch64Subtarget::getHwModeSet() // (without the use of the table-gen'd predicates). -def SMEWithZPRPredicateSpills : HwMode<"", [Predicate<"false">]>; +def SMEWithZPRPredicateSpills : HwMode<[Predicate<"false">]>; def PPRSpillFillRI : RegInfoByHwMode< [DefaultMode, SMEWithZPRPredicateSpills], @@ -1164,25 +1164,22 @@ class PPRVectorListMul<int ElementWidth, int NumRegs> : PPRVectorList<ElementWid ", AArch64::PPRMul2RegClassID>"; } +class PPR2MulRegOp<string Suffix, int Size, ElementSizeEnum ES> + : RegisterOperand<PPR2Mul2, "printTypedVectorList<0,'"#Suffix#"'>"> { + ElementSizeEnum ElementSize; + let ElementSize = ES; + let ParserMatchClass = PPRVectorListMul<Size, 2>; +} + let EncoderMethod = "EncodeRegMul_MinMax<2, 0, 14>", DecoderMethod = "DecodePPR2Mul2RegisterClass" in { - def PP_b_mul_r : RegisterOperand<PPR2Mul2, "printTypedVectorList<0,'b'>"> { - let ParserMatchClass = PPRVectorListMul<8, 2>; - } - - def PP_h_mul_r : RegisterOperand<PPR2Mul2, "printTypedVectorList<0,'h'>"> { - let ParserMatchClass = PPRVectorListMul<16, 2>; - } - def PP_s_mul_r : RegisterOperand<PPR2Mul2, "printTypedVectorList<0,'s'>"> { - let ParserMatchClass = PPRVectorListMul<32, 2>; - } - - def PP_d_mul_r : RegisterOperand<PPR2Mul2, "printTypedVectorList<0,'d'>"> { - let ParserMatchClass = PPRVectorListMul<64, 2>; - } -} // end let EncoderMethod/DecoderMethod + def PP_b_mul_r : PPR2MulRegOp<"b", 8, ElementSizeB>; + def PP_h_mul_r : PPR2MulRegOp<"h", 16, ElementSizeH>; + def PP_s_mul_r : PPR2MulRegOp<"s", 32, ElementSizeS>; + def PP_d_mul_r : PPR2MulRegOp<"d", 64, ElementSizeD>; +} // end let EncoderMethod/DecoderMethod //===----------------------------------------------------------------------===// // SVE vector register classes diff --git a/llvm/lib/Target/AArch64/AArch64SVEInstrInfo.td b/llvm/lib/Target/AArch64/AArch64SVEInstrInfo.td index 05a3eab638ea..7604ffdc9f64 100644 --- a/llvm/lib/Target/AArch64/AArch64SVEInstrInfo.td +++ b/llvm/lib/Target/AArch64/AArch64SVEInstrInfo.td @@ -412,6 +412,7 @@ def SDT_AArch64PTest : SDTypeProfile<1, 2, [ ]>; def AArch64ptest : SDNode<"AArch64ISD::PTEST", SDT_AArch64PTest>; def AArch64ptest_any : SDNode<"AArch64ISD::PTEST_ANY", SDT_AArch64PTest>; +def AArch64ptest_first : SDNode<"AArch64ISD::PTEST_FIRST", SDT_AArch64PTest>; def SDT_AArch64DUP_PRED : SDTypeProfile<1, 3, [SDTCisVec<0>, SDTCisSameAs<0, 3>, SDTCisVec<1>, SDTCVecEltisVT<1,i1>, SDTCisSameNumEltsAs<0, 1>]>; @@ -1071,7 +1072,7 @@ let Predicates = [HasSVE_or_SME] in { defm BRKB_PPmP : sve_int_break_m<0b101, "brkb", int_aarch64_sve_brkb>; defm BRKBS_PPzP : sve_int_break_z<0b110, "brkbs", null_frag>; - defm PTEST_PP : sve_int_ptest<0b010000, "ptest", AArch64ptest, AArch64ptest_any>; + defm PTEST_PP : sve_int_ptest<0b010000, "ptest", AArch64ptest, AArch64ptest_any, AArch64ptest_first>; defm PFALSE : sve_int_pfalse<0b000000, "pfalse">; defm PFIRST : sve_int_pfirst<0b00000, "pfirst", int_aarch64_sve_pfirst>; defm PNEXT : sve_int_pnext<0b00110, "pnext", int_aarch64_sve_pnext>; diff --git a/llvm/lib/Target/AArch64/AArch64Subtarget.cpp b/llvm/lib/Target/AArch64/AArch64Subtarget.cpp index 0f4f0129e9cd..98e0a1180510 100644 --- a/llvm/lib/Target/AArch64/AArch64Subtarget.cpp +++ b/llvm/lib/Target/AArch64/AArch64Subtarget.cpp @@ -91,6 +91,10 @@ static cl::opt<bool> EnableZPRPredicateSpills( cl::desc( "Enables spilling/reloading SVE predicates as data vectors (ZPRs)")); +static cl::opt<unsigned> + VScaleForTuningOpt("sve-vscale-for-tuning", cl::Hidden, + cl::desc("Force a vscale for tuning factor for SVE")); + // Subreg liveness tracking is disabled by default for now until all issues // are ironed out. This option allows the feature to be used in tests. static cl::opt<bool> @@ -364,6 +368,8 @@ void AArch64Subtarget::initializeProperties(bool HasMinSize) { if (AArch64MinimumJumpTableEntries.getNumOccurrences() > 0 || !HasMinSize) MinimumJumpTableEntries = AArch64MinimumJumpTableEntries; + if (VScaleForTuningOpt.getNumOccurrences() > 0) + VScaleForTuning = VScaleForTuningOpt; } AArch64Subtarget::AArch64Subtarget(const Triple &TT, StringRef CPU, diff --git a/llvm/lib/Target/AArch64/AArch64TargetMachine.cpp b/llvm/lib/Target/AArch64/AArch64TargetMachine.cpp index 4650b2d0c815..dde1d88403bf 100644 --- a/llvm/lib/Target/AArch64/AArch64TargetMachine.cpp +++ b/llvm/lib/Target/AArch64/AArch64TargetMachine.cpp @@ -295,27 +295,6 @@ static std::unique_ptr<TargetLoweringObjectFile> createTLOF(const Triple &TT) { return std::make_unique<AArch64_ELFTargetObjectFile>(); } -// Helper function to build a DataLayout string -static std::string computeDataLayout(const Triple &TT, - const MCTargetOptions &Options, - bool LittleEndian) { - if (TT.isOSBinFormatMachO()) { - if (TT.getArch() == Triple::aarch64_32) - return "e-m:o-p:32:32-p270:32:32-p271:32:32-p272:64:64-i64:64-i128:128-" - "n32:64-S128-Fn32"; - return "e-m:o-p270:32:32-p271:32:32-p272:64:64-i64:64-i128:128-n32:64-S128-" - "Fn32"; - } - if (TT.isOSBinFormatCOFF()) - return "e-m:w-p270:32:32-p271:32:32-p272:64:64-p:64:64-i32:32-i64:64-i128:" - "128-n32:64-S128-Fn32"; - std::string Endian = LittleEndian ? "e" : "E"; - std::string Ptr32 = TT.getEnvironment() == Triple::GNUILP32 ? "-p:32:32" : ""; - return Endian + "-m:e" + Ptr32 + - "-p270:32:32-p271:32:32-p272:64:64-i8:8:32-i16:16:32-i64:64-i128:128-" - "n32:64-S128-Fn32"; -} - static StringRef computeDefaultCPU(const Triple &TT, StringRef CPU) { if (CPU.empty() && TT.isArm64e()) return "apple-a12"; @@ -368,11 +347,10 @@ AArch64TargetMachine::AArch64TargetMachine(const Target &T, const Triple &TT, std::optional<CodeModel::Model> CM, CodeGenOptLevel OL, bool JIT, bool LittleEndian) - : CodeGenTargetMachineImpl( - T, computeDataLayout(TT, Options.MCOptions, LittleEndian), TT, - computeDefaultCPU(TT, CPU), FS, Options, - getEffectiveRelocModel(TT, RM), - getEffectiveAArch64CodeModel(TT, CM, JIT), OL), + : CodeGenTargetMachineImpl(T, TT.computeDataLayout(), TT, + computeDefaultCPU(TT, CPU), FS, Options, + getEffectiveRelocModel(TT, RM), + getEffectiveAArch64CodeModel(TT, CM, JIT), OL), TLOF(createTLOF(getTargetTriple())), isLittle(LittleEndian), UseNewSMEABILowering(EnableNewSMEABILowering) { initAsmInfo(); diff --git a/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp b/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp index 47e51a1b5d12..92321a76dbd8 100644 --- a/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp +++ b/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp @@ -25,6 +25,7 @@ #include "llvm/Support/Debug.h" #include "llvm/TargetParser/AArch64TargetParser.h" #include "llvm/Transforms/InstCombine/InstCombiner.h" +#include "llvm/Transforms/Utils/UnrollLoop.h" #include "llvm/Transforms/Vectorize/LoopVectorizationLegality.h" #include <algorithm> #include <optional> @@ -4969,6 +4970,23 @@ getAppleRuntimeUnrollPreferences(Loop *L, ScalarEvolution &SE, if (!L->getExitBlock()) return; + // Check if the loop contains any reductions that could be parallelized when + // unrolling. If so, enable partial unrolling, if the trip count is know to be + // a multiple of 2. + bool HasParellelizableReductions = + L->getNumBlocks() == 1 && + any_of(L->getHeader()->phis(), + [&SE, L](PHINode &Phi) { + return canParallelizeReductionWhenUnrolling(Phi, L, &SE); + }) && + isLoopSizeWithinBudget(L, TTI, 12, nullptr); + if (HasParellelizableReductions && + SE.getSmallConstantTripMultiple(L, L->getExitingBlock()) % 2 == 0) { + UP.Partial = true; + UP.MaxCount = 4; + UP.AddAdditionalAccumulators = true; + } + const SCEV *BTC = SE.getSymbolicMaxBackedgeTakenCount(L); if (isa<SCEVConstant>(BTC) || isa<SCEVCouldNotCompute>(BTC) || (SE.getSmallConstantMaxTripCount(L) > 0 && @@ -4984,6 +5002,12 @@ getAppleRuntimeUnrollPreferences(Loop *L, ScalarEvolution &SE, // Limit to loops with trip counts that are cheap to expand. UP.SCEVExpansionBudget = 1; + if (HasParellelizableReductions) { + UP.Runtime = true; + UP.DefaultUnrollRuntimeCount = 4; + UP.AddAdditionalAccumulators = true; + } + // Try to unroll small loops, of few-blocks with low budget, if they have // load/store dependencies, to expose more parallel memory access streams, // or if they do little work inside a block (i.e. load -> X -> store pattern). diff --git a/llvm/lib/Target/AArch64/CMakeLists.txt b/llvm/lib/Target/AArch64/CMakeLists.txt index 71017b37cf49..a8185358d6df 100644 --- a/llvm/lib/Target/AArch64/CMakeLists.txt +++ b/llvm/lib/Target/AArch64/CMakeLists.txt @@ -92,6 +92,7 @@ add_llvm_target(AArch64CodeGen SVEIntrinsicOpts.cpp MachineSMEABIPass.cpp AArch64SIMDInstrOpt.cpp + AArch64PrologueEpilogue.cpp DEPENDS intrinsics_gen diff --git a/llvm/lib/Target/AArch64/Disassembler/AArch64Disassembler.cpp b/llvm/lib/Target/AArch64/Disassembler/AArch64Disassembler.cpp index 8c1e9f61693f..aa1c1c882e22 100644 --- a/llvm/lib/Target/AArch64/Disassembler/AArch64Disassembler.cpp +++ b/llvm/lib/Target/AArch64/Disassembler/AArch64Disassembler.cpp @@ -198,11 +198,7 @@ static DecodeStatus DecodePCRelLabel16(MCInst &Inst, unsigned Imm, static DecodeStatus DecodePCRelLabel19(MCInst &Inst, unsigned Imm, uint64_t Addr, const MCDisassembler *Decoder) { - int64_t ImmVal = Imm; - - // Sign-extend 19-bit immediate. - if (ImmVal & (1 << (19 - 1))) - ImmVal |= ~((1LL << 19) - 1); + int64_t ImmVal = SignExtend64<19>(Imm); if (!Decoder->tryAddingSymbolicOperand( Inst, ImmVal * 4, Addr, Inst.getOpcode() != AArch64::LDRXl, 0, 0, 4)) @@ -212,11 +208,7 @@ static DecodeStatus DecodePCRelLabel19(MCInst &Inst, unsigned Imm, static DecodeStatus DecodePCRelLabel9(MCInst &Inst, unsigned Imm, uint64_t Addr, const MCDisassembler *Decoder) { - int64_t ImmVal = Imm; - - // Sign-extend 9-bit immediate. - if (ImmVal & (1 << (9 - 1))) - ImmVal |= ~((1LL << 9) - 1); + int64_t ImmVal = SignExtend64<9>(Imm); if (!Decoder->tryAddingSymbolicOperand(Inst, (ImmVal * 4), Addr, /*IsBranch=*/true, 0, 0, 4)) @@ -533,12 +525,7 @@ static DecodeStatus DecodeSignedLdStInstruction(MCInst &Inst, uint32_t insn, const MCDisassembler *Decoder) { unsigned Rt = fieldFromInstruction(insn, 0, 5); unsigned Rn = fieldFromInstruction(insn, 5, 5); - int64_t offset = fieldFromInstruction(insn, 12, 9); - - // offset is a 9-bit signed immediate, so sign extend it to - // fill the unsigned. - if (offset & (1 << (9 - 1))) - offset |= ~((1LL << 9) - 1); + int64_t offset = SignExtend64<9>(fieldFromInstruction(insn, 12, 9)); // First operand is always the writeback to the address register, if needed. switch (Inst.getOpcode()) { @@ -835,14 +822,9 @@ static DecodeStatus DecodePairLdStInstruction(MCInst &Inst, uint32_t insn, unsigned Rt = fieldFromInstruction(insn, 0, 5); unsigned Rn = fieldFromInstruction(insn, 5, 5); unsigned Rt2 = fieldFromInstruction(insn, 10, 5); - int64_t offset = fieldFromInstruction(insn, 15, 7); + int64_t offset = SignExtend64<7>(fieldFromInstruction(insn, 15, 7)); bool IsLoad = fieldFromInstruction(insn, 22, 1); - // offset is a 7-bit signed immediate, so sign extend it to - // fill the unsigned. - if (offset & (1 << (7 - 1))) - offset |= ~((1LL << 7) - 1); - unsigned Opcode = Inst.getOpcode(); bool NeedsDisjointWritebackTransfer = false; @@ -1211,12 +1193,8 @@ static DecodeStatus DecodeAdrInstruction(MCInst &Inst, uint32_t insn, uint64_t Addr, const MCDisassembler *Decoder) { unsigned Rd = fieldFromInstruction(insn, 0, 5); - int64_t imm = fieldFromInstruction(insn, 5, 19) << 2; - imm |= fieldFromInstruction(insn, 29, 2); - - // Sign-extend the 21-bit immediate. - if (imm & (1 << (21 - 1))) - imm |= ~((1LL << 21) - 1); + int64_t imm = SignExtend64<21>((fieldFromInstruction(insn, 5, 19) << 2) | + fieldFromInstruction(insn, 29, 2)); DecodeSimpleRegisterClass<AArch64::GPR64RegClassID, 0, 32>(Inst, Rd, Addr, Decoder); @@ -1270,11 +1248,7 @@ static DecodeStatus DecodeAddSubImmShift(MCInst &Inst, uint32_t insn, static DecodeStatus DecodeUnconditionalBranch(MCInst &Inst, uint32_t insn, uint64_t Addr, const MCDisassembler *Decoder) { - int64_t imm = fieldFromInstruction(insn, 0, 26); - - // Sign-extend the 26-bit immediate. - if (imm & (1 << (26 - 1))) - imm |= ~((1LL << 26) - 1); + int64_t imm = SignExtend64<26>(fieldFromInstruction(insn, 0, 26)); if (!Decoder->tryAddingSymbolicOperand(Inst, imm * 4, Addr, true, 0, 0, 4)) Inst.addOperand(MCOperand::createImm(imm)); @@ -1337,11 +1311,7 @@ static DecodeStatus DecodeTestAndBranch(MCInst &Inst, uint32_t insn, uint64_t Rt = fieldFromInstruction(insn, 0, 5); uint64_t bit = fieldFromInstruction(insn, 31, 1) << 5; bit |= fieldFromInstruction(insn, 19, 5); - int64_t dst = fieldFromInstruction(insn, 5, 14); - - // Sign-extend 14-bit immediate. - if (dst & (1 << (14 - 1))) - dst |= ~((1LL << 14) - 1); + int64_t dst = SignExtend64<14>(fieldFromInstruction(insn, 5, 14)); if (fieldFromInstruction(insn, 31, 1) == 0) DecodeSimpleRegisterClass<AArch64::GPR32RegClassID, 0, 32>(Inst, Rt, Addr, diff --git a/llvm/lib/Target/AArch64/MachineSMEABIPass.cpp b/llvm/lib/Target/AArch64/MachineSMEABIPass.cpp index d95d170a813a..c39a5cc2fcb1 100644 --- a/llvm/lib/Target/AArch64/MachineSMEABIPass.cpp +++ b/llvm/lib/Target/AArch64/MachineSMEABIPass.cpp @@ -7,7 +7,7 @@ //===----------------------------------------------------------------------===// // // This pass implements the SME ABI requirements for ZA state. This includes -// implementing the lazy ZA state save schemes around calls. +// implementing the lazy (and agnostic) ZA state save schemes around calls. // //===----------------------------------------------------------------------===// // @@ -215,9 +215,44 @@ struct MachineSMEABI : public MachineFunctionPass { void emitZAOff(MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI, bool ClearTPIDR2); + // Emission routines for agnostic ZA functions. + void emitSetupFullZASave(MachineBasicBlock &MBB, + MachineBasicBlock::iterator MBBI, + LiveRegs PhysLiveRegs); + // Emit a "full" ZA save or restore. It is "full" in the sense that this + // function will emit a call to __arm_sme_save or __arm_sme_restore, which + // handles saving and restoring both ZA and ZT0. + void emitFullZASaveRestore(MachineBasicBlock &MBB, + MachineBasicBlock::iterator MBBI, + LiveRegs PhysLiveRegs, bool IsSave); + void emitAllocateFullZASaveBuffer(MachineBasicBlock &MBB, + MachineBasicBlock::iterator MBBI, + LiveRegs PhysLiveRegs); + void emitStateChange(MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI, ZAState From, ZAState To, LiveRegs PhysLiveRegs); + // Helpers for switching between lazy/full ZA save/restore routines. + void emitZASave(MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI, + LiveRegs PhysLiveRegs) { + if (AFI->getSMEFnAttrs().hasAgnosticZAInterface()) + return emitFullZASaveRestore(MBB, MBBI, PhysLiveRegs, /*IsSave=*/true); + return emitSetupLazySave(MBB, MBBI); + } + void emitZARestore(MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI, + LiveRegs PhysLiveRegs) { + if (AFI->getSMEFnAttrs().hasAgnosticZAInterface()) + return emitFullZASaveRestore(MBB, MBBI, PhysLiveRegs, /*IsSave=*/false); + return emitRestoreLazySave(MBB, MBBI, PhysLiveRegs); + } + void emitAllocateZASaveBuffer(MachineBasicBlock &MBB, + MachineBasicBlock::iterator MBBI, + LiveRegs PhysLiveRegs) { + if (AFI->getSMEFnAttrs().hasAgnosticZAInterface()) + return emitAllocateFullZASaveBuffer(MBB, MBBI, PhysLiveRegs); + return emitAllocateLazySaveBuffer(MBB, MBBI); + } + /// Save live physical registers to virtual registers. PhysRegSave createPhysRegSave(LiveRegs PhysLiveRegs, MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI, DebugLoc DL); @@ -228,6 +263,8 @@ struct MachineSMEABI : public MachineFunctionPass { /// Get or create a TPIDR2 block in this function. TPIDR2State getTPIDR2Block(); + Register getAgnosticZABufferPtr(); + private: /// Contains the needed ZA state (and live registers) at an instruction. struct InstInfo { @@ -241,6 +278,7 @@ private: struct BlockInfo { ZAState FixedEntryState{ZAState::ANY}; SmallVector<InstInfo> Insts; + LiveRegs PhysLiveRegsAtEntry = LiveRegs::None; LiveRegs PhysLiveRegsAtExit = LiveRegs::None; }; @@ -250,18 +288,22 @@ private: SmallVector<ZAState> BundleStates; std::optional<TPIDR2State> TPIDR2Block; std::optional<MachineBasicBlock::iterator> AfterSMEProloguePt; + Register AgnosticZABufferPtr = AArch64::NoRegister; + LiveRegs PhysLiveRegsAfterSMEPrologue = LiveRegs::None; } State; MachineFunction *MF = nullptr; EdgeBundles *Bundles = nullptr; const AArch64Subtarget *Subtarget = nullptr; const AArch64RegisterInfo *TRI = nullptr; + const AArch64FunctionInfo *AFI = nullptr; const TargetInstrInfo *TII = nullptr; MachineRegisterInfo *MRI = nullptr; }; void MachineSMEABI::collectNeededZAStates(SMEAttrs SMEFnAttrs) { - assert((SMEFnAttrs.hasZT0State() || SMEFnAttrs.hasZAState()) && + assert((SMEFnAttrs.hasAgnosticZAInterface() || SMEFnAttrs.hasZT0State() || + SMEFnAttrs.hasZAState()) && "Expected function to have ZA/ZT0 state!"); State.Blocks.resize(MF->getNumBlockIDs()); @@ -295,6 +337,7 @@ void MachineSMEABI::collectNeededZAStates(SMEAttrs SMEFnAttrs) { Block.PhysLiveRegsAtExit = GetPhysLiveRegs(); auto FirstTerminatorInsertPt = MBB.getFirstTerminator(); + auto FirstNonPhiInsertPt = MBB.getFirstNonPHI(); for (MachineInstr &MI : reverse(MBB)) { MachineBasicBlock::iterator MBBI(MI); LiveUnits.stepBackward(MI); @@ -303,8 +346,11 @@ void MachineSMEABI::collectNeededZAStates(SMEAttrs SMEFnAttrs) { // buffer was allocated in SelectionDAG. It marks the end of the // allocation -- which is a safe point for this pass to insert any TPIDR2 // block setup. - if (MI.getOpcode() == AArch64::SMEStateAllocPseudo) + if (MI.getOpcode() == AArch64::SMEStateAllocPseudo) { State.AfterSMEProloguePt = MBBI; + State.PhysLiveRegsAfterSMEPrologue = PhysLiveRegs; + } + // Note: We treat Agnostic ZA as inout_za with an alternate save/restore. auto [NeededState, InsertPt] = getZAStateBeforeInst( *TRI, MI, /*ZAOffAtReturn=*/SMEFnAttrs.hasPrivateZAInterface()); assert((InsertPt == MBBI || @@ -313,6 +359,8 @@ void MachineSMEABI::collectNeededZAStates(SMEAttrs SMEFnAttrs) { // TODO: Do something to avoid state changes where NZCV is live. if (MBBI == FirstTerminatorInsertPt) Block.PhysLiveRegsAtExit = PhysLiveRegs; + if (MBBI == FirstNonPhiInsertPt) + Block.PhysLiveRegsAtEntry = PhysLiveRegs; if (NeededState != ZAState::ANY) Block.Insts.push_back({NeededState, InsertPt, PhysLiveRegs}); } @@ -536,8 +584,6 @@ void MachineSMEABI::emitZAOff(MachineBasicBlock &MBB, void MachineSMEABI::emitAllocateLazySaveBuffer( MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI) { MachineFrameInfo &MFI = MF->getFrameInfo(); - auto *AFI = MF->getInfo<AArch64FunctionInfo>(); - DebugLoc DL = getDebugLoc(MBB, MBBI); Register SP = MRI->createVirtualRegister(&AArch64::GPR64RegClass); Register SVL = MRI->createVirtualRegister(&AArch64::GPR64RegClass); @@ -601,8 +647,7 @@ void MachineSMEABI::emitNewZAPrologue(MachineBasicBlock &MBB, .addImm(AArch64SysReg::TPIDR2_EL0); // If TPIDR2_EL0 is non-zero, commit the lazy save. // NOTE: Functions that only use ZT0 don't need to zero ZA. - bool ZeroZA = - MF->getInfo<AArch64FunctionInfo>()->getSMEFnAttrs().hasZAState(); + bool ZeroZA = AFI->getSMEFnAttrs().hasZAState(); auto CommitZASave = BuildMI(MBB, MBBI, DL, TII->get(AArch64::CommitZASavePseudo)) .addReg(TPIDR2EL0) @@ -617,6 +662,86 @@ void MachineSMEABI::emitNewZAPrologue(MachineBasicBlock &MBB, .addImm(1); } +Register MachineSMEABI::getAgnosticZABufferPtr() { + if (State.AgnosticZABufferPtr != AArch64::NoRegister) + return State.AgnosticZABufferPtr; + Register BufferPtr = AFI->getEarlyAllocSMESaveBuffer(); + State.AgnosticZABufferPtr = + BufferPtr != AArch64::NoRegister + ? BufferPtr + : MF->getRegInfo().createVirtualRegister(&AArch64::GPR64RegClass); + return State.AgnosticZABufferPtr; +} + +void MachineSMEABI::emitFullZASaveRestore(MachineBasicBlock &MBB, + MachineBasicBlock::iterator MBBI, + LiveRegs PhysLiveRegs, bool IsSave) { + auto *TLI = Subtarget->getTargetLowering(); + DebugLoc DL = getDebugLoc(MBB, MBBI); + Register BufferPtr = AArch64::X0; + + PhysRegSave RegSave = createPhysRegSave(PhysLiveRegs, MBB, MBBI, DL); + + // Copy the buffer pointer into X0. + BuildMI(MBB, MBBI, DL, TII->get(TargetOpcode::COPY), BufferPtr) + .addReg(getAgnosticZABufferPtr()); + + // Call __arm_sme_save/__arm_sme_restore. + BuildMI(MBB, MBBI, DL, TII->get(AArch64::BL)) + .addReg(BufferPtr, RegState::Implicit) + .addExternalSymbol(TLI->getLibcallName( + IsSave ? RTLIB::SMEABI_SME_SAVE : RTLIB::SMEABI_SME_RESTORE)) + .addRegMask(TRI->getCallPreservedMask( + *MF, + CallingConv::AArch64_SME_ABI_Support_Routines_PreserveMost_From_X1)); + + restorePhyRegSave(RegSave, MBB, MBBI, DL); +} + +void MachineSMEABI::emitAllocateFullZASaveBuffer( + MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI, + LiveRegs PhysLiveRegs) { + // Buffer already allocated in SelectionDAG. + if (AFI->getEarlyAllocSMESaveBuffer()) + return; + + DebugLoc DL = getDebugLoc(MBB, MBBI); + Register BufferPtr = getAgnosticZABufferPtr(); + Register BufferSize = MRI->createVirtualRegister(&AArch64::GPR64RegClass); + + PhysRegSave RegSave = createPhysRegSave(PhysLiveRegs, MBB, MBBI, DL); + + // Calculate the SME state size. + { + auto *TLI = Subtarget->getTargetLowering(); + const AArch64RegisterInfo *TRI = Subtarget->getRegisterInfo(); + BuildMI(MBB, MBBI, DL, TII->get(AArch64::BL)) + .addExternalSymbol(TLI->getLibcallName(RTLIB::SMEABI_SME_STATE_SIZE)) + .addReg(AArch64::X0, RegState::ImplicitDefine) + .addRegMask(TRI->getCallPreservedMask( + *MF, CallingConv:: + AArch64_SME_ABI_Support_Routines_PreserveMost_From_X1)); + BuildMI(MBB, MBBI, DL, TII->get(TargetOpcode::COPY), BufferSize) + .addReg(AArch64::X0); + } + + // Allocate a buffer object of the size given __arm_sme_state_size. + { + MachineFrameInfo &MFI = MF->getFrameInfo(); + BuildMI(MBB, MBBI, DL, TII->get(AArch64::SUBXrx64), AArch64::SP) + .addReg(AArch64::SP) + .addReg(BufferSize) + .addImm(AArch64_AM::getArithExtendImm(AArch64_AM::UXTX, 0)); + BuildMI(MBB, MBBI, DL, TII->get(TargetOpcode::COPY), BufferPtr) + .addReg(AArch64::SP); + + // We have just allocated a variable sized object, tell this to PEI. + MFI.CreateVariableSizedObject(Align(16), nullptr); + } + + restorePhyRegSave(RegSave, MBB, MBBI, DL); +} + void MachineSMEABI::emitStateChange(MachineBasicBlock &MBB, MachineBasicBlock::iterator InsertPt, ZAState From, ZAState To, @@ -634,10 +759,7 @@ void MachineSMEABI::emitStateChange(MachineBasicBlock &MBB, // TODO: Avoid setting up the save buffer if there's no transition to // LOCAL_SAVED. if (From == ZAState::CALLER_DORMANT) { - assert(MBB.getParent() - ->getInfo<AArch64FunctionInfo>() - ->getSMEFnAttrs() - .hasPrivateZAInterface() && + assert(AFI->getSMEFnAttrs().hasPrivateZAInterface() && "CALLER_DORMANT state requires private ZA interface"); assert(&MBB == &MBB.getParent()->front() && "CALLER_DORMANT state only valid in entry block"); @@ -652,12 +774,14 @@ void MachineSMEABI::emitStateChange(MachineBasicBlock &MBB, } if (From == ZAState::ACTIVE && To == ZAState::LOCAL_SAVED) - emitSetupLazySave(MBB, InsertPt); + emitZASave(MBB, InsertPt, PhysLiveRegs); else if (From == ZAState::LOCAL_SAVED && To == ZAState::ACTIVE) - emitRestoreLazySave(MBB, InsertPt, PhysLiveRegs); + emitZARestore(MBB, InsertPt, PhysLiveRegs); else if (To == ZAState::OFF) { assert(From != ZAState::CALLER_DORMANT && "CALLER_DORMANT to OFF should have already been handled"); + assert(!AFI->getSMEFnAttrs().hasAgnosticZAInterface() && + "Should not turn ZA off in agnostic ZA function"); emitZAOff(MBB, InsertPt, /*ClearTPIDR2=*/From == ZAState::LOCAL_SAVED); } else { dbgs() << "Error: Transition from " << getZAStateString(From) << " to " @@ -675,9 +799,10 @@ bool MachineSMEABI::runOnMachineFunction(MachineFunction &MF) { if (!MF.getSubtarget<AArch64Subtarget>().hasSME()) return false; - auto *AFI = MF.getInfo<AArch64FunctionInfo>(); + AFI = MF.getInfo<AArch64FunctionInfo>(); SMEAttrs SMEFnAttrs = AFI->getSMEFnAttrs(); - if (!SMEFnAttrs.hasZAState() && !SMEFnAttrs.hasZT0State()) + if (!SMEFnAttrs.hasZAState() && !SMEFnAttrs.hasZT0State() && + !SMEFnAttrs.hasAgnosticZAInterface()) return false; assert(MF.getRegInfo().isSSA() && "Expected to be run on SSA form!"); @@ -696,15 +821,18 @@ bool MachineSMEABI::runOnMachineFunction(MachineFunction &MF) { insertStateChanges(); // Allocate save buffer (if needed). - if (State.TPIDR2Block) { + if (State.AgnosticZABufferPtr != AArch64::NoRegister || State.TPIDR2Block) { if (State.AfterSMEProloguePt) { // Note: With inline stack probes the AfterSMEProloguePt may not be in the // entry block (due to the probing loop). - emitAllocateLazySaveBuffer(*(*State.AfterSMEProloguePt)->getParent(), - *State.AfterSMEProloguePt); + emitAllocateZASaveBuffer(*(*State.AfterSMEProloguePt)->getParent(), + *State.AfterSMEProloguePt, + State.PhysLiveRegsAfterSMEPrologue); } else { MachineBasicBlock &EntryBlock = MF.front(); - emitAllocateLazySaveBuffer(EntryBlock, EntryBlock.getFirstNonPHI()); + emitAllocateZASaveBuffer( + EntryBlock, EntryBlock.getFirstNonPHI(), + State.Blocks[EntryBlock.getNumber()].PhysLiveRegsAtEntry); } } diff --git a/llvm/lib/Target/AArch64/SVEInstrFormats.td b/llvm/lib/Target/AArch64/SVEInstrFormats.td index 18deeef42852..166219de9dfe 100644 --- a/llvm/lib/Target/AArch64/SVEInstrFormats.td +++ b/llvm/lib/Target/AArch64/SVEInstrFormats.td @@ -893,13 +893,17 @@ class sve_int_ptest<bits<6> opc, string asm, SDPatternOperator op> } multiclass sve_int_ptest<bits<6> opc, string asm, SDPatternOperator op, - SDPatternOperator op_any> { + SDPatternOperator op_any, SDPatternOperator op_first> { def NAME : sve_int_ptest<opc, asm, op>; let hasNoSchedulingInfo = 1, isCompare = 1, Defs = [NZCV] in { def _ANY : Pseudo<(outs), (ins PPRAny:$Pg, PPR8:$Pn), [(set NZCV, (op_any (nxv16i1 PPRAny:$Pg), (nxv16i1 PPR8:$Pn)))]>, PseudoInstExpansion<(!cast<Instruction>(NAME) PPRAny:$Pg, PPR8:$Pn)>; + + def _FIRST : Pseudo<(outs), (ins PPRAny:$Pg, PPR8:$Pn), + [(set NZCV, (op_first (nxv16i1 PPRAny:$Pg), (nxv16i1 PPR8:$Pn)))]>, + PseudoInstExpansion<(!cast<Instruction>(NAME) PPRAny:$Pg, PPR8:$Pn)>; } } @@ -10401,7 +10405,7 @@ multiclass sve2p1_int_while_rr_pn<string mnemonic, bits<3> opc> { // SVE integer compare scalar count and limit (predicate pair) class sve2p1_int_while_rr_pair<string mnemonic, bits<2> sz, bits<3> opc, - RegisterOperand ppr_ty> + PPR2MulRegOp ppr_ty> : I<(outs ppr_ty:$Pd), (ins GPR64:$Rn, GPR64:$Rm), mnemonic, "\t$Pd, $Rn, $Rm", "", []>, Sched<[]> { @@ -10421,6 +10425,8 @@ class sve2p1_int_while_rr_pair<string mnemonic, bits<2> sz, bits<3> opc, let Defs = [NZCV]; let hasSideEffects = 0; + let ElementSize = ppr_ty.ElementSize; + let isWhile = 1; } diff --git a/llvm/lib/Target/AMDGPU/AMDGPU.td b/llvm/lib/Target/AMDGPU/AMDGPU.td index a366db1c580b..ffbda14dcd84 100644 --- a/llvm/lib/Target/AMDGPU/AMDGPU.td +++ b/llvm/lib/Target/AMDGPU/AMDGPU.td @@ -289,12 +289,6 @@ def FeatureSafeCUPrefetch : SubtargetFeature<"safe-cu-prefetch", "VMEM CU scope prefetches do not fail on illegal address" >; -def FeatureCUStores : SubtargetFeature<"cu-stores", - "HasCUStores", - "true", - "Whether SCOPE_CU stores can be used on GFX12.5" ->; - def FeatureVcmpxExecWARHazard : SubtargetFeature<"vcmpx-exec-war-hazard", "HasVcmpxExecWARHazard", "true", @@ -2042,7 +2036,6 @@ def FeatureISAVersion12_50 : FeatureSet< [FeatureGFX12, FeatureGFX1250Insts, FeatureRequiresAlignedVGPRs, - FeatureCUStores, FeatureAddressableLocalMemorySize327680, FeatureCuMode, Feature1024AddressableVGPRs, diff --git a/llvm/lib/Target/AMDGPU/AMDGPUAsmPrinter.cpp b/llvm/lib/Target/AMDGPU/AMDGPUAsmPrinter.cpp index 1fff188c7581..29f8f9bc8b54 100644 --- a/llvm/lib/Target/AMDGPU/AMDGPUAsmPrinter.cpp +++ b/llvm/lib/Target/AMDGPU/AMDGPUAsmPrinter.cpp @@ -557,7 +557,6 @@ const MCExpr *AMDGPUAsmPrinter::getAmdhsaKernelCodeProperties( MCContext &Ctx = MF.getContext(); uint16_t KernelCodeProperties = 0; const GCNUserSGPRUsageInfo &UserSGPRInfo = MFI.getUserSGPRInfo(); - const GCNSubtarget &ST = MF.getSubtarget<GCNSubtarget>(); if (UserSGPRInfo.hasPrivateSegmentBuffer()) { KernelCodeProperties |= @@ -587,13 +586,10 @@ const MCExpr *AMDGPUAsmPrinter::getAmdhsaKernelCodeProperties( KernelCodeProperties |= amdhsa::KERNEL_CODE_PROPERTY_ENABLE_SGPR_PRIVATE_SEGMENT_SIZE; } - if (ST.isWave32()) { + if (MF.getSubtarget<GCNSubtarget>().isWave32()) { KernelCodeProperties |= amdhsa::KERNEL_CODE_PROPERTY_ENABLE_WAVEFRONT_SIZE32; } - if (isGFX1250(ST) && ST.hasCUStores()) { - KernelCodeProperties |= amdhsa::KERNEL_CODE_PROPERTY_USES_CU_STORES; - } // CurrentProgramInfo.DynamicCallStack is a MCExpr and could be // un-evaluatable at this point so it cannot be conditionally checked here. diff --git a/llvm/lib/Target/AMDGPU/AMDGPUCombine.td b/llvm/lib/Target/AMDGPU/AMDGPUCombine.td index b5dac95b57a2..e8b211f7866a 100644 --- a/llvm/lib/Target/AMDGPU/AMDGPUCombine.td +++ b/llvm/lib/Target/AMDGPU/AMDGPUCombine.td @@ -71,6 +71,12 @@ def int_minmax_to_med3 : GICombineRule< [{ return matchIntMinMaxToMed3(*${min_or_max}, ${matchinfo}); }]), (apply [{ applyMed3(*${min_or_max}, ${matchinfo}); }])>; +let Predicates = [Predicate<"Subtarget->d16PreservesUnusedBits()">] in +def d16_load : GICombineRule< + (defs root:$bitcast), + (combine (G_BITCAST $dst, $src):$bitcast, + [{ return combineD16Load(*${bitcast} ); }])>; + def fp_minmax_to_med3 : GICombineRule< (defs root:$min_or_max, med3_matchdata:$matchinfo), (match (wip_match_opcode G_FMAXNUM, @@ -219,5 +225,6 @@ def AMDGPURegBankCombiner : GICombiner< zext_trunc_fold, int_minmax_to_med3, ptr_add_immed_chain, fp_minmax_to_clamp, fp_minmax_to_med3, fmed3_intrinsic_to_clamp, identity_combines, redundant_and, constant_fold_cast_op, - cast_of_cast_combines, sext_trunc, zext_of_shift_amount_combines]> { + cast_of_cast_combines, sext_trunc, zext_of_shift_amount_combines, + d16_load]> { } diff --git a/llvm/lib/Target/AMDGPU/AMDGPUGISel.td b/llvm/lib/Target/AMDGPU/AMDGPUGISel.td index 0c112d1787c1..bb4bf742fb86 100644 --- a/llvm/lib/Target/AMDGPU/AMDGPUGISel.td +++ b/llvm/lib/Target/AMDGPU/AMDGPUGISel.td @@ -315,6 +315,13 @@ def : GINodeEquiv<G_AMDGPU_S_BUFFER_LOAD_SSHORT, SIsbuffer_load_short>; def : GINodeEquiv<G_AMDGPU_S_BUFFER_LOAD_USHORT, SIsbuffer_load_ushort>; def : GINodeEquiv<G_AMDGPU_S_BUFFER_PREFETCH, SIsbuffer_prefetch>; +def : GINodeEquiv<G_AMDGPU_LOAD_D16_LO, SIload_d16_lo>; +def : GINodeEquiv<G_AMDGPU_LOAD_D16_LO_U8, SIload_d16_lo_u8>; +def : GINodeEquiv<G_AMDGPU_LOAD_D16_LO_I8, SIload_d16_lo_i8>; +def : GINodeEquiv<G_AMDGPU_LOAD_D16_HI, SIload_d16_hi>; +def : GINodeEquiv<G_AMDGPU_LOAD_D16_HI_U8, SIload_d16_hi_u8>; +def : GINodeEquiv<G_AMDGPU_LOAD_D16_HI_I8, SIload_d16_hi_i8>; + def : GINodeEquiv<G_AMDGPU_WHOLE_WAVE_FUNC_SETUP, AMDGPUwhole_wave_setup>; // G_AMDGPU_WHOLE_WAVE_FUNC_RETURN is simpler than AMDGPUwhole_wave_return, // so we don't mark it as equivalent. diff --git a/llvm/lib/Target/AMDGPU/AMDGPUISelDAGToDAG.cpp b/llvm/lib/Target/AMDGPU/AMDGPUISelDAGToDAG.cpp index 3785d0f7f268..c2fca79979e1 100644 --- a/llvm/lib/Target/AMDGPU/AMDGPUISelDAGToDAG.cpp +++ b/llvm/lib/Target/AMDGPU/AMDGPUISelDAGToDAG.cpp @@ -3302,29 +3302,52 @@ bool AMDGPUDAGToDAGISel::SelectVOP3ModsImpl(SDValue In, SDValue &Src, if (IsCanonicalizing) return true; - unsigned Opc = Src->getOpcode(); + // v2i32 xor/or/and are legal. A vselect using these instructions as operands + // is scalarised into two selects with EXTRACT_VECTOR_ELT operands. Peek + // through the extract to the bitwise op. + SDValue PeekSrc = + Src->getOpcode() == ISD::EXTRACT_VECTOR_ELT ? Src->getOperand(0) : Src; + // Convert various sign-bit masks to src mods. Currently disabled for 16-bit + // types as the codegen replaces the operand without adding a srcmod. + // This is intentionally finding the cases where we are performing float neg + // and abs on int types, the goal is not to obtain two's complement neg or + // abs. + // TODO: Add 16-bit support. + unsigned Opc = PeekSrc.getOpcode(); EVT VT = Src.getValueType(); if ((Opc != ISD::AND && Opc != ISD::OR && Opc != ISD::XOR) || - (VT != MVT::i32 && VT != MVT::i64)) + (VT != MVT::i32 && VT != MVT::v2i32 && VT != MVT::i64)) return true; - ConstantSDNode *CRHS = dyn_cast<ConstantSDNode>(Src->getOperand(1)); + ConstantSDNode *CRHS = isConstOrConstSplat(PeekSrc->getOperand(1)); if (!CRHS) return true; - // Recognise (xor a, 0x80000000) as NEG SrcMod. - // Recognise (and a, 0x7fffffff) as ABS SrcMod. - // Recognise (or a, 0x80000000) as NEG+ABS SrcModifiers. + auto ReplaceSrc = [&]() -> SDValue { + if (Src->getOpcode() != ISD::EXTRACT_VECTOR_ELT) + return Src.getOperand(0); + + SDValue LHS = PeekSrc->getOperand(0); + SDValue Index = Src->getOperand(1); + return CurDAG->getNode(ISD::EXTRACT_VECTOR_ELT, SDLoc(Src), + Src.getValueType(), LHS, Index); + }; + + // Recognise Srcmods: + // (xor a, 0x80000000) or v2i32 (xor a, {0x80000000,0x80000000}) as NEG. + // (and a, 0x7fffffff) or v2i32 (and a, {0x7fffffff,0x7fffffff}) as ABS. + // (or a, 0x80000000) or v2i32 (or a, {0x80000000,0x80000000}) as NEG+ABS + // SrcModifiers. if (Opc == ISD::XOR && CRHS->getAPIntValue().isSignMask()) { Mods |= SISrcMods::NEG; - Src = Src.getOperand(0); + Src = ReplaceSrc(); } else if (Opc == ISD::AND && AllowAbs && CRHS->getAPIntValue().isMaxSignedValue()) { Mods |= SISrcMods::ABS; - Src = Src.getOperand(0); + Src = ReplaceSrc(); } else if (Opc == ISD::OR && AllowAbs && CRHS->getAPIntValue().isSignMask()) { Mods |= SISrcMods::ABS | SISrcMods::NEG; - Src = Src.getOperand(0); + Src = ReplaceSrc(); } return true; diff --git a/llvm/lib/Target/AMDGPU/AMDGPUISelLowering.cpp b/llvm/lib/Target/AMDGPU/AMDGPUISelLowering.cpp index 5c9b616e9bc2..a44af5f854c1 100644 --- a/llvm/lib/Target/AMDGPU/AMDGPUISelLowering.cpp +++ b/llvm/lib/Target/AMDGPU/AMDGPUISelLowering.cpp @@ -4119,8 +4119,6 @@ SDValue AMDGPUTargetLowering::performShlCombine(SDNode *N, if (VT.getScalarType() != MVT::i64) return SDValue(); - // i64 (shl x, C) -> (build_pair 0, (shl x, C - 32)) - // On some subtargets, 64-bit shift is a quarter rate instruction. In the // common case, splitting this into a move and a 32-bit shift is faster and // the same code size. @@ -4210,12 +4208,12 @@ SDValue AMDGPUTargetLowering::performSraCombine(SDNode *N, (ElementType.getSizeInBits() - 1)) { ShiftAmt = ShiftFullAmt; } else { - SDValue truncShiftAmt = DAG.getNode(ISD::TRUNCATE, SL, TargetType, RHS); + SDValue TruncShiftAmt = DAG.getNode(ISD::TRUNCATE, SL, TargetType, RHS); const SDValue ShiftMask = DAG.getConstant(TargetScalarType.getSizeInBits() - 1, SL, TargetType); // This AND instruction will clamp out of bounds shift values. // It will also be removed during later instruction selection. - ShiftAmt = DAG.getNode(ISD::AND, SL, TargetType, truncShiftAmt, ShiftMask); + ShiftAmt = DAG.getNode(ISD::AND, SL, TargetType, TruncShiftAmt, ShiftMask); } EVT ConcatType; diff --git a/llvm/lib/Target/AMDGPU/AMDGPUMachineModuleInfo.cpp b/llvm/lib/Target/AMDGPU/AMDGPUMachineModuleInfo.cpp index eda479064d7b..d09b7cffe9f2 100644 --- a/llvm/lib/Target/AMDGPU/AMDGPUMachineModuleInfo.cpp +++ b/llvm/lib/Target/AMDGPU/AMDGPUMachineModuleInfo.cpp @@ -23,6 +23,7 @@ AMDGPUMachineModuleInfo::AMDGPUMachineModuleInfo(const MachineModuleInfo &MMI) AgentSSID = CTX.getOrInsertSyncScopeID("agent"); WorkgroupSSID = CTX.getOrInsertSyncScopeID("workgroup"); WavefrontSSID = CTX.getOrInsertSyncScopeID("wavefront"); + ClusterSSID = CTX.getOrInsertSyncScopeID("cluster"); SystemOneAddressSpaceSSID = CTX.getOrInsertSyncScopeID("one-as"); AgentOneAddressSpaceSSID = @@ -33,4 +34,5 @@ AMDGPUMachineModuleInfo::AMDGPUMachineModuleInfo(const MachineModuleInfo &MMI) CTX.getOrInsertSyncScopeID("wavefront-one-as"); SingleThreadOneAddressSpaceSSID = CTX.getOrInsertSyncScopeID("singlethread-one-as"); + ClusterOneAddressSpaceSSID = CTX.getOrInsertSyncScopeID("cluster-one-as"); } diff --git a/llvm/lib/Target/AMDGPU/AMDGPUMachineModuleInfo.h b/llvm/lib/Target/AMDGPU/AMDGPUMachineModuleInfo.h index fcb0c8cfb7ca..bf852bb38376 100644 --- a/llvm/lib/Target/AMDGPU/AMDGPUMachineModuleInfo.h +++ b/llvm/lib/Target/AMDGPU/AMDGPUMachineModuleInfo.h @@ -32,6 +32,8 @@ private: SyncScope::ID WorkgroupSSID; /// Wavefront synchronization scope ID (cross address space). SyncScope::ID WavefrontSSID; + /// Cluster synchronization scope ID (cross address space). + SyncScope::ID ClusterSSID; /// System synchronization scope ID (single address space). SyncScope::ID SystemOneAddressSpaceSSID; /// Agent synchronization scope ID (single address space). @@ -42,6 +44,8 @@ private: SyncScope::ID WavefrontOneAddressSpaceSSID; /// Single thread synchronization scope ID (single address space). SyncScope::ID SingleThreadOneAddressSpaceSSID; + /// Cluster synchronization scope ID (single address space). + SyncScope::ID ClusterOneAddressSpaceSSID; /// In AMDGPU target synchronization scopes are inclusive, meaning a /// larger synchronization scope is inclusive of a smaller synchronization @@ -60,12 +64,15 @@ private: else if (SSID == getWorkgroupSSID() || SSID == getWorkgroupOneAddressSpaceSSID()) return 2; + else if (SSID == getClusterSSID() || + SSID == getClusterOneAddressSpaceSSID()) + return 3; else if (SSID == getAgentSSID() || SSID == getAgentOneAddressSpaceSSID()) - return 3; + return 4; else if (SSID == SyncScope::System || SSID == getSystemOneAddressSpaceSSID()) - return 4; + return 5; return std::nullopt; } @@ -73,7 +80,8 @@ private: /// \returns True if \p SSID is restricted to single address space, false /// otherwise bool isOneAddressSpace(SyncScope::ID SSID) const { - return SSID == getSingleThreadOneAddressSpaceSSID() || + return SSID == getClusterOneAddressSpaceSSID() || + SSID == getSingleThreadOneAddressSpaceSSID() || SSID == getWavefrontOneAddressSpaceSSID() || SSID == getWorkgroupOneAddressSpaceSSID() || SSID == getAgentOneAddressSpaceSSID() || @@ -95,6 +103,8 @@ public: SyncScope::ID getWavefrontSSID() const { return WavefrontSSID; } + /// \returns Cluster synchronization scope ID (cross address space). + SyncScope::ID getClusterSSID() const { return ClusterSSID; } /// \returns System synchronization scope ID (single address space). SyncScope::ID getSystemOneAddressSpaceSSID() const { return SystemOneAddressSpaceSSID; @@ -115,6 +125,10 @@ public: SyncScope::ID getSingleThreadOneAddressSpaceSSID() const { return SingleThreadOneAddressSpaceSSID; } + /// \returns Single thread synchronization scope ID (single address space). + SyncScope::ID getClusterOneAddressSpaceSSID() const { + return ClusterOneAddressSpaceSSID; + } /// In AMDGPU target synchronization scopes are inclusive, meaning a /// larger synchronization scope is inclusive of a smaller synchronization diff --git a/llvm/lib/Target/AMDGPU/AMDGPUPromoteAlloca.cpp b/llvm/lib/Target/AMDGPU/AMDGPUPromoteAlloca.cpp index 8617d868ef8a..7dbe1235a98b 100644 --- a/llvm/lib/Target/AMDGPU/AMDGPUPromoteAlloca.cpp +++ b/llvm/lib/Target/AMDGPU/AMDGPUPromoteAlloca.cpp @@ -443,9 +443,10 @@ static Value *GEPToVectorIndex(GetElementPtrInst *GEP, AllocaInst *Alloca, return nullptr; APInt IndexQuot; - uint64_t Rem; - APInt::udivrem(ConstOffset, VecElemSize, IndexQuot, Rem); - if (Rem != 0) + APInt Rem; + APInt::sdivrem(ConstOffset, APInt(ConstOffset.getBitWidth(), VecElemSize), + IndexQuot, Rem); + if (!Rem.isZero()) return nullptr; if (VarOffsets.size() == 0) return ConstantInt::get(GEP->getContext(), IndexQuot); @@ -454,8 +455,10 @@ static Value *GEPToVectorIndex(GetElementPtrInst *GEP, AllocaInst *Alloca, const auto &VarOffset = VarOffsets.front(); APInt OffsetQuot; - APInt::udivrem(VarOffset.second, VecElemSize, OffsetQuot, Rem); - if (Rem != 0 || OffsetQuot.isZero()) + APInt::sdivrem(VarOffset.second, + APInt(VarOffset.second.getBitWidth(), VecElemSize), OffsetQuot, + Rem); + if (!Rem.isZero() || OffsetQuot.isZero()) return nullptr; Value *Offset = VarOffset.first; @@ -465,7 +468,7 @@ static Value *GEPToVectorIndex(GetElementPtrInst *GEP, AllocaInst *Alloca, if (!OffsetQuot.isOne()) { ConstantInt *ConstMul = - ConstantInt::get(OffsetType, OffsetQuot.getZExtValue()); + ConstantInt::get(OffsetType, OffsetQuot.getSExtValue()); Offset = Builder.CreateMul(Offset, ConstMul); if (Instruction *NewInst = dyn_cast<Instruction>(Offset)) NewInsts.push_back(NewInst); @@ -474,8 +477,8 @@ static Value *GEPToVectorIndex(GetElementPtrInst *GEP, AllocaInst *Alloca, return Offset; ConstantInt *ConstIndex = - ConstantInt::get(OffsetType, IndexQuot.getZExtValue()); - Value *IndexAdd = Builder.CreateAdd(ConstIndex, Offset); + ConstantInt::get(OffsetType, IndexQuot.getSExtValue()); + Value *IndexAdd = Builder.CreateAdd(Offset, ConstIndex); if (Instruction *NewInst = dyn_cast<Instruction>(IndexAdd)) NewInsts.push_back(NewInst); return IndexAdd; diff --git a/llvm/lib/Target/AMDGPU/AMDGPURegBankCombiner.cpp b/llvm/lib/Target/AMDGPU/AMDGPURegBankCombiner.cpp index ee324a5e93f0..fd604e1b19cd 100644 --- a/llvm/lib/Target/AMDGPU/AMDGPURegBankCombiner.cpp +++ b/llvm/lib/Target/AMDGPU/AMDGPURegBankCombiner.cpp @@ -89,6 +89,10 @@ public: void applyCanonicalizeZextShiftAmt(MachineInstr &MI, MachineInstr &Ext) const; + bool combineD16Load(MachineInstr &MI) const; + bool applyD16Load(unsigned D16Opc, MachineInstr &DstMI, + MachineInstr *SmallLoad, Register ToOverwriteD16) const; + private: SIModeRegisterDefaults getMode() const; bool getIEEE() const; @@ -392,6 +396,88 @@ void AMDGPURegBankCombinerImpl::applyCanonicalizeZextShiftAmt( MI.eraseFromParent(); } +bool AMDGPURegBankCombinerImpl::combineD16Load(MachineInstr &MI) const { + Register Dst; + MachineInstr *Load, *SextLoad; + const int64_t CleanLo16 = 0xFFFFFFFFFFFF0000; + const int64_t CleanHi16 = 0x000000000000FFFF; + + // Load lo + if (mi_match(MI.getOperand(1).getReg(), MRI, + m_GOr(m_GAnd(m_GBitcast(m_Reg(Dst)), + m_Copy(m_SpecificICst(CleanLo16))), + m_MInstr(Load)))) { + + if (Load->getOpcode() == AMDGPU::G_ZEXTLOAD) { + const MachineMemOperand *MMO = *Load->memoperands_begin(); + unsigned LoadSize = MMO->getSizeInBits().getValue(); + if (LoadSize == 8) + return applyD16Load(AMDGPU::G_AMDGPU_LOAD_D16_LO_U8, MI, Load, Dst); + if (LoadSize == 16) + return applyD16Load(AMDGPU::G_AMDGPU_LOAD_D16_LO, MI, Load, Dst); + return false; + } + + if (mi_match( + Load, MRI, + m_GAnd(m_MInstr(SextLoad), m_Copy(m_SpecificICst(CleanHi16))))) { + if (SextLoad->getOpcode() != AMDGPU::G_SEXTLOAD) + return false; + + const MachineMemOperand *MMO = *SextLoad->memoperands_begin(); + if (MMO->getSizeInBits().getValue() != 8) + return false; + + return applyD16Load(AMDGPU::G_AMDGPU_LOAD_D16_LO_I8, MI, SextLoad, Dst); + } + + return false; + } + + // Load hi + if (mi_match(MI.getOperand(1).getReg(), MRI, + m_GOr(m_GAnd(m_GBitcast(m_Reg(Dst)), + m_Copy(m_SpecificICst(CleanHi16))), + m_GShl(m_MInstr(Load), m_Copy(m_SpecificICst(16)))))) { + + if (Load->getOpcode() == AMDGPU::G_ZEXTLOAD) { + const MachineMemOperand *MMO = *Load->memoperands_begin(); + unsigned LoadSize = MMO->getSizeInBits().getValue(); + if (LoadSize == 8) + return applyD16Load(AMDGPU::G_AMDGPU_LOAD_D16_HI_U8, MI, Load, Dst); + if (LoadSize == 16) + return applyD16Load(AMDGPU::G_AMDGPU_LOAD_D16_HI, MI, Load, Dst); + return false; + } + + if (mi_match( + Load, MRI, + m_GAnd(m_MInstr(SextLoad), m_Copy(m_SpecificICst(CleanHi16))))) { + if (SextLoad->getOpcode() != AMDGPU::G_SEXTLOAD) + return false; + const MachineMemOperand *MMO = *SextLoad->memoperands_begin(); + if (MMO->getSizeInBits().getValue() != 8) + return false; + + return applyD16Load(AMDGPU::G_AMDGPU_LOAD_D16_HI_I8, MI, SextLoad, Dst); + } + + return false; + } + + return false; +} + +bool AMDGPURegBankCombinerImpl::applyD16Load( + unsigned D16Opc, MachineInstr &DstMI, MachineInstr *SmallLoad, + Register SrcReg32ToOverwriteD16) const { + B.buildInstr(D16Opc, {DstMI.getOperand(0).getReg()}, + {SmallLoad->getOperand(1).getReg(), SrcReg32ToOverwriteD16}) + .setMemRefs(SmallLoad->memoperands()); + DstMI.eraseFromParent(); + return true; +} + SIModeRegisterDefaults AMDGPURegBankCombinerImpl::getMode() const { return MF.getInfo<SIMachineFunctionInfo>()->getMode(); } diff --git a/llvm/lib/Target/AMDGPU/AMDGPURegBankLegalizeHelper.cpp b/llvm/lib/Target/AMDGPU/AMDGPURegBankLegalizeHelper.cpp index b45627d9c1c5..73b266072734 100644 --- a/llvm/lib/Target/AMDGPU/AMDGPURegBankLegalizeHelper.cpp +++ b/llvm/lib/Target/AMDGPU/AMDGPURegBankLegalizeHelper.cpp @@ -352,6 +352,32 @@ void RegBankLegalizeHelper::widenLoad(MachineInstr &MI, LLT WideTy, MI.eraseFromParent(); } +void RegBankLegalizeHelper::widenMMOToS32(GAnyLoad &MI) const { + Register Dst = MI.getDstReg(); + Register Ptr = MI.getPointerReg(); + MachineMemOperand &MMO = MI.getMMO(); + unsigned MemSize = 8 * MMO.getSize().getValue(); + + MachineMemOperand *WideMMO = B.getMF().getMachineMemOperand(&MMO, 0, S32); + + if (MI.getOpcode() == G_LOAD) { + B.buildLoad(Dst, Ptr, *WideMMO); + } else { + auto Load = B.buildLoad(SgprRB_S32, Ptr, *WideMMO); + + if (MI.getOpcode() == G_ZEXTLOAD) { + APInt Mask = APInt::getLowBitsSet(S32.getSizeInBits(), MemSize); + auto MaskCst = B.buildConstant(SgprRB_S32, Mask); + B.buildAnd(Dst, Load, MaskCst); + } else { + assert(MI.getOpcode() == G_SEXTLOAD); + B.buildSExtInReg(Dst, Load, MemSize); + } + } + + MI.eraseFromParent(); +} + void RegBankLegalizeHelper::lowerVccExtToSel(MachineInstr &MI) { Register Dst = MI.getOperand(0).getReg(); LLT Ty = MRI.getType(Dst); @@ -744,6 +770,8 @@ void RegBankLegalizeHelper::lower(MachineInstr &MI, } break; } + case WidenMMOToS32: + return widenMMOToS32(cast<GAnyLoad>(MI)); } if (!WaterfallSgprs.empty()) { @@ -759,6 +787,7 @@ LLT RegBankLegalizeHelper::getTyFromID(RegBankLLTMappingApplyID ID) { return LLT::scalar(1); case Sgpr16: case Vgpr16: + case UniInVgprS16: return LLT::scalar(16); case Sgpr32: case Sgpr32_WF: @@ -895,6 +924,7 @@ RegBankLegalizeHelper::getRegBankFromID(RegBankLLTMappingApplyID ID) { case SgprB256: case SgprB512: case UniInVcc: + case UniInVgprS16: case UniInVgprS32: case UniInVgprV2S16: case UniInVgprV4S32: @@ -1015,6 +1045,18 @@ void RegBankLegalizeHelper::applyMappingDst( B.buildTrunc(Reg, CopyS32_Vcc); break; } + case UniInVgprS16: { + assert(Ty == getTyFromID(MethodIDs[OpIdx])); + assert(RB == SgprRB); + Register NewVgprDstS16 = MRI.createVirtualRegister({VgprRB, S16}); + Register NewVgprDstS32 = MRI.createVirtualRegister({VgprRB, S32}); + Register NewSgprDstS32 = MRI.createVirtualRegister({SgprRB, S32}); + Op.setReg(NewVgprDstS16); + B.buildAnyExt(NewVgprDstS32, NewVgprDstS16); + buildReadAnyLane(B, NewSgprDstS32, NewVgprDstS32, RBI); + B.buildTrunc(Reg, NewSgprDstS32); + break; + } case UniInVgprS32: case UniInVgprV2S16: case UniInVgprV4S32: { diff --git a/llvm/lib/Target/AMDGPU/AMDGPURegBankLegalizeHelper.h b/llvm/lib/Target/AMDGPU/AMDGPURegBankLegalizeHelper.h index db965d8c000d..7affe5ab3da7 100644 --- a/llvm/lib/Target/AMDGPU/AMDGPURegBankLegalizeHelper.h +++ b/llvm/lib/Target/AMDGPU/AMDGPURegBankLegalizeHelper.h @@ -11,6 +11,7 @@ #include "AMDGPURegBankLegalizeRules.h" #include "llvm/ADT/SmallSet.h" +#include "llvm/CodeGen/GlobalISel/GenericMachineInstrs.h" #include "llvm/CodeGen/MachineRegisterInfo.h" namespace llvm { @@ -107,6 +108,7 @@ private: void splitLoad(MachineInstr &MI, ArrayRef<LLT> LLTBreakdown, LLT MergeTy = LLT()); void widenLoad(MachineInstr &MI, LLT WideTy, LLT MergeTy = LLT()); + void widenMMOToS32(GAnyLoad &MI) const; void lower(MachineInstr &MI, const RegBankLLTMapping &Mapping, SmallSet<Register, 4> &SgprWaterfallOperandRegs); diff --git a/llvm/lib/Target/AMDGPU/AMDGPURegBankLegalizeRules.cpp b/llvm/lib/Target/AMDGPU/AMDGPURegBankLegalizeRules.cpp index 8c56c2162112..0776d14a8406 100644 --- a/llvm/lib/Target/AMDGPU/AMDGPURegBankLegalizeRules.cpp +++ b/llvm/lib/Target/AMDGPU/AMDGPURegBankLegalizeRules.cpp @@ -467,6 +467,8 @@ RegBankLegalizeRules::RegBankLegalizeRules(const GCNSubtarget &_ST, : ST(&_ST), MRI(&_MRI) { addRulesForGOpcs({G_ADD, G_SUB}, Standard) + .Uni(S16, {{Sgpr32Trunc}, {Sgpr32AExt, Sgpr32AExt}}) + .Div(S16, {{Vgpr16}, {Vgpr16, Vgpr16}}) .Uni(S32, {{Sgpr32}, {Sgpr32, Sgpr32}}) .Div(S32, {{Vgpr32}, {Vgpr32, Vgpr32}}); @@ -615,8 +617,9 @@ RegBankLegalizeRules::RegBankLegalizeRules(const GCNSubtarget &_ST, .Any({{UniS64, S64}, {{Sgpr64}, {Sgpr64}}}) .Any({{DivS64, S64}, {{Vgpr64}, {Vgpr64}, SplitTo32SExtInReg}}); - bool hasUnalignedLoads = ST->getGeneration() >= AMDGPUSubtarget::GFX12; + bool hasSMRDx3 = ST->hasScalarDwordx3Loads(); bool hasSMRDSmall = ST->hasScalarSubwordLoads(); + bool usesTrue16 = ST->useRealTrue16Insts(); Predicate isAlign16([](const MachineInstr &MI) -> bool { return (*MI.memoperands_begin())->getAlign() >= Align(16); @@ -654,54 +657,187 @@ RegBankLegalizeRules::RegBankLegalizeRules(const GCNSubtarget &_ST, return (*MI.memoperands_begin())->getFlags() & MONoClobber; }); - Predicate isNaturalAlignedSmall([](const MachineInstr &MI) -> bool { + Predicate isNaturalAligned([](const MachineInstr &MI) -> bool { + const MachineMemOperand *MMO = *MI.memoperands_begin(); + return MMO->getAlign() >= Align(MMO->getSize().getValue()); + }); + + Predicate is8Or16BitMMO([](const MachineInstr &MI) -> bool { const MachineMemOperand *MMO = *MI.memoperands_begin(); const unsigned MemSize = 8 * MMO->getSize().getValue(); - return (MemSize == 16 && MMO->getAlign() >= Align(2)) || - (MemSize == 8 && MMO->getAlign() >= Align(1)); + return MemSize == 16 || MemSize == 8; + }); + + Predicate is32BitMMO([](const MachineInstr &MI) -> bool { + const MachineMemOperand *MMO = *MI.memoperands_begin(); + return 8 * MMO->getSize().getValue() == 32; }); auto isUL = !isAtomicMMO && isUniMMO && (isConst || !isVolatileMMO) && (isConst || isInvMMO || isNoClobberMMO); // clang-format off + // TODO: S32Dst, 16-bit any-extending load should not appear on True16 targets addRulesForGOpcs({G_LOAD}) - .Any({{DivB32, DivP0}, {{VgprB32}, {VgprP0}}}) - .Any({{DivB32, UniP0}, {{VgprB32}, {VgprP0}}}) - - .Any({{DivB32, DivP1}, {{VgprB32}, {VgprP1}}}) - .Any({{{UniB256, UniP1}, isAlign4 && isUL}, {{SgprB256}, {SgprP1}}}) - .Any({{{UniB512, UniP1}, isAlign4 && isUL}, {{SgprB512}, {SgprP1}}}) - .Any({{{UniB32, UniP1}, !isAlign4 || !isUL}, {{UniInVgprB32}, {SgprP1}}}) - .Any({{{UniB64, UniP1}, !isAlign4 || !isUL}, {{UniInVgprB64}, {SgprP1}}}) - .Any({{{UniB96, UniP1}, !isAlign4 || !isUL}, {{UniInVgprB96}, {SgprP1}}}) - .Any({{{UniB128, UniP1}, !isAlign4 || !isUL}, {{UniInVgprB128}, {SgprP1}}}) - .Any({{{UniB256, UniP1}, !isAlign4 || !isUL}, {{UniInVgprB256}, {VgprP1}, SplitLoad}}) - .Any({{{UniB512, UniP1}, !isAlign4 || !isUL}, {{UniInVgprB512}, {VgprP1}, SplitLoad}}) - - .Any({{DivB32, UniP3}, {{VgprB32}, {VgprP3}}}) - .Any({{{UniB32, UniP3}, isAlign4 && isUL}, {{SgprB32}, {SgprP3}}}) - .Any({{{UniB32, UniP3}, !isAlign4 || !isUL}, {{UniInVgprB32}, {VgprP3}}}) - - .Any({{{DivB256, DivP4}}, {{VgprB256}, {VgprP4}, SplitLoad}}) - .Any({{{UniB32, UniP4}, isNaturalAlignedSmall && isUL}, {{SgprB32}, {SgprP4}}}, hasSMRDSmall) // i8 and i16 load - .Any({{{UniB32, UniP4}, isAlign4 && isUL}, {{SgprB32}, {SgprP4}}}) - .Any({{{UniB96, UniP4}, isAlign16 && isUL}, {{SgprB96}, {SgprP4}, WidenLoad}}, !hasUnalignedLoads) - .Any({{{UniB96, UniP4}, isAlign4 && !isAlign16 && isUL}, {{SgprB96}, {SgprP4}, SplitLoad}}, !hasUnalignedLoads) - .Any({{{UniB96, UniP4}, isAlign4 && isUL}, {{SgprB96}, {SgprP4}}}, hasUnalignedLoads) - .Any({{{UniB128, UniP4}, isAlign4 && isUL}, {{SgprB128}, {SgprP4}}}) - .Any({{{UniB256, UniP4}, isAlign4 && isUL}, {{SgprB256}, {SgprP4}}}) - .Any({{{UniB512, UniP4}, isAlign4 && isUL}, {{SgprB512}, {SgprP4}}}) - .Any({{{UniB32, UniP4}, !isNaturalAlignedSmall || !isUL}, {{UniInVgprB32}, {VgprP4}}}, hasSMRDSmall) // i8 and i16 load - .Any({{{UniB32, UniP4}, !isAlign4 || !isUL}, {{UniInVgprB32}, {VgprP4}}}) - .Any({{{UniB256, UniP4}, !isAlign4 || !isUL}, {{UniInVgprB256}, {VgprP4}, SplitLoad}}) - .Any({{{UniB512, UniP4}, !isAlign4 || !isUL}, {{UniInVgprB512}, {VgprP4}, SplitLoad}}) - - .Any({{DivB32, P5}, {{VgprB32}, {VgprP5}}}); - - addRulesForGOpcs({G_ZEXTLOAD}) // i8 and i16 zero-extending loads - .Any({{{UniB32, UniP3}, !isAlign4 || !isUL}, {{UniInVgprB32}, {VgprP3}}}) - .Any({{{UniB32, UniP4}, !isAlign4 || !isUL}, {{UniInVgprB32}, {VgprP4}}}); + // flat, addrspace(0), never uniform - flat_load + .Any({{DivS16, P0}, {{Vgpr16}, {VgprP0}}}, usesTrue16) + .Any({{DivB32, P0}, {{VgprB32}, {VgprP0}}}) // 32-bit load, 8-bit and 16-bit any-extending load + .Any({{DivB64, P0}, {{VgprB64}, {VgprP0}}}) + .Any({{DivB96, P0}, {{VgprB96}, {VgprP0}}}) + .Any({{DivB128, P0}, {{VgprB128}, {VgprP0}}}) + + // global, addrspace(1) + // divergent - global_load + .Any({{DivS16, P1}, {{Vgpr16}, {VgprP1}}}, usesTrue16) + .Any({{DivB32, P1}, {{VgprB32}, {VgprP1}}}) //32-bit load, 8-bit and 16-bit any-extending load + .Any({{DivB64, P1}, {{VgprB64}, {VgprP1}}}) + .Any({{DivB96, P1}, {{VgprB96}, {VgprP1}}}) + .Any({{DivB128, P1}, {{VgprB128}, {VgprP1}}}) + .Any({{DivB256, P1}, {{VgprB256}, {VgprP1}, SplitLoad}}) + .Any({{DivB512, P1}, {{VgprB512}, {VgprP1}, SplitLoad}}) + + // uniform - s_load + .Any({{{UniS16, P1}, isNaturalAligned && isUL}, {{Sgpr32Trunc}, {SgprP1}}}, usesTrue16 && hasSMRDSmall) // s16 load + .Any({{{UniS16, P1}, isAlign4 && isUL}, {{Sgpr32Trunc}, {SgprP1}, WidenMMOToS32}}, usesTrue16 && !hasSMRDSmall) // s16 load to 32-bit load + .Any({{{UniB32, P1}, isNaturalAligned && isUL}, {{SgprB32}, {SgprP1}}}, hasSMRDSmall) //32-bit load, 8-bit and 16-bit any-extending load + // TODO: SplitLoad when !isNaturalAligned && isUL and target hasSMRDSmall + .Any({{{UniB32, P1}, is8Or16BitMMO && isAlign4 && isUL}, {{SgprB32}, {SgprP1}, WidenMMOToS32}}, !hasSMRDSmall) //8-bit and 16-bit any-extending load to 32-bit load + .Any({{{UniB32, P1}, is32BitMMO && isAlign4 && isUL}, {{SgprB32}, {SgprP1}}}) //32-bit load + .Any({{{UniB64, P1}, isAlign4 && isUL}, {{SgprB64}, {SgprP1}}}) + .Any({{{UniB96, P1}, isAlign16 && isUL}, {{SgprB96}, {SgprP1}, WidenLoad}}, !hasSMRDx3) + .Any({{{UniB96, P1}, isAlign4 && !isAlign16 && isUL}, {{SgprB96}, {SgprP1}, SplitLoad}}, !hasSMRDx3) + .Any({{{UniB96, P1}, isAlign4 && isUL}, {{SgprB96}, {SgprP1}}}, hasSMRDx3) + .Any({{{UniB128, P1}, isAlign4 && isUL}, {{SgprB128}, {SgprP1}}}) + .Any({{{UniB256, P1}, isAlign4 && isUL}, {{SgprB256}, {SgprP1}}}) + .Any({{{UniB512, P1}, isAlign4 && isUL}, {{SgprB512}, {SgprP1}}}) + + // Uniform via global or buffer load, for example volatile or non-aligned + // uniform load. Not using standard {{UniInVgprTy}, {VgprP1}} since it is + // selected as global_load, use SgprP1 for pointer instead to match + // patterns without flat-for-global, default for GFX7 and older. + // -> +flat-for-global + {{UniInVgprTy}, {SgprP1}} - global_load + // -> -flat-for-global + {{UniInVgprTy}, {SgprP1}} - buffer_load + .Any({{{UniS16, P1}, !isNaturalAligned || !isUL}, {{UniInVgprS16}, {SgprP1}}}, usesTrue16 && hasSMRDSmall) // s16 load + .Any({{{UniS16, P1}, !isAlign4 || !isUL}, {{UniInVgprS16}, {SgprP1}}}, usesTrue16 && !hasSMRDSmall) // s16 load + .Any({{{UniB32, P1}, !isNaturalAligned || !isUL}, {{UniInVgprB32}, {SgprP1}}}, hasSMRDSmall) //32-bit load, 8-bit and 16-bit any-extending load + .Any({{{UniB32, P1}, !isAlign4 || !isUL}, {{UniInVgprB32}, {SgprP1}}}, !hasSMRDSmall) //32-bit load, 8-bit and 16-bit any-extending load + .Any({{{UniB64, P1}, !isAlign4 || !isUL}, {{UniInVgprB64}, {SgprP1}}}) + .Any({{{UniB96, P1}, !isAlign4 || !isUL}, {{UniInVgprB96}, {SgprP1}}}) + .Any({{{UniB128, P1}, !isAlign4 || !isUL}, {{UniInVgprB128}, {SgprP1}}}) + .Any({{{UniB256, P1}, !isAlign4 || !isUL}, {{UniInVgprB256}, {SgprP1}, SplitLoad}}) + .Any({{{UniB512, P1}, !isAlign4 || !isUL}, {{UniInVgprB512}, {SgprP1}, SplitLoad}}) + + // local, addrspace(3) - ds_load + .Any({{DivS16, P3}, {{Vgpr16}, {VgprP3}}}, usesTrue16) + .Any({{DivB32, P3}, {{VgprB32}, {VgprP3}}}) // 32-bit load, 8-bit and 16-bit any-extending load + .Any({{DivB64, P3}, {{VgprB64}, {VgprP3}}}) + .Any({{DivB96, P3}, {{VgprB96}, {VgprP3}}}) + .Any({{DivB128, P3}, {{VgprB128}, {VgprP3}}}) + + .Any({{UniS16, P3}, {{UniInVgprS16}, {SgprP3}}}, usesTrue16) // 16-bit load + .Any({{UniB32, P3}, {{UniInVgprB32}, {VgprP3}}}) // 32-bit load, 8-bit and 16-bit any-extending load + .Any({{UniB64, P3}, {{UniInVgprB64}, {VgprP3}}}) + .Any({{UniB96, P3}, {{UniInVgprB96}, {VgprP3}}}) + .Any({{UniB128, P3}, {{UniInVgprB128}, {VgprP3}}}) + + // constant, addrspace(4) + // divergent - global_load + .Any({{DivS16, P4}, {{Vgpr16}, {VgprP4}}}, usesTrue16) + .Any({{DivB32, P4}, {{VgprB32}, {VgprP4}}}) //32-bit load, 8-bit and 16-bit any-extending load + .Any({{DivB64, P4}, {{VgprB64}, {VgprP4}}}) + .Any({{DivB96, P4}, {{VgprB96}, {VgprP4}}}) + .Any({{DivB128, P4}, {{VgprB128}, {VgprP4}}}) + .Any({{DivB256, P4}, {{VgprB256}, {VgprP4}, SplitLoad}}) + .Any({{DivB512, P4}, {{VgprB512}, {VgprP4}, SplitLoad}}) + + // uniform - s_load + .Any({{{UniS16, P4}, isNaturalAligned && isUL}, {{Sgpr32Trunc}, {SgprP4}}}, usesTrue16 && hasSMRDSmall) // s16 load + .Any({{{UniS16, P4}, isAlign4 && isUL}, {{Sgpr32Trunc}, {SgprP4}, WidenMMOToS32}}, usesTrue16 && !hasSMRDSmall) // s16 load to 32-bit load + .Any({{{UniB32, P4}, isNaturalAligned && isUL}, {{SgprB32}, {SgprP4}}}, hasSMRDSmall) //32-bit load, 8-bit and 16-bit any-extending load + .Any({{{UniB32, P4}, is8Or16BitMMO && isAlign4 && isUL}, {{SgprB32}, {SgprP4}, WidenMMOToS32}}, !hasSMRDSmall) //8-bit and 16-bit any-extending load to 32-bit load + .Any({{{UniB32, P4}, is32BitMMO && isAlign4 && isUL}, {{SgprB32}, {SgprP4}}}) //32-bit load + .Any({{{UniB64, P4}, isAlign4 && isUL}, {{SgprB64}, {SgprP4}}}) + .Any({{{UniB96, P4}, isAlign16 && isUL}, {{SgprB96}, {SgprP4}, WidenLoad}}, !hasSMRDx3) + .Any({{{UniB96, P4}, isAlign4 && !isAlign16 && isUL}, {{SgprB96}, {SgprP4}, SplitLoad}}, !hasSMRDx3) + .Any({{{UniB96, P4}, isAlign4 && isUL}, {{SgprB96}, {SgprP4}}}, hasSMRDx3) + .Any({{{UniB128, P4}, isAlign4 && isUL}, {{SgprB128}, {SgprP4}}}) + .Any({{{UniB256, P4}, isAlign4 && isUL}, {{SgprB256}, {SgprP4}}}) + .Any({{{UniB512, P4}, isAlign4 && isUL}, {{SgprB512}, {SgprP4}}}) + + // uniform in vgpr - global_load or buffer_load + .Any({{{UniS16, P4}, !isNaturalAligned || !isUL}, {{UniInVgprS16}, {SgprP4}}}, usesTrue16 && hasSMRDSmall) // s16 load + .Any({{{UniS16, P4}, !isAlign4 || !isUL}, {{UniInVgprS16}, {SgprP4}}}, usesTrue16 && !hasSMRDSmall) // s16 load + .Any({{{UniB32, P4}, !isNaturalAligned || !isUL}, {{UniInVgprB32}, {SgprP4}}}, hasSMRDSmall) //32-bit load, 8-bit and 16-bit any-extending load + .Any({{{UniB32, P4}, !isAlign4 || !isUL}, {{UniInVgprB32}, {SgprP4}}}, !hasSMRDSmall) //32-bit load, 8-bit and 16-bit any-extending load + .Any({{{UniB64, P4}, !isAlign4 || !isUL}, {{UniInVgprB64}, {SgprP4}}}) + .Any({{{UniB96, P4}, !isAlign4 || !isUL}, {{UniInVgprB96}, {SgprP4}}}) + .Any({{{UniB128, P4}, !isAlign4 || !isUL}, {{UniInVgprB128}, {SgprP4}}}) + .Any({{{UniB256, P4}, !isAlign4 || !isUL}, {{UniInVgprB256}, {SgprP4}, SplitLoad}}) + .Any({{{UniB512, P4}, !isAlign4 || !isUL}, {{UniInVgprB512}, {SgprP4}, SplitLoad}}) + + // private, addrspace(5), never uniform - scratch_load + .Any({{DivS16, P5}, {{Vgpr16}, {VgprP5}}}, usesTrue16) + .Any({{DivB32, P5}, {{VgprB32}, {VgprP5}}}) // 32-bit load, 8-bit and 16-bit any-extending load + .Any({{DivB64, P5}, {{VgprB64}, {VgprP5}}}) + .Any({{DivB96, P5}, {{VgprB96}, {VgprP5}}}) + .Any({{DivB128, P5}, {{VgprB128}, {VgprP5}}}) + + .Any({{DivS32, Ptr128}, {{Vgpr32}, {VgprPtr128}}}); + + + addRulesForGOpcs({G_ZEXTLOAD, G_SEXTLOAD}) // i8 and i16 zeroextending loads + .Any({{DivS32, P0}, {{Vgpr32}, {VgprP0}}}) + + .Any({{DivS32, P1}, {{Vgpr32}, {VgprP1}}}) + .Any({{{UniS32, P1}, isAlign4 && isUL}, {{Sgpr32}, {SgprP1}, WidenMMOToS32}}, !hasSMRDSmall) + .Any({{{UniS32, P1}, isNaturalAligned && isUL}, {{Sgpr32}, {SgprP1}}}, hasSMRDSmall) + .Any({{{UniS32, P1}, !isAlign4 || !isUL}, {{UniInVgprS32}, {SgprP1}}}, !hasSMRDSmall) + .Any({{{UniS32, P1}, !isNaturalAligned || !isUL}, {{UniInVgprS32}, {SgprP1}}}, hasSMRDSmall) + + .Any({{DivS32, P3}, {{Vgpr32}, {VgprP3}}}) + .Any({{UniS32, P3}, {{UniInVgprS32}, {VgprP3}}}) + + .Any({{DivS32, P4}, {{Vgpr32}, {VgprP4}}}) + .Any({{{UniS32, P4}, isAlign4 && isUL}, {{Sgpr32}, {SgprP4}, WidenMMOToS32}}, !hasSMRDSmall) + .Any({{{UniS32, P4}, isNaturalAligned && isUL}, {{Sgpr32}, {SgprP4}}}, hasSMRDSmall) + .Any({{{UniS32, P4}, !isAlign4 || !isUL}, {{UniInVgprS32}, {SgprP4}}}, !hasSMRDSmall) + .Any({{{UniS32, P4}, !isNaturalAligned || !isUL}, {{UniInVgprS32}, {SgprP4}}}, hasSMRDSmall) + + .Any({{DivS32, P5}, {{Vgpr32}, {VgprP5}}}); + + addRulesForGOpcs({G_STORE}) + // addrspace(0) + .Any({{S16, P0}, {{}, {Vgpr16, VgprP0}}}, usesTrue16) // 16-bit store + .Any({{B32, P0}, {{}, {VgprB32, VgprP0}}}) // 32-bit store, 8-bit and 16-bit truncating store + .Any({{B64, P0}, {{}, {VgprB64, VgprP0}}}) + .Any({{B96, P0}, {{}, {VgprB96, VgprP0}}}) + .Any({{B128, P0}, {{}, {VgprB128, VgprP0}}}) + + // addrspace(1), there are no stores to addrspace(4) + // For targets: + // - with "+flat-for-global" - global_store + // - without(-flat-for-global) - buffer_store addr64 + .Any({{S16, DivP1}, {{}, {Vgpr16, VgprP1}}}, usesTrue16) // 16-bit store + .Any({{B32, DivP1}, {{}, {VgprB32, VgprP1}}}) // 32-bit store, 8-bit and 16-bit truncating store + .Any({{B64, DivP1}, {{}, {VgprB64, VgprP1}}}) + .Any({{B96, DivP1}, {{}, {VgprB96, VgprP1}}}) + .Any({{B128, DivP1}, {{}, {VgprB128, VgprP1}}}) + + // For UniP1, use sgpr ptr to match flat-for-global patterns. Targets: + // - with "+flat-for-global" - global_store for both sgpr and vgpr ptr + // - without(-flat-for-global) - need sgpr ptr to select buffer_store + .Any({{S16, UniP1}, {{}, {Vgpr16, SgprP1}}}, usesTrue16) // 16-bit store + .Any({{B32, UniP1}, {{}, {VgprB32, SgprP1}}}) // 32-bit store, 8-bit and 16-bit truncating store + .Any({{B64, UniP1}, {{}, {VgprB64, SgprP1}}}) + .Any({{B96, UniP1}, {{}, {VgprB96, SgprP1}}}) + .Any({{B128, UniP1}, {{}, {VgprB128, SgprP1}}}) + + // addrspace(3) and addrspace(5) + .Any({{S16, Ptr32}, {{}, {Vgpr16, VgprPtr32}}}, usesTrue16) // 16-bit store + .Any({{B32, Ptr32}, {{}, {VgprB32, VgprPtr32}}}) // 32-bit store, 8-bit and 16-bit truncating store + .Any({{B64, Ptr32}, {{}, {VgprB64, VgprPtr32}}}) + .Any({{B96, Ptr32}, {{}, {VgprB96, VgprPtr32}}}) + .Any({{B128, Ptr32}, {{}, {VgprB128, VgprPtr32}}}); // clang-format on addRulesForGOpcs({G_AMDGPU_BUFFER_LOAD}, StandardB) @@ -714,12 +850,6 @@ RegBankLegalizeRules::RegBankLegalizeRules(const GCNSubtarget &_ST, .Div(B128, {{VgprB128}, {SgprV4S32_WF, Vgpr32, Vgpr32, Sgpr32_WF}}) .Uni(B128, {{UniInVgprB128}, {SgprV4S32_WF, Vgpr32, Vgpr32, Sgpr32_WF}}); - addRulesForGOpcs({G_STORE}) - .Any({{S32, P0}, {{}, {Vgpr32, VgprP0}}}) - .Any({{S32, P1}, {{}, {Vgpr32, VgprP1}}}) - .Any({{S64, P1}, {{}, {Vgpr64, VgprP1}}}) - .Any({{V4S32, P1}, {{}, {VgprV4S32, VgprP1}}}); - addRulesForGOpcs({G_AMDGPU_BUFFER_STORE}) .Any({{S32}, {{}, {Vgpr32, SgprV4S32, Vgpr32, Vgpr32, Sgpr32}}}); diff --git a/llvm/lib/Target/AMDGPU/AMDGPURegBankLegalizeRules.h b/llvm/lib/Target/AMDGPU/AMDGPURegBankLegalizeRules.h index 13914403c439..d0c69105356b 100644 --- a/llvm/lib/Target/AMDGPU/AMDGPURegBankLegalizeRules.h +++ b/llvm/lib/Target/AMDGPU/AMDGPURegBankLegalizeRules.h @@ -176,6 +176,7 @@ enum RegBankLLTMappingApplyID { // Dst only modifiers: read-any-lane and truncs UniInVcc, + UniInVgprS16, UniInVgprS32, UniInVgprV2S16, UniInVgprV4S32, @@ -221,6 +222,7 @@ enum LoweringMethodID { UniCstExt, SplitLoad, WidenLoad, + WidenMMOToS32 }; enum FastRulesTypes { diff --git a/llvm/lib/Target/AMDGPU/AMDGPUTargetMachine.cpp b/llvm/lib/Target/AMDGPU/AMDGPUTargetMachine.cpp index 9afe7590fe4e..92a587b5771b 100644 --- a/llvm/lib/Target/AMDGPU/AMDGPUTargetMachine.cpp +++ b/llvm/lib/Target/AMDGPU/AMDGPUTargetMachine.cpp @@ -720,25 +720,6 @@ static MachineSchedRegistry GCNILPSchedRegistry( "Run GCN iterative scheduler for ILP scheduling (experimental)", createIterativeILPMachineScheduler); -static StringRef computeDataLayout(const Triple &TT) { - if (TT.getArch() == Triple::r600) { - // 32-bit pointers. - return "e-p:32:32-i64:64-v16:16-v24:32-v32:32-v48:64-v96:128" - "-v192:256-v256:256-v512:512-v1024:1024-v2048:2048-n32:64-S32-A5-G1"; - } - - // 32-bit private, local, and region pointers. 64-bit global, constant and - // flat. 160-bit non-integral fat buffer pointers that include a 128-bit - // buffer descriptor and a 32-bit offset, which are indexed by 32-bit values - // (address space 7), and 128-bit non-integral buffer resourcees (address - // space 8) which cannot be non-trivilally accessed by LLVM memory operations - // like getelementptr. - return "e-p:64:64-p1:64:64-p2:32:32-p3:32:32-p4:64:64-p5:32:32-p6:32:32" - "-p7:160:256:256:32-p8:128:128:128:48-p9:192:256:256:32-i64:64-" - "v16:16-v24:32-v32:32-v48:64-v96:128-v192:256-v256:256-v512:512-" - "v1024:1024-v2048:2048-n32:64-S32-A5-G1-ni:7:8:9"; -} - LLVM_READNONE static StringRef getGPUOrDefault(const Triple &TT, StringRef GPU) { if (!GPU.empty()) @@ -764,7 +745,7 @@ AMDGPUTargetMachine::AMDGPUTargetMachine(const Target &T, const Triple &TT, std::optional<CodeModel::Model> CM, CodeGenOptLevel OptLevel) : CodeGenTargetMachineImpl( - T, computeDataLayout(TT), TT, getGPUOrDefault(TT, CPU), FS, Options, + T, TT.computeDataLayout(), TT, getGPUOrDefault(TT, CPU), FS, Options, getEffectiveRelocModel(RM), getEffectiveCodeModel(CM, CodeModel::Small), OptLevel), TLOF(createTLOF(getTargetTriple())) { diff --git a/llvm/lib/Target/AMDGPU/AsmParser/AMDGPUAsmParser.cpp b/llvm/lib/Target/AMDGPU/AsmParser/AMDGPUAsmParser.cpp index 2ce1e9e410b2..e420f2ad676f 100644 --- a/llvm/lib/Target/AMDGPU/AsmParser/AMDGPUAsmParser.cpp +++ b/llvm/lib/Target/AMDGPU/AsmParser/AMDGPUAsmParser.cpp @@ -6181,12 +6181,6 @@ bool AMDGPUAsmParser::ParseDirectiveAMDHSAKernel() { ExprVal, ValRange); if (Val) ImpliedUserSGPRCount += 1; - } else if (ID == ".amdhsa_uses_cu_stores") { - if (!isGFX1250()) - return Error(IDRange.Start, "directive requires gfx12.5", IDRange); - - PARSE_BITS_ENTRY(KD.kernel_code_properties, - KERNEL_CODE_PROPERTY_USES_CU_STORES, ExprVal, ValRange); } else if (ID == ".amdhsa_wavefront_size32") { EXPR_RESOLVE_OR_ERROR(EvaluatableExpr); if (IVersion.Major < 10) diff --git a/llvm/lib/Target/AMDGPU/Disassembler/AMDGPUDisassembler.cpp b/llvm/lib/Target/AMDGPU/Disassembler/AMDGPUDisassembler.cpp index b50b2a2e6e23..d3db1b739467 100644 --- a/llvm/lib/Target/AMDGPU/Disassembler/AMDGPUDisassembler.cpp +++ b/llvm/lib/Target/AMDGPU/Disassembler/AMDGPUDisassembler.cpp @@ -368,51 +368,9 @@ static DecodeStatus decodeOperandVOPDDstY(MCInst &Inst, unsigned Val, return addOperand(Inst, DAsm->decodeVOPDDstYOp(Inst, Val)); } -static bool IsAGPROperand(const MCInst &Inst, int OpIdx, - const MCRegisterInfo *MRI) { - if (OpIdx < 0) - return false; - - const MCOperand &Op = Inst.getOperand(OpIdx); - if (!Op.isReg()) - return false; - - MCRegister Sub = MRI->getSubReg(Op.getReg(), AMDGPU::sub0); - auto Reg = Sub ? Sub : Op.getReg(); - return Reg >= AMDGPU::AGPR0 && Reg <= AMDGPU::AGPR255; -} - static DecodeStatus decodeAVLdSt(MCInst &Inst, unsigned Imm, unsigned Opw, const MCDisassembler *Decoder) { const auto *DAsm = static_cast<const AMDGPUDisassembler *>(Decoder); - if (!DAsm->isGFX90A()) { - Imm &= 511; - } else { - // If atomic has both vdata and vdst their register classes are tied. - // The bit is decoded along with the vdst, first operand. We need to - // change register class to AGPR if vdst was AGPR. - // If a DS instruction has both data0 and data1 their register classes - // are also tied. - unsigned Opc = Inst.getOpcode(); - uint64_t TSFlags = DAsm->getMCII()->get(Opc).TSFlags; - AMDGPU::OpName DataName = (TSFlags & SIInstrFlags::DS) - ? AMDGPU::OpName::data0 - : AMDGPU::OpName::vdata; - const MCRegisterInfo *MRI = DAsm->getContext().getRegisterInfo(); - int DataIdx = AMDGPU::getNamedOperandIdx(Opc, DataName); - if ((int)Inst.getNumOperands() == DataIdx) { - int DstIdx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::vdst); - if (IsAGPROperand(Inst, DstIdx, MRI)) - Imm |= 512; - } - - if (TSFlags & SIInstrFlags::DS) { - int Data2Idx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::data1); - if ((int)Inst.getNumOperands() == Data2Idx && - IsAGPROperand(Inst, DataIdx, MRI)) - Imm |= 512; - } - } return addOperand(Inst, DAsm->decodeSrcOp(Opw, Imm | 256)); } @@ -2639,9 +2597,6 @@ Expected<bool> AMDGPUDisassembler::decodeKernelDescriptorDirective( KERNEL_CODE_PROPERTY_ENABLE_SGPR_FLAT_SCRATCH_INIT); PRINT_DIRECTIVE(".amdhsa_user_sgpr_private_segment_size", KERNEL_CODE_PROPERTY_ENABLE_SGPR_PRIVATE_SEGMENT_SIZE); - if (isGFX1250()) - PRINT_DIRECTIVE(".amdhsa_uses_cu_stores", - KERNEL_CODE_PROPERTY_USES_CU_STORES); if (TwoByteBuffer & KERNEL_CODE_PROPERTY_RESERVED0) return createReservedKDBitsError(KERNEL_CODE_PROPERTY_RESERVED0, diff --git a/llvm/lib/Target/AMDGPU/GCNSubtarget.h b/llvm/lib/Target/AMDGPU/GCNSubtarget.h index 4475c8d1d160..cbd6f64976d2 100644 --- a/llvm/lib/Target/AMDGPU/GCNSubtarget.h +++ b/llvm/lib/Target/AMDGPU/GCNSubtarget.h @@ -252,7 +252,6 @@ protected: bool HasVmemPrefInsts = false; bool HasSafeSmemPrefetch = false; bool HasSafeCUPrefetch = false; - bool HasCUStores = false; bool HasVcmpxExecWARHazard = false; bool HasLdsBranchVmemWARHazard = false; bool HasNSAtoVMEMBug = false; @@ -1017,8 +1016,6 @@ public: bool hasSafeCUPrefetch() const { return HasSafeCUPrefetch; } - bool hasCUStores() const { return HasCUStores; } - // Has s_cmpk_* instructions. bool hasSCmpK() const { return getGeneration() < GFX12; } @@ -1835,6 +1832,13 @@ public: bool hasScratchBaseForwardingHazard() const { return GFX1250Insts && getGeneration() == GFX12; } + + /// \returns true if the subtarget supports clusters of workgroups. + bool hasClusters() const { return GFX1250Insts; } + + /// \returns true if the subtarget requires a wait for xcnt before atomic + /// flat/global stores & rmw. + bool requiresWaitXCntBeforeAtomicStores() const { return GFX1250Insts; } }; class GCNUserSGPRUsageInfo { diff --git a/llvm/lib/Target/AMDGPU/MCTargetDesc/AMDGPUTargetStreamer.cpp b/llvm/lib/Target/AMDGPU/MCTargetDesc/AMDGPUTargetStreamer.cpp index 0bbab29dbda1..ff6a21239345 100644 --- a/llvm/lib/Target/AMDGPU/MCTargetDesc/AMDGPUTargetStreamer.cpp +++ b/llvm/lib/Target/AMDGPU/MCTargetDesc/AMDGPUTargetStreamer.cpp @@ -448,11 +448,6 @@ void AMDGPUTargetAsmStreamer::EmitAmdhsaKernelDescriptor( amdhsa::KERNEL_CODE_PROPERTY_ENABLE_SGPR_PRIVATE_SEGMENT_SIZE_SHIFT, amdhsa::KERNEL_CODE_PROPERTY_ENABLE_SGPR_PRIVATE_SEGMENT_SIZE, ".amdhsa_user_sgpr_private_segment_size"); - if (isGFX1250(STI)) - PrintField(KD.kernel_code_properties, - amdhsa::KERNEL_CODE_PROPERTY_USES_CU_STORES_SHIFT, - amdhsa::KERNEL_CODE_PROPERTY_USES_CU_STORES, - ".amdhsa_uses_cu_stores"); if (IVersion.Major >= 10) PrintField(KD.kernel_code_properties, amdhsa::KERNEL_CODE_PROPERTY_ENABLE_WAVEFRONT_SIZE32_SHIFT, diff --git a/llvm/lib/Target/AMDGPU/MIMGInstructions.td b/llvm/lib/Target/AMDGPU/MIMGInstructions.td index bf787b230067..291c03ab223a 100644 --- a/llvm/lib/Target/AMDGPU/MIMGInstructions.td +++ b/llvm/lib/Target/AMDGPU/MIMGInstructions.td @@ -272,13 +272,13 @@ class MIMGNSAHelper<int num_addrs, !if(!le(num_addrs, 13), 3, ?)))); } -class PartialNSAHelper<int num_addrs, int max_addr, RegisterClass LastAddrRC> +class PartialNSAHelper<int num_addrs, int max_addr, RegisterOperand LastAddrRC> : NSAHelper<> { - list<RegisterClass> addr_types = + list<RegisterOperand> addr_types = !if(!ge(num_addrs, max_addr), - !listconcat(!listsplat(VGPR_32, !sub(max_addr, 1)), [LastAddrRC]), - !listsplat(VGPR_32, num_addrs)); + !listconcat(!listsplat(VGPROp_32, !sub(max_addr, 1)), [LastAddrRC]), + !listsplat(VGPROp_32, num_addrs)); int VAddrCount = !if(!gt(num_addrs, max_addr), max_addr, num_addrs); list<string> AddrAsmNames = !foreach(i, !range(VAddrCount), "vaddr" # i); @@ -359,7 +359,7 @@ class MIMG_gfx11<int op, dag outs, string dns = ""> // Note that 1-dword addresses always use non-NSA variants. class MIMG_nsa_gfx11<int op, dag outs, int num_addrs, string dns="", list<RegisterClass> addr_types=[], - RegisterClass LastAddrRC = VGPR_32> + RegisterOperand LastAddrRC = VGPROp_32> : MIMG<outs, dns>, MIMGe_gfx11<op> { let SubtargetPredicate = isGFX11Only; let AssemblerPredicate = isGFX11Only; @@ -400,7 +400,7 @@ class VIMAGE_gfx12<int op, dag outs, int num_addrs, string dns="", } class VSAMPLE_gfx12<int op, dag outs, int num_addrs, string dns="", - RegisterClass Addr3RC> + RegisterOperand Addr3RC> : VSAMPLE<outs, dns>, VSAMPLEe<op> { let SubtargetPredicate = isGFX12Plus; let AssemblerPredicate = isGFX12Plus; @@ -421,7 +421,7 @@ class VSAMPLE_gfx12<int op, dag outs, int num_addrs, string dns="", class MIMG_NoSampler_Helper <mimgopc op, string asm, RegisterOperand dst_rc, - RegisterClass addr_rc, + RegisterOperand addr_rc, string dns=""> : MIMG_gfx6789 <op.GFX10M, (outs dst_rc:$vdata), dns> { let InOperandList = !con((ins addr_rc:$vaddr, SReg_256_XNULL:$srsrc, @@ -434,10 +434,10 @@ class MIMG_NoSampler_Helper <mimgopc op, string asm, class MIMG_NoSampler_Helper_gfx90a <mimgopc op, string asm, RegisterOperand dst_rc, - RegisterClass addr_rc, + RegisterOperand addr_rc, string dns=""> : MIMG_gfx90a <op.GFX10M, (outs getAlign2RegOp<dst_rc>.ret:$vdata), dns> { - let InOperandList = !con((ins addr_rc:$vaddr, SReg_256_XNULL:$srsrc, + let InOperandList = !con((ins getAlign2RegOp<addr_rc>.ret:$vaddr, SReg_256_XNULL:$srsrc, DMask:$dmask, UNorm:$unorm, CPol:$cpol, R128A16:$r128, LWE:$lwe, DA:$da), !if(BaseOpcode.HasD16, (ins D16:$d16), (ins))); @@ -446,7 +446,7 @@ class MIMG_NoSampler_Helper_gfx90a <mimgopc op, string asm, } class MIMG_NoSampler_gfx10<mimgopc op, string opcode, - RegisterOperand DataRC, RegisterClass AddrRC, + RegisterOperand DataRC, RegisterOperand AddrRC, string dns=""> : MIMG_gfx10<op.GFX10M, (outs DataRC:$vdata), dns> { let InOperandList = !con((ins AddrRC:$vaddr0, SReg_256_XNULL:$srsrc, DMask:$dmask, @@ -471,7 +471,7 @@ class MIMG_NoSampler_nsa_gfx10<mimgopc op, string opcode, } class MIMG_NoSampler_gfx11<mimgopc op, string opcode, - RegisterOperand DataRC, RegisterClass AddrRC, + RegisterOperand DataRC, RegisterOperand AddrRC, string dns=""> : MIMG_gfx11<op.GFX11, (outs DataRC:$vdata), dns> { let InOperandList = !con((ins AddrRC:$vaddr0, SReg_256_XNULL:$srsrc, DMask:$dmask, @@ -508,7 +508,7 @@ class VIMAGE_NoSampler_gfx12<mimgopc op, string opcode, } class VSAMPLE_Sampler_gfx12<mimgopc op, string opcode, RegisterOperand DataRC, - int num_addrs, RegisterClass Addr3RC = VGPR_32, + int num_addrs, RegisterOperand Addr3RC = VGPROp_32, string dns=""> : VSAMPLE_gfx12<op.GFX12, (outs DataRC:$vdata), num_addrs, dns, Addr3RC> { let InOperandList = !con(AddrIns, @@ -525,7 +525,7 @@ class VSAMPLE_Sampler_gfx12<mimgopc op, string opcode, RegisterOperand DataRC, } class VSAMPLE_Sampler_nortn_gfx12<mimgopc op, string opcode, - int num_addrs, RegisterClass Addr3RC = VGPR_32, + int num_addrs, RegisterOperand Addr3RC = VGPROp_32, string dns=""> : VSAMPLE_gfx12<op.GFX12, (outs), num_addrs, dns, Addr3RC> { let InOperandList = !con(AddrIns, @@ -550,16 +550,16 @@ multiclass MIMG_NoSampler_Src_Helper <mimgopc op, string asm, let VAddrDwords = 1 in { let ssamp = 0 in { if op.HAS_GFX10M then { - def _V1 : MIMG_NoSampler_Helper <op, asm, dst_rc, VGPR_32, + def _V1 : MIMG_NoSampler_Helper <op, asm, dst_rc, VGPROp_32, !if(enableDisasm, "GFX8", "")>; if !not(ExtendedImageInst) then - def _V1_gfx90a : MIMG_NoSampler_Helper_gfx90a <op, asm, dst_rc, VGPR_32, + def _V1_gfx90a : MIMG_NoSampler_Helper_gfx90a <op, asm, dst_rc, VGPROp_32, !if(enableDisasm, "GFX90A", "")>; - def _V1_gfx10 : MIMG_NoSampler_gfx10<op, asm, dst_rc, VGPR_32, + def _V1_gfx10 : MIMG_NoSampler_gfx10<op, asm, dst_rc, VGPROp_32, !if(enableDisasm, "GFX10", "")>; } if op.HAS_GFX11 then { - def _V1_gfx11 : MIMG_NoSampler_gfx11<op, asm, dst_rc, VGPR_32, + def _V1_gfx11 : MIMG_NoSampler_gfx11<op, asm, dst_rc, VGPROp_32, !if(enableDisasm, "GFX11", "")>; } } @@ -576,14 +576,14 @@ multiclass MIMG_NoSampler_Src_Helper <mimgopc op, string asm, let VAddrDwords = 2 in { let ssamp = 0 in { if op.HAS_GFX10M then { - def _V2 : MIMG_NoSampler_Helper <op, asm, dst_rc, VReg_64>; + def _V2 : MIMG_NoSampler_Helper <op, asm, dst_rc, VGPROp_64>; if !not(ExtendedImageInst) then - def _V2_gfx90a : MIMG_NoSampler_Helper_gfx90a <op, asm, dst_rc, VReg_64_Align2>; - def _V2_gfx10 : MIMG_NoSampler_gfx10<op, asm, dst_rc, VReg_64>; + def _V2_gfx90a : MIMG_NoSampler_Helper_gfx90a <op, asm, dst_rc, VGPROp_64_Align2>; + def _V2_gfx10 : MIMG_NoSampler_gfx10<op, asm, dst_rc, VGPROp_64>; def _V2_nsa_gfx10 : MIMG_NoSampler_nsa_gfx10<op, asm, dst_rc, 2>; } if op.HAS_GFX11 then { - def _V2_gfx11 : MIMG_NoSampler_gfx11<op, asm, dst_rc, VReg_64>; + def _V2_gfx11 : MIMG_NoSampler_gfx11<op, asm, dst_rc, VGPROp_64>; def _V2_nsa_gfx11 : MIMG_NoSampler_nsa_gfx11<op, asm, dst_rc, 2>; } } @@ -600,14 +600,14 @@ multiclass MIMG_NoSampler_Src_Helper <mimgopc op, string asm, let VAddrDwords = 3 in { let ssamp = 0 in { if op.HAS_GFX10M then { - def _V3 : MIMG_NoSampler_Helper <op, asm, dst_rc, VReg_96>; + def _V3 : MIMG_NoSampler_Helper <op, asm, dst_rc, VGPROp_96>; if !not(ExtendedImageInst) then - def _V3_gfx90a : MIMG_NoSampler_Helper_gfx90a <op, asm, dst_rc, VReg_96_Align2>; - def _V3_gfx10 : MIMG_NoSampler_gfx10<op, asm, dst_rc, VReg_96>; + def _V3_gfx90a : MIMG_NoSampler_Helper_gfx90a <op, asm, dst_rc, VGPROp_96_Align2>; + def _V3_gfx10 : MIMG_NoSampler_gfx10<op, asm, dst_rc, VGPROp_96>; def _V3_nsa_gfx10 : MIMG_NoSampler_nsa_gfx10<op, asm, dst_rc, 3>; } if op.HAS_GFX11 then { - def _V3_gfx11 : MIMG_NoSampler_gfx11<op, asm, dst_rc, VReg_96>; + def _V3_gfx11 : MIMG_NoSampler_gfx11<op, asm, dst_rc, VGPROp_96>; def _V3_nsa_gfx11 : MIMG_NoSampler_nsa_gfx11<op, asm, dst_rc, 3>; } } @@ -624,15 +624,15 @@ multiclass MIMG_NoSampler_Src_Helper <mimgopc op, string asm, let VAddrDwords = 4 in { let ssamp = 0 in { if op.HAS_GFX10M then { - def _V4 : MIMG_NoSampler_Helper <op, asm, dst_rc, VReg_128>; + def _V4 : MIMG_NoSampler_Helper <op, asm, dst_rc, VGPROp_128>; if !not(ExtendedImageInst) then - def _V4_gfx90a : MIMG_NoSampler_Helper_gfx90a <op, asm, dst_rc, VReg_128_Align2>; - def _V4_gfx10 : MIMG_NoSampler_gfx10<op, asm, dst_rc, VReg_128>; + def _V4_gfx90a : MIMG_NoSampler_Helper_gfx90a <op, asm, dst_rc, VGPROp_128_Align2>; + def _V4_gfx10 : MIMG_NoSampler_gfx10<op, asm, dst_rc, VGPROp_128>; def _V4_nsa_gfx10 : MIMG_NoSampler_nsa_gfx10<op, asm, dst_rc, 4, !if(enableDisasm, "GFX10", "")>; } if op.HAS_GFX11 then { - def _V4_gfx11 : MIMG_NoSampler_gfx11<op, asm, dst_rc, VReg_128>; + def _V4_gfx11 : MIMG_NoSampler_gfx11<op, asm, dst_rc, VGPROp_128>; def _V4_nsa_gfx11 : MIMG_NoSampler_nsa_gfx11<op, asm, dst_rc, 4, !if(enableDisasm, "GFX11", "")>; } @@ -640,7 +640,7 @@ multiclass MIMG_NoSampler_Src_Helper <mimgopc op, string asm, if op.HAS_GFX12 then { if isVSample then { let samp = 0 in - def _V4_gfx12 : VSAMPLE_Sampler_gfx12<op, asm, dst_rc, 4, VGPR_32, + def _V4_gfx12 : VSAMPLE_Sampler_gfx12<op, asm, dst_rc, 4, VGPROp_32, !if(enableDisasm, "GFX12", "")>; } else { @@ -1128,7 +1128,7 @@ multiclass MIMG_Atomic_Renamed <mimgopc op, string asm, string renamed, : MIMG_Atomic <op, asm, isCmpSwap, isFP, renamed>; class MIMG_Sampler_Helper <mimgopc op, string asm, RegisterOperand dst_rc, - RegisterClass src_rc, string dns=""> + RegisterOperand src_rc, string dns=""> : MIMG_gfx6789 <op.VI, (outs dst_rc:$vdata), dns> { let InOperandList = !con((ins src_rc:$vaddr, SReg_256_XNULL:$srsrc, SReg_128_XNULL:$ssamp, DMask:$dmask, UNorm:$unorm, CPol:$cpol, @@ -1139,7 +1139,7 @@ class MIMG_Sampler_Helper <mimgopc op, string asm, RegisterOperand dst_rc, } class MIMG_Sampler_gfx90a<mimgopc op, string asm, RegisterOperand dst_rc, - RegisterClass src_rc, string dns=""> + RegisterOperand src_rc, string dns=""> : MIMG_gfx90a<op.GFX10M, (outs dst_rc:$vdata), dns> { let InOperandList = !con((ins src_rc:$vaddr, SReg_256_XNULL:$srsrc, SReg_128_XNULL:$ssamp, DMask:$dmask, UNorm:$unorm, CPol:$cpol, @@ -1164,7 +1164,7 @@ class MIMG_Sampler_Asm_gfx10p<string opcode, string AsmPrefix, bit HasD16> { } class MIMG_Sampler_gfx10<mimgopc op, string opcode, - RegisterOperand DataRC, RegisterClass AddrRC, + RegisterOperand DataRC, RegisterOperand AddrRC, string dns=""> : MIMG_gfx10<op.GFX10M, (outs DataRC:$vdata), dns> { let InOperandList = MIMG_Sampler_OpList_gfx10p<(ins AddrRC:$vaddr0), BaseOpcode.HasD16>.ret; @@ -1180,7 +1180,7 @@ class MIMG_Sampler_nsa_gfx10<mimgopc op, string opcode, } class MIMG_Sampler_nortn_gfx10<mimgopc op, string opcode, - RegisterClass AddrRC, + RegisterOperand AddrRC, string dns=""> : MIMG_gfx10<op.GFX10M, (outs), dns> { let InOperandList = MIMG_Sampler_OpList_gfx10p<(ins AddrRC:$vaddr0), BaseOpcode.HasD16>.ret; @@ -1200,7 +1200,7 @@ class MIMG_Sampler_nortn_nsa_gfx10<mimgopc op, string opcode, } class MIMG_Sampler_gfx11<mimgopc op, string opcode, - RegisterOperand DataRC, RegisterClass AddrRC, + RegisterOperand DataRC, RegisterOperand AddrRC, string dns=""> : MIMG_gfx11<op.GFX11, (outs DataRC:$vdata), dns> { let InOperandList = MIMG_Sampler_OpList_gfx10p<(ins AddrRC:$vaddr0), BaseOpcode.HasD16>.ret; @@ -1209,7 +1209,7 @@ class MIMG_Sampler_gfx11<mimgopc op, string opcode, class MIMG_Sampler_nsa_gfx11<mimgopc op, string opcode, RegisterOperand DataRC, int num_addrs, - RegisterClass LastVAddrSize, string dns=""> + RegisterOperand LastVAddrSize, string dns=""> : MIMG_nsa_gfx11<op.GFX11, (outs DataRC:$vdata), num_addrs, dns, [], LastVAddrSize> { let InOperandList = MIMG_Sampler_OpList_gfx10p<AddrIns, BaseOpcode.HasD16>.ret; @@ -1217,7 +1217,7 @@ class MIMG_Sampler_nsa_gfx11<mimgopc op, string opcode, } class MIMG_Sampler_nortn_gfx11<mimgopc op, string opcode, - RegisterClass AddrRC, + RegisterOperand AddrRC, string dns=""> : MIMG_gfx11<op.GFX11, (outs), dns> { let InOperandList = MIMG_Sampler_OpList_gfx10p<(ins AddrRC:$vaddr0), BaseOpcode.HasD16>.ret; @@ -1227,7 +1227,7 @@ class MIMG_Sampler_nortn_gfx11<mimgopc op, string opcode, class MIMG_Sampler_nortn_nsa_gfx11<mimgopc op, string opcode, int num_addrs, - RegisterClass LastVAddrSize, string dns=""> + RegisterOperand LastVAddrSize, string dns=""> : MIMG_nsa_gfx11<op.GFX11, (outs), num_addrs, dns, [], LastVAddrSize> { let InOperandList = MIMG_Sampler_OpList_gfx10p<AddrIns, BaseOpcode.HasD16>.ret; let AsmString = MIMG_Sampler_Asm_gfx10p<opcode, "off, "#AddrAsm, BaseOpcode.HasD16>.ret; @@ -1237,20 +1237,20 @@ class MIMG_Sampler_nortn_nsa_gfx11<mimgopc op, string opcode, class MIMGAddrSize<int dw, bit enable_disasm, int AddrDW = dw> { int NumWords = dw; - RegisterClass RegClass = !if(!le(AddrDW, 0), ?, - !if(!eq(AddrDW, 1), VGPR_32, - !if(!eq(AddrDW, 2), VReg_64, - !if(!eq(AddrDW, 3), VReg_96, - !if(!eq(AddrDW, 4), VReg_128, - !if(!eq(AddrDW, 5), VReg_160, - !if(!eq(AddrDW, 6), VReg_192, - !if(!eq(AddrDW, 7), VReg_224, - !if(!eq(AddrDW, 8), VReg_256, - !if(!eq(AddrDW, 9), VReg_288, - !if(!eq(AddrDW, 10), VReg_320, - !if(!eq(AddrDW, 11), VReg_352, - !if(!eq(AddrDW, 12), VReg_384, - !if(!le(AddrDW, 16), VReg_512, ?)))))))))))))); + RegisterOperand RegClass = !if(!le(AddrDW, 0), ?, + !if(!eq(AddrDW, 1), VGPROp_32, + !if(!eq(AddrDW, 2), VGPROp_64, + !if(!eq(AddrDW, 3), VGPROp_96, + !if(!eq(AddrDW, 4), VGPROp_128, + !if(!eq(AddrDW, 5), VGPROp_160, + !if(!eq(AddrDW, 6), VGPROp_192, + !if(!eq(AddrDW, 7), VGPROp_224, + !if(!eq(AddrDW, 8), VGPROp_256, + !if(!eq(AddrDW, 9), VGPROp_288, + !if(!eq(AddrDW, 10), VGPROp_320, + !if(!eq(AddrDW, 11), VGPROp_352, + !if(!eq(AddrDW, 12), VGPROp_384, + !if(!le(AddrDW, 16), VGPROp_512, ?)))))))))))))); // Whether the instruction variant with this vaddr size should be enabled for // the auto-generated disassembler. @@ -1514,8 +1514,10 @@ multiclass MIMG_Gather_WQM <mimgopc op, AMDGPUSampleVariant sample> class MIMG_IntersectRay_Helper<bit Is64, bit IsA16, bit isDual, bit isBVH8> { int num_addrs = !if(isBVH8, 11, !if(Is64, !if(IsA16, 9, 12), !if(IsA16, 8, 11))); - RegisterClass RegClass = MIMGAddrSize<num_addrs, 0>.RegClass; - int VAddrDwords = !srl(RegClass.Size, 5); + RegisterOperand RegClass = MIMGAddrSize<num_addrs, 0>.RegClass; + + defvar Size = !cast<SIRegisterClass>(RegClass.RegClass).Size; + int VAddrDwords = !srl(Size, 5); int GFX11PlusNSAAddrs = !if(IsA16, 4, 5); RegisterClass node_ptr_type = !if(Is64, VReg_64, VGPR_32); @@ -1526,7 +1528,7 @@ class MIMG_IntersectRay_Helper<bit Is64, bit IsA16, bit isDual, bit isBVH8> { true : [node_ptr_type, VGPR_32, VReg_96, VReg_96, VReg_96]); } -class MIMG_IntersectRay_gfx10<mimgopc op, string opcode, RegisterClass AddrRC> +class MIMG_IntersectRay_gfx10<mimgopc op, string opcode, RegisterOperand AddrRC> : MIMG_gfx10<op.GFX10M, (outs VReg_128:$vdata), "GFX10"> { let InOperandList = (ins AddrRC:$vaddr0, SReg_128_XNULL:$srsrc, A16:$a16); let AsmString = opcode#" $vdata, $vaddr0, $srsrc$a16"; @@ -1540,7 +1542,7 @@ class MIMG_IntersectRay_nsa_gfx10<mimgopc op, string opcode, int num_addrs> let AsmString = opcode#" $vdata, "#nsah.AddrAsm#", $srsrc$a16"; } -class MIMG_IntersectRay_gfx11<mimgopc op, string opcode, RegisterClass AddrRC> +class MIMG_IntersectRay_gfx11<mimgopc op, string opcode, RegisterOperand AddrRC> : MIMG_gfx11<op.GFX11, (outs VReg_128:$vdata), "GFX11"> { let InOperandList = (ins AddrRC:$vaddr0, SReg_128_XNULL:$srsrc, A16:$a16); let AsmString = opcode#" $vdata, $vaddr0, $srsrc$a16"; diff --git a/llvm/lib/Target/AMDGPU/SIISelLowering.cpp b/llvm/lib/Target/AMDGPU/SIISelLowering.cpp index 65cfffd7ec0e..cb3e544449bb 100644 --- a/llvm/lib/Target/AMDGPU/SIISelLowering.cpp +++ b/llvm/lib/Target/AMDGPU/SIISelLowering.cpp @@ -440,6 +440,14 @@ SITargetLowering::SITargetLowering(const TargetMachine &TM, setOperationAction(ISD::VECTOR_SHUFFLE, {MVT::v2i32, MVT::v2f32}, Legal); } + setOperationAction({ISD::AND, ISD::OR, ISD::XOR}, MVT::v2i32, Legal); + // Prevent SELECT v2i32 from being implemented with the above bitwise ops and + // instead lower to cndmask in SITargetLowering::LowerSELECT(). + setOperationAction(ISD::SELECT, MVT::v2i32, Custom); + // Enable MatchRotate to produce ISD::ROTR, which is later transformed to + // alignbit. + setOperationAction(ISD::ROTR, MVT::v2i32, Custom); + setOperationAction(ISD::BUILD_VECTOR, {MVT::v4f16, MVT::v4i16, MVT::v4bf16}, Custom); @@ -3452,7 +3460,7 @@ bool SITargetLowering::CanLowerReturn( // We must use the stack if return would require unavailable registers. unsigned MaxNumVGPRs = Subtarget->getMaxNumVGPRs(MF); - unsigned TotalNumVGPRs = AMDGPU::VGPR_32RegClass.getNumRegs(); + unsigned TotalNumVGPRs = Subtarget->getAddressableNumArchVGPRs(); for (unsigned i = MaxNumVGPRs; i < TotalNumVGPRs; ++i) if (CCInfo.isAllocated(AMDGPU::VGPR_32RegClass.getRegister(i))) return false; @@ -5270,7 +5278,58 @@ static MachineBasicBlock *emitIndirectDst(MachineInstr &MI, return LoopBB; } -static uint32_t getIdentityValueForWaveReduction(unsigned Opc) { +static MachineBasicBlock *Expand64BitScalarArithmetic(MachineInstr &MI, + MachineBasicBlock *BB) { + // For targets older than GFX12, we emit a sequence of 32-bit operations. + // For GFX12, we emit s_add_u64 and s_sub_u64. + MachineFunction *MF = BB->getParent(); + const SIInstrInfo *TII = MF->getSubtarget<GCNSubtarget>().getInstrInfo(); + const GCNSubtarget &ST = MF->getSubtarget<GCNSubtarget>(); + MachineRegisterInfo &MRI = BB->getParent()->getRegInfo(); + const DebugLoc &DL = MI.getDebugLoc(); + MachineOperand &Dest = MI.getOperand(0); + MachineOperand &Src0 = MI.getOperand(1); + MachineOperand &Src1 = MI.getOperand(2); + bool IsAdd = (MI.getOpcode() == AMDGPU::S_ADD_U64_PSEUDO); + if (ST.hasScalarAddSub64()) { + unsigned Opc = IsAdd ? AMDGPU::S_ADD_U64 : AMDGPU::S_SUB_U64; + // clang-format off + BuildMI(*BB, MI, DL, TII->get(Opc), Dest.getReg()) + .add(Src0) + .add(Src1); + // clang-format on + } else { + const SIRegisterInfo *TRI = ST.getRegisterInfo(); + const TargetRegisterClass *BoolRC = TRI->getBoolRC(); + + Register DestSub0 = MRI.createVirtualRegister(&AMDGPU::SReg_32RegClass); + Register DestSub1 = MRI.createVirtualRegister(&AMDGPU::SReg_32RegClass); + + MachineOperand Src0Sub0 = TII->buildExtractSubRegOrImm( + MI, MRI, Src0, BoolRC, AMDGPU::sub0, &AMDGPU::SReg_32RegClass); + MachineOperand Src0Sub1 = TII->buildExtractSubRegOrImm( + MI, MRI, Src0, BoolRC, AMDGPU::sub1, &AMDGPU::SReg_32RegClass); + + MachineOperand Src1Sub0 = TII->buildExtractSubRegOrImm( + MI, MRI, Src1, BoolRC, AMDGPU::sub0, &AMDGPU::SReg_32RegClass); + MachineOperand Src1Sub1 = TII->buildExtractSubRegOrImm( + MI, MRI, Src1, BoolRC, AMDGPU::sub1, &AMDGPU::SReg_32RegClass); + + unsigned LoOpc = IsAdd ? AMDGPU::S_ADD_U32 : AMDGPU::S_SUB_U32; + unsigned HiOpc = IsAdd ? AMDGPU::S_ADDC_U32 : AMDGPU::S_SUBB_U32; + BuildMI(*BB, MI, DL, TII->get(LoOpc), DestSub0).add(Src0Sub0).add(Src1Sub0); + BuildMI(*BB, MI, DL, TII->get(HiOpc), DestSub1).add(Src0Sub1).add(Src1Sub1); + BuildMI(*BB, MI, DL, TII->get(TargetOpcode::REG_SEQUENCE), Dest.getReg()) + .addReg(DestSub0) + .addImm(AMDGPU::sub0) + .addReg(DestSub1) + .addImm(AMDGPU::sub1); + } + MI.eraseFromParent(); + return BB; +} + +static uint32_t getIdentityValueFor32BitWaveReduction(unsigned Opc) { switch (Opc) { case AMDGPU::S_MIN_U32: return std::numeric_limits<uint32_t>::max(); @@ -5288,10 +5347,42 @@ static uint32_t getIdentityValueForWaveReduction(unsigned Opc) { case AMDGPU::S_AND_B32: return std::numeric_limits<uint32_t>::max(); default: - llvm_unreachable("Unexpected opcode in getIdentityValueForWaveReduction"); + llvm_unreachable( + "Unexpected opcode in getIdentityValueFor32BitWaveReduction"); } } +static uint64_t getIdentityValueFor64BitWaveReduction(unsigned Opc) { + switch (Opc) { + case AMDGPU::V_CMP_LT_U64_e64: // umin.u64 + return std::numeric_limits<uint64_t>::max(); + case AMDGPU::V_CMP_LT_I64_e64: // min.i64 + return std::numeric_limits<int64_t>::max(); + case AMDGPU::V_CMP_GT_U64_e64: // umax.u64 + return std::numeric_limits<uint64_t>::min(); + case AMDGPU::V_CMP_GT_I64_e64: // max.i64 + return std::numeric_limits<int64_t>::min(); + case AMDGPU::S_ADD_U64_PSEUDO: + case AMDGPU::S_SUB_U64_PSEUDO: + case AMDGPU::S_OR_B64: + case AMDGPU::S_XOR_B64: + return std::numeric_limits<uint64_t>::min(); + case AMDGPU::S_AND_B64: + return std::numeric_limits<uint64_t>::max(); + default: + llvm_unreachable( + "Unexpected opcode in getIdentityValueFor64BitWaveReduction"); + } +} + +static bool is32bitWaveReduceOperation(unsigned Opc) { + return Opc == AMDGPU::S_MIN_U32 || Opc == AMDGPU::S_MIN_I32 || + Opc == AMDGPU::S_MAX_U32 || Opc == AMDGPU::S_MAX_I32 || + Opc == AMDGPU::S_ADD_I32 || Opc == AMDGPU::S_SUB_I32 || + Opc == AMDGPU::S_AND_B32 || Opc == AMDGPU::S_OR_B32 || + Opc == AMDGPU::S_XOR_B32; +} + static MachineBasicBlock *lowerWaveReduce(MachineInstr &MI, MachineBasicBlock &BB, const GCNSubtarget &ST, @@ -5319,53 +5410,99 @@ static MachineBasicBlock *lowerWaveReduce(MachineInstr &MI, RetBB = &BB; break; } + case AMDGPU::V_CMP_LT_U64_e64: // umin + case AMDGPU::V_CMP_LT_I64_e64: // min + case AMDGPU::V_CMP_GT_U64_e64: // umax + case AMDGPU::V_CMP_GT_I64_e64: // max + case AMDGPU::S_AND_B64: + case AMDGPU::S_OR_B64: { + // Idempotent operations. + BuildMI(BB, MI, DL, TII->get(AMDGPU::S_MOV_B64), DstReg).addReg(SrcReg); + RetBB = &BB; + break; + } case AMDGPU::S_XOR_B32: + case AMDGPU::S_XOR_B64: case AMDGPU::S_ADD_I32: - case AMDGPU::S_SUB_I32: { + case AMDGPU::S_ADD_U64_PSEUDO: + case AMDGPU::S_SUB_I32: + case AMDGPU::S_SUB_U64_PSEUDO: { const TargetRegisterClass *WaveMaskRegClass = TRI->getWaveMaskRegClass(); const TargetRegisterClass *DstRegClass = MRI.getRegClass(DstReg); Register ExecMask = MRI.createVirtualRegister(WaveMaskRegClass); - Register ActiveLanes = MRI.createVirtualRegister(DstRegClass); + Register NumActiveLanes = + MRI.createVirtualRegister(&AMDGPU::SReg_32RegClass); bool IsWave32 = ST.isWave32(); unsigned MovOpc = IsWave32 ? AMDGPU::S_MOV_B32 : AMDGPU::S_MOV_B64; MCRegister ExecReg = IsWave32 ? AMDGPU::EXEC_LO : AMDGPU::EXEC; - unsigned CountReg = + unsigned BitCountOpc = IsWave32 ? AMDGPU::S_BCNT1_I32_B32 : AMDGPU::S_BCNT1_I32_B64; - auto Exec = - BuildMI(BB, MI, DL, TII->get(MovOpc), ExecMask).addReg(ExecReg); + BuildMI(BB, MI, DL, TII->get(MovOpc), ExecMask).addReg(ExecReg); - auto NewAccumulator = BuildMI(BB, MI, DL, TII->get(CountReg), ActiveLanes) - .addReg(Exec->getOperand(0).getReg()); + auto NewAccumulator = + BuildMI(BB, MI, DL, TII->get(BitCountOpc), NumActiveLanes) + .addReg(ExecMask); switch (Opc) { - case AMDGPU::S_XOR_B32: { + case AMDGPU::S_XOR_B32: + case AMDGPU::S_XOR_B64: { // Performing an XOR operation on a uniform value // depends on the parity of the number of active lanes. // For even parity, the result will be 0, for odd // parity the result will be the same as the input value. - Register ParityRegister = MRI.createVirtualRegister(DstRegClass); - - auto ParityReg = - BuildMI(BB, MI, DL, TII->get(AMDGPU::S_AND_B32), ParityRegister) - .addReg(NewAccumulator->getOperand(0).getReg()) - .addImm(1); - BuildMI(BB, MI, DL, TII->get(AMDGPU::S_MUL_I32), DstReg) - .addReg(SrcReg) - .addReg(ParityReg->getOperand(0).getReg()); + Register ParityRegister = + MRI.createVirtualRegister(&AMDGPU::SReg_32RegClass); + + BuildMI(BB, MI, DL, TII->get(AMDGPU::S_AND_B32), ParityRegister) + .addReg(NewAccumulator->getOperand(0).getReg()) + .addImm(1) + .setOperandDead(3); // Dead scc + if (Opc == AMDGPU::S_XOR_B32) { + BuildMI(BB, MI, DL, TII->get(AMDGPU::S_MUL_I32), DstReg) + .addReg(SrcReg) + .addReg(ParityRegister); + } else { + Register DestSub0 = + MRI.createVirtualRegister(&AMDGPU::SReg_32RegClass); + Register DestSub1 = + MRI.createVirtualRegister(&AMDGPU::SReg_32RegClass); + + const TargetRegisterClass *SrcRC = MRI.getRegClass(SrcReg); + const TargetRegisterClass *SrcSubRC = + TRI->getSubRegisterClass(SrcRC, AMDGPU::sub0); + + MachineOperand Op1L = TII->buildExtractSubRegOrImm( + MI, MRI, MI.getOperand(1), SrcRC, AMDGPU::sub0, SrcSubRC); + MachineOperand Op1H = TII->buildExtractSubRegOrImm( + MI, MRI, MI.getOperand(1), SrcRC, AMDGPU::sub1, SrcSubRC); + + BuildMI(BB, MI, DL, TII->get(AMDGPU::S_MUL_I32), DestSub0) + .add(Op1L) + .addReg(ParityRegister); + + BuildMI(BB, MI, DL, TII->get(AMDGPU::S_MUL_I32), DestSub1) + .add(Op1H) + .addReg(ParityRegister); + + BuildMI(BB, MI, DL, TII->get(TargetOpcode::REG_SEQUENCE), DstReg) + .addReg(DestSub0) + .addImm(AMDGPU::sub0) + .addReg(DestSub1) + .addImm(AMDGPU::sub1); + } break; } case AMDGPU::S_SUB_I32: { Register NegatedVal = MRI.createVirtualRegister(DstRegClass); // Take the negation of the source operand. - auto InvertedValReg = - BuildMI(BB, MI, DL, TII->get(AMDGPU::S_MUL_I32), NegatedVal) - .addImm(-1) - .addReg(SrcReg); + BuildMI(BB, MI, DL, TII->get(AMDGPU::S_SUB_I32), NegatedVal) + .addImm(0) + .addReg(SrcReg); BuildMI(BB, MI, DL, TII->get(AMDGPU::S_MUL_I32), DstReg) - .addReg(InvertedValReg->getOperand(0).getReg()) + .addReg(NegatedVal) .addReg(NewAccumulator->getOperand(0).getReg()); break; } @@ -5375,6 +5512,75 @@ static MachineBasicBlock *lowerWaveReduce(MachineInstr &MI, .addReg(NewAccumulator->getOperand(0).getReg()); break; } + case AMDGPU::S_ADD_U64_PSEUDO: + case AMDGPU::S_SUB_U64_PSEUDO: { + Register DestSub0 = MRI.createVirtualRegister(&AMDGPU::SReg_32RegClass); + Register DestSub1 = MRI.createVirtualRegister(&AMDGPU::SReg_32RegClass); + Register Op1H_Op0L_Reg = + MRI.createVirtualRegister(&AMDGPU::SReg_32RegClass); + Register Op1L_Op0H_Reg = + MRI.createVirtualRegister(&AMDGPU::SReg_32RegClass); + Register CarryReg = MRI.createVirtualRegister(&AMDGPU::SReg_32RegClass); + Register AddReg = MRI.createVirtualRegister(&AMDGPU::SReg_32RegClass); + Register NegatedValLo = + MRI.createVirtualRegister(&AMDGPU::SReg_32RegClass); + Register NegatedValHi = + MRI.createVirtualRegister(&AMDGPU::SReg_32RegClass); + + const TargetRegisterClass *Src1RC = MRI.getRegClass(SrcReg); + const TargetRegisterClass *Src1SubRC = + TRI->getSubRegisterClass(Src1RC, AMDGPU::sub0); + + MachineOperand Op1L = TII->buildExtractSubRegOrImm( + MI, MRI, MI.getOperand(1), Src1RC, AMDGPU::sub0, Src1SubRC); + MachineOperand Op1H = TII->buildExtractSubRegOrImm( + MI, MRI, MI.getOperand(1), Src1RC, AMDGPU::sub1, Src1SubRC); + + if (Opc == AMDGPU::S_SUB_U64_PSEUDO) { + BuildMI(BB, MI, DL, TII->get(AMDGPU::S_SUB_I32), NegatedValLo) + .addImm(0) + .addReg(NewAccumulator->getOperand(0).getReg()) + .setOperandDead(3); // Dead scc + BuildMI(BB, MI, DL, TII->get(AMDGPU::S_ASHR_I32), NegatedValHi) + .addReg(NegatedValLo) + .addImm(31) + .setOperandDead(3); // Dead scc + BuildMI(BB, MI, DL, TII->get(AMDGPU::S_MUL_I32), Op1L_Op0H_Reg) + .add(Op1L) + .addReg(NegatedValHi); + } + Register LowOpcode = Opc == AMDGPU::S_SUB_U64_PSEUDO + ? NegatedValLo + : NewAccumulator->getOperand(0).getReg(); + BuildMI(BB, MI, DL, TII->get(AMDGPU::S_MUL_I32), DestSub0) + .add(Op1L) + .addReg(LowOpcode); + BuildMI(BB, MI, DL, TII->get(AMDGPU::S_MUL_HI_U32), CarryReg) + .add(Op1L) + .addReg(LowOpcode); + BuildMI(BB, MI, DL, TII->get(AMDGPU::S_MUL_I32), Op1H_Op0L_Reg) + .add(Op1H) + .addReg(LowOpcode); + + Register HiVal = Opc == AMDGPU::S_SUB_U64_PSEUDO ? AddReg : DestSub1; + BuildMI(BB, MI, DL, TII->get(AMDGPU::S_ADD_U32), HiVal) + .addReg(CarryReg) + .addReg(Op1H_Op0L_Reg) + .setOperandDead(3); // Dead scc + + if (Opc == AMDGPU::S_SUB_U64_PSEUDO) { + BuildMI(BB, MI, DL, TII->get(AMDGPU::S_ADD_U32), DestSub1) + .addReg(HiVal) + .addReg(Op1L_Op0H_Reg) + .setOperandDead(3); // Dead scc + } + BuildMI(BB, MI, DL, TII->get(TargetOpcode::REG_SEQUENCE), DstReg) + .addReg(DestSub0) + .addImm(AMDGPU::sub0) + .addReg(DestSub1) + .addImm(AMDGPU::sub1); + break; + } } RetBB = &BB; } @@ -5391,6 +5597,7 @@ static MachineBasicBlock *lowerWaveReduce(MachineInstr &MI, // so that we will get the next active lane for next iteration. MachineBasicBlock::iterator I = BB.end(); Register SrcReg = MI.getOperand(1).getReg(); + bool is32BitOpc = is32bitWaveReduceOperation(Opc); // Create Control flow for loop // Split MI's Machine Basic block into For loop @@ -5400,73 +5607,160 @@ static MachineBasicBlock *lowerWaveReduce(MachineInstr &MI, const TargetRegisterClass *WaveMaskRegClass = TRI->getWaveMaskRegClass(); const TargetRegisterClass *DstRegClass = MRI.getRegClass(DstReg); Register LoopIterator = MRI.createVirtualRegister(WaveMaskRegClass); - Register InitalValReg = MRI.createVirtualRegister(DstRegClass); - + Register IdentityValReg = MRI.createVirtualRegister(DstRegClass); Register AccumulatorReg = MRI.createVirtualRegister(DstRegClass); Register ActiveBitsReg = MRI.createVirtualRegister(WaveMaskRegClass); Register NewActiveBitsReg = MRI.createVirtualRegister(WaveMaskRegClass); - - Register FF1Reg = MRI.createVirtualRegister(DstRegClass); - Register LaneValueReg = - MRI.createVirtualRegister(&AMDGPU::SReg_32_XM0RegClass); + Register FF1Reg = MRI.createVirtualRegister(&AMDGPU::SReg_32RegClass); + Register LaneValueReg = MRI.createVirtualRegister(DstRegClass); bool IsWave32 = ST.isWave32(); - unsigned MovOpc = IsWave32 ? AMDGPU::S_MOV_B32 : AMDGPU::S_MOV_B64; + unsigned MovOpcForExec = IsWave32 ? AMDGPU::S_MOV_B32 : AMDGPU::S_MOV_B64; unsigned ExecReg = IsWave32 ? AMDGPU::EXEC_LO : AMDGPU::EXEC; // Create initial values of induction variable from Exec, Accumulator and // insert branch instr to newly created ComputeBlock - uint32_t InitalValue = getIdentityValueForWaveReduction(Opc); - auto TmpSReg = - BuildMI(BB, I, DL, TII->get(MovOpc), LoopIterator).addReg(ExecReg); - BuildMI(BB, I, DL, TII->get(AMDGPU::S_MOV_B32), InitalValReg) - .addImm(InitalValue); + BuildMI(BB, I, DL, TII->get(MovOpcForExec), LoopIterator).addReg(ExecReg); + if (is32BitOpc) { + uint32_t IdentityValue = getIdentityValueFor32BitWaveReduction(Opc); + BuildMI(BB, I, DL, TII->get(AMDGPU::S_MOV_B32), IdentityValReg) + .addImm(IdentityValue); + } else { + uint64_t IdentityValue = getIdentityValueFor64BitWaveReduction(Opc); + BuildMI(BB, I, DL, TII->get(AMDGPU::S_MOV_B64_IMM_PSEUDO), IdentityValReg) + .addImm(IdentityValue); + } // clang-format off BuildMI(BB, I, DL, TII->get(AMDGPU::S_BRANCH)) .addMBB(ComputeLoop); // clang-format on // Start constructing ComputeLoop - I = ComputeLoop->end(); + I = ComputeLoop->begin(); auto Accumulator = BuildMI(*ComputeLoop, I, DL, TII->get(AMDGPU::PHI), AccumulatorReg) - .addReg(InitalValReg) + .addReg(IdentityValReg) .addMBB(&BB); auto ActiveBits = BuildMI(*ComputeLoop, I, DL, TII->get(AMDGPU::PHI), ActiveBitsReg) - .addReg(TmpSReg->getOperand(0).getReg()) + .addReg(LoopIterator) .addMBB(&BB); + I = ComputeLoop->end(); + MachineInstr *NewAccumulator; // Perform the computations unsigned SFFOpc = IsWave32 ? AMDGPU::S_FF1_I32_B32 : AMDGPU::S_FF1_I32_B64; - auto FF1 = BuildMI(*ComputeLoop, I, DL, TII->get(SFFOpc), FF1Reg) - .addReg(ActiveBits->getOperand(0).getReg()); - auto LaneValue = BuildMI(*ComputeLoop, I, DL, - TII->get(AMDGPU::V_READLANE_B32), LaneValueReg) - .addReg(SrcReg) - .addReg(FF1->getOperand(0).getReg()); - auto NewAccumulator = BuildMI(*ComputeLoop, I, DL, TII->get(Opc), DstReg) - .addReg(Accumulator->getOperand(0).getReg()) - .addReg(LaneValue->getOperand(0).getReg()); - + BuildMI(*ComputeLoop, I, DL, TII->get(SFFOpc), FF1Reg) + .addReg(ActiveBitsReg); + if (is32BitOpc) { + BuildMI(*ComputeLoop, I, DL, TII->get(AMDGPU::V_READLANE_B32), + LaneValueReg) + .addReg(SrcReg) + .addReg(FF1Reg); + NewAccumulator = BuildMI(*ComputeLoop, I, DL, TII->get(Opc), DstReg) + .addReg(Accumulator->getOperand(0).getReg()) + .addReg(LaneValueReg); + } else { + Register LaneValueLoReg = + MRI.createVirtualRegister(&AMDGPU::SReg_32_XM0RegClass); + Register LaneValueHiReg = + MRI.createVirtualRegister(&AMDGPU::SReg_32_XM0RegClass); + Register LaneValReg = MRI.createVirtualRegister(&AMDGPU::SReg_64RegClass); + const TargetRegisterClass *SrcRC = MRI.getRegClass(SrcReg); + const TargetRegisterClass *SrcSubRC = + TRI->getSubRegisterClass(SrcRC, AMDGPU::sub0); + MachineOperand Op1L = TII->buildExtractSubRegOrImm( + MI, MRI, MI.getOperand(1), SrcRC, AMDGPU::sub0, SrcSubRC); + MachineOperand Op1H = TII->buildExtractSubRegOrImm( + MI, MRI, MI.getOperand(1), SrcRC, AMDGPU::sub1, SrcSubRC); + // lane value input should be in an sgpr + BuildMI(*ComputeLoop, I, DL, TII->get(AMDGPU::V_READLANE_B32), + LaneValueLoReg) + .add(Op1L) + .addReg(FF1Reg); + BuildMI(*ComputeLoop, I, DL, TII->get(AMDGPU::V_READLANE_B32), + LaneValueHiReg) + .add(Op1H) + .addReg(FF1Reg); + auto LaneValue = BuildMI(*ComputeLoop, I, DL, + TII->get(TargetOpcode::REG_SEQUENCE), LaneValReg) + .addReg(LaneValueLoReg) + .addImm(AMDGPU::sub0) + .addReg(LaneValueHiReg) + .addImm(AMDGPU::sub1); + switch (Opc) { + case AMDGPU::S_OR_B64: + case AMDGPU::S_AND_B64: + case AMDGPU::S_XOR_B64: { + NewAccumulator = BuildMI(*ComputeLoop, I, DL, TII->get(Opc), DstReg) + .addReg(Accumulator->getOperand(0).getReg()) + .addReg(LaneValue->getOperand(0).getReg()) + .setOperandDead(3); // Dead scc + break; + } + case AMDGPU::V_CMP_GT_I64_e64: + case AMDGPU::V_CMP_GT_U64_e64: + case AMDGPU::V_CMP_LT_I64_e64: + case AMDGPU::V_CMP_LT_U64_e64: { + Register LaneMaskReg = MRI.createVirtualRegister(WaveMaskRegClass); + Register ComparisonResultReg = + MRI.createVirtualRegister(WaveMaskRegClass); + const TargetRegisterClass *VregClass = TRI->getVGPR64Class(); + const TargetRegisterClass *VSubRegClass = + TRI->getSubRegisterClass(VregClass, AMDGPU::sub0); + Register AccumulatorVReg = MRI.createVirtualRegister(VregClass); + MachineOperand SrcReg0Sub0 = + TII->buildExtractSubRegOrImm(MI, MRI, Accumulator->getOperand(0), + VregClass, AMDGPU::sub0, VSubRegClass); + MachineOperand SrcReg0Sub1 = + TII->buildExtractSubRegOrImm(MI, MRI, Accumulator->getOperand(0), + VregClass, AMDGPU::sub1, VSubRegClass); + BuildMI(*ComputeLoop, I, DL, TII->get(TargetOpcode::REG_SEQUENCE), + AccumulatorVReg) + .add(SrcReg0Sub0) + .addImm(AMDGPU::sub0) + .add(SrcReg0Sub1) + .addImm(AMDGPU::sub1); + BuildMI(*ComputeLoop, I, DL, TII->get(Opc), LaneMaskReg) + .addReg(LaneValue->getOperand(0).getReg()) + .addReg(AccumulatorVReg); + + unsigned AndOpc = IsWave32 ? AMDGPU::S_AND_B32 : AMDGPU::S_AND_B64; + BuildMI(*ComputeLoop, I, DL, TII->get(AndOpc), ComparisonResultReg) + .addReg(LaneMaskReg) + .addReg(ActiveBitsReg); + + NewAccumulator = BuildMI(*ComputeLoop, I, DL, + TII->get(AMDGPU::S_CSELECT_B64), DstReg) + .addReg(LaneValue->getOperand(0).getReg()) + .addReg(Accumulator->getOperand(0).getReg()); + break; + } + case AMDGPU::S_ADD_U64_PSEUDO: + case AMDGPU::S_SUB_U64_PSEUDO: { + NewAccumulator = BuildMI(*ComputeLoop, I, DL, TII->get(Opc), DstReg) + .addReg(Accumulator->getOperand(0).getReg()) + .addReg(LaneValue->getOperand(0).getReg()); + ComputeLoop = Expand64BitScalarArithmetic(*NewAccumulator, ComputeLoop); + break; + } + } + } // Manipulate the iterator to get the next active lane unsigned BITSETOpc = IsWave32 ? AMDGPU::S_BITSET0_B32 : AMDGPU::S_BITSET0_B64; - auto NewActiveBits = - BuildMI(*ComputeLoop, I, DL, TII->get(BITSETOpc), NewActiveBitsReg) - .addReg(FF1->getOperand(0).getReg()) - .addReg(ActiveBits->getOperand(0).getReg()); + BuildMI(*ComputeLoop, I, DL, TII->get(BITSETOpc), NewActiveBitsReg) + .addReg(FF1Reg) + .addReg(ActiveBitsReg); // Add phi nodes - Accumulator.addReg(NewAccumulator->getOperand(0).getReg()) - .addMBB(ComputeLoop); - ActiveBits.addReg(NewActiveBits->getOperand(0).getReg()) - .addMBB(ComputeLoop); + Accumulator.addReg(DstReg).addMBB(ComputeLoop); + ActiveBits.addReg(NewActiveBitsReg).addMBB(ComputeLoop); // Creating branching unsigned CMPOpc = IsWave32 ? AMDGPU::S_CMP_LG_U32 : AMDGPU::S_CMP_LG_U64; BuildMI(*ComputeLoop, I, DL, TII->get(CMPOpc)) - .addReg(NewActiveBits->getOperand(0).getReg()) + .addReg(NewActiveBitsReg) .addImm(0); BuildMI(*ComputeLoop, I, DL, TII->get(AMDGPU::S_CBRANCH_SCC1)) .addMBB(ComputeLoop); @@ -5488,22 +5782,40 @@ SITargetLowering::EmitInstrWithCustomInserter(MachineInstr &MI, switch (MI.getOpcode()) { case AMDGPU::WAVE_REDUCE_UMIN_PSEUDO_U32: return lowerWaveReduce(MI, *BB, *getSubtarget(), AMDGPU::S_MIN_U32); + case AMDGPU::WAVE_REDUCE_UMIN_PSEUDO_U64: + return lowerWaveReduce(MI, *BB, *getSubtarget(), AMDGPU::V_CMP_LT_U64_e64); case AMDGPU::WAVE_REDUCE_MIN_PSEUDO_I32: return lowerWaveReduce(MI, *BB, *getSubtarget(), AMDGPU::S_MIN_I32); + case AMDGPU::WAVE_REDUCE_MIN_PSEUDO_I64: + return lowerWaveReduce(MI, *BB, *getSubtarget(), AMDGPU::V_CMP_LT_I64_e64); case AMDGPU::WAVE_REDUCE_UMAX_PSEUDO_U32: return lowerWaveReduce(MI, *BB, *getSubtarget(), AMDGPU::S_MAX_U32); + case AMDGPU::WAVE_REDUCE_UMAX_PSEUDO_U64: + return lowerWaveReduce(MI, *BB, *getSubtarget(), AMDGPU::V_CMP_GT_U64_e64); case AMDGPU::WAVE_REDUCE_MAX_PSEUDO_I32: return lowerWaveReduce(MI, *BB, *getSubtarget(), AMDGPU::S_MAX_I32); + case AMDGPU::WAVE_REDUCE_MAX_PSEUDO_I64: + return lowerWaveReduce(MI, *BB, *getSubtarget(), AMDGPU::V_CMP_GT_I64_e64); case AMDGPU::WAVE_REDUCE_ADD_PSEUDO_I32: return lowerWaveReduce(MI, *BB, *getSubtarget(), AMDGPU::S_ADD_I32); + case AMDGPU::WAVE_REDUCE_ADD_PSEUDO_U64: + return lowerWaveReduce(MI, *BB, *getSubtarget(), AMDGPU::S_ADD_U64_PSEUDO); case AMDGPU::WAVE_REDUCE_SUB_PSEUDO_I32: return lowerWaveReduce(MI, *BB, *getSubtarget(), AMDGPU::S_SUB_I32); + case AMDGPU::WAVE_REDUCE_SUB_PSEUDO_U64: + return lowerWaveReduce(MI, *BB, *getSubtarget(), AMDGPU::S_SUB_U64_PSEUDO); case AMDGPU::WAVE_REDUCE_AND_PSEUDO_B32: return lowerWaveReduce(MI, *BB, *getSubtarget(), AMDGPU::S_AND_B32); + case AMDGPU::WAVE_REDUCE_AND_PSEUDO_B64: + return lowerWaveReduce(MI, *BB, *getSubtarget(), AMDGPU::S_AND_B64); case AMDGPU::WAVE_REDUCE_OR_PSEUDO_B32: return lowerWaveReduce(MI, *BB, *getSubtarget(), AMDGPU::S_OR_B32); + case AMDGPU::WAVE_REDUCE_OR_PSEUDO_B64: + return lowerWaveReduce(MI, *BB, *getSubtarget(), AMDGPU::S_OR_B64); case AMDGPU::WAVE_REDUCE_XOR_PSEUDO_B32: return lowerWaveReduce(MI, *BB, *getSubtarget(), AMDGPU::S_XOR_B32); + case AMDGPU::WAVE_REDUCE_XOR_PSEUDO_B64: + return lowerWaveReduce(MI, *BB, *getSubtarget(), AMDGPU::S_XOR_B64); case AMDGPU::S_UADDO_PSEUDO: case AMDGPU::S_USUBO_PSEUDO: { const DebugLoc &DL = MI.getDebugLoc(); @@ -5530,55 +5842,7 @@ SITargetLowering::EmitInstrWithCustomInserter(MachineInstr &MI, } case AMDGPU::S_ADD_U64_PSEUDO: case AMDGPU::S_SUB_U64_PSEUDO: { - // For targets older than GFX12, we emit a sequence of 32-bit operations. - // For GFX12, we emit s_add_u64 and s_sub_u64. - const GCNSubtarget &ST = MF->getSubtarget<GCNSubtarget>(); - MachineRegisterInfo &MRI = BB->getParent()->getRegInfo(); - const DebugLoc &DL = MI.getDebugLoc(); - MachineOperand &Dest = MI.getOperand(0); - MachineOperand &Src0 = MI.getOperand(1); - MachineOperand &Src1 = MI.getOperand(2); - bool IsAdd = (MI.getOpcode() == AMDGPU::S_ADD_U64_PSEUDO); - if (Subtarget->hasScalarAddSub64()) { - unsigned Opc = IsAdd ? AMDGPU::S_ADD_U64 : AMDGPU::S_SUB_U64; - // clang-format off - BuildMI(*BB, MI, DL, TII->get(Opc), Dest.getReg()) - .add(Src0) - .add(Src1); - // clang-format on - } else { - const SIRegisterInfo *TRI = ST.getRegisterInfo(); - const TargetRegisterClass *BoolRC = TRI->getBoolRC(); - - Register DestSub0 = MRI.createVirtualRegister(&AMDGPU::SReg_32RegClass); - Register DestSub1 = MRI.createVirtualRegister(&AMDGPU::SReg_32RegClass); - - MachineOperand Src0Sub0 = TII->buildExtractSubRegOrImm( - MI, MRI, Src0, BoolRC, AMDGPU::sub0, &AMDGPU::SReg_32RegClass); - MachineOperand Src0Sub1 = TII->buildExtractSubRegOrImm( - MI, MRI, Src0, BoolRC, AMDGPU::sub1, &AMDGPU::SReg_32RegClass); - - MachineOperand Src1Sub0 = TII->buildExtractSubRegOrImm( - MI, MRI, Src1, BoolRC, AMDGPU::sub0, &AMDGPU::SReg_32RegClass); - MachineOperand Src1Sub1 = TII->buildExtractSubRegOrImm( - MI, MRI, Src1, BoolRC, AMDGPU::sub1, &AMDGPU::SReg_32RegClass); - - unsigned LoOpc = IsAdd ? AMDGPU::S_ADD_U32 : AMDGPU::S_SUB_U32; - unsigned HiOpc = IsAdd ? AMDGPU::S_ADDC_U32 : AMDGPU::S_SUBB_U32; - BuildMI(*BB, MI, DL, TII->get(LoOpc), DestSub0) - .add(Src0Sub0) - .add(Src1Sub0); - BuildMI(*BB, MI, DL, TII->get(HiOpc), DestSub1) - .add(Src0Sub1) - .add(Src1Sub1); - BuildMI(*BB, MI, DL, TII->get(TargetOpcode::REG_SEQUENCE), Dest.getReg()) - .addReg(DestSub0) - .addImm(AMDGPU::sub0) - .addReg(DestSub1) - .addImm(AMDGPU::sub1); - } - MI.eraseFromParent(); - return BB; + return Expand64BitScalarArithmetic(MI, BB); } case AMDGPU::V_ADD_U64_PSEUDO: case AMDGPU::V_SUB_U64_PSEUDO: { @@ -6272,6 +6536,20 @@ SDValue SITargetLowering::splitUnaryVectorOp(SDValue Op, return DAG.getNode(ISD::CONCAT_VECTORS, SDLoc(Op), VT, OpLo, OpHi); } +// Enable lowering of ROTR for vxi32 types. This is a workaround for a +// regression whereby extra unnecessary instructions were added to codegen +// for rotr operations, casued by legalising v2i32 or. This resulted in extra +// instructions to extract the result from the vector. +SDValue SITargetLowering::lowerROTR(SDValue Op, SelectionDAG &DAG) const { + [[maybe_unused]] EVT VT = Op.getValueType(); + + assert((VT == MVT::v2i32 || VT == MVT::v4i32 || VT == MVT::v8i32 || + VT == MVT::v16i32) && + "Unexpected ValueType."); + + return DAG.UnrollVectorOp(Op.getNode()); +} + // Work around LegalizeDAG doing the wrong thing and fully scalarizing if the // wider vector type is legal. SDValue SITargetLowering::splitBinaryVectorOp(SDValue Op, @@ -6463,6 +6741,8 @@ SDValue SITargetLowering::LowerOperation(SDValue Op, SelectionDAG &DAG) const { return lowerGET_FPENV(Op, DAG); case ISD::SET_FPENV: return lowerSET_FPENV(Op, DAG); + case ISD::ROTR: + return lowerROTR(Op, DAG); } return SDValue(); } @@ -13545,6 +13825,47 @@ SDValue SITargetLowering::performOrCombine(SDNode *N, } } + // Detect identity v2i32 OR and replace with identity source node. + // Specifically an Or that has operands constructed from the same source node + // via extract_vector_elt and build_vector. I.E. + // v2i32 or( + // v2i32 build_vector( + // i32 extract_elt(%IdentitySrc, 0), + // i32 0 + // ), + // v2i32 build_vector( + // i32 0, + // i32 extract_elt(%IdentitySrc, 1) + // ) ) + // => + // v2i32 %IdentitySrc + + if (VT == MVT::v2i32 && LHS->getOpcode() == ISD::BUILD_VECTOR && + RHS->getOpcode() == ISD::BUILD_VECTOR) { + + ConstantSDNode *LC = dyn_cast<ConstantSDNode>(LHS->getOperand(1)); + ConstantSDNode *RC = dyn_cast<ConstantSDNode>(RHS->getOperand(0)); + + // Test for and normalise build vectors. + if (LC && RC && LC->getZExtValue() == 0 && RC->getZExtValue() == 0) { + + // Get the extract_vector_element operands. + SDValue LEVE = LHS->getOperand(0); + SDValue REVE = RHS->getOperand(1); + + if (LEVE->getOpcode() == ISD::EXTRACT_VECTOR_ELT && + REVE->getOpcode() == ISD::EXTRACT_VECTOR_ELT) { + // Check that different elements from the same vector are + // extracted. + if (LEVE->getOperand(0) == REVE->getOperand(0) && + LEVE->getOperand(1) != REVE->getOperand(1)) { + SDValue IdentitySrc = LEVE.getOperand(0); + return IdentitySrc; + } + } + } + } + if (VT != MVT::i64 || DCI.isBeforeLegalizeOps()) return SDValue(); @@ -13592,7 +13913,7 @@ SDValue SITargetLowering::performXorCombine(SDNode *N, SDValue LHS = N->getOperand(0); SDValue RHS = N->getOperand(1); - const ConstantSDNode *CRHS = dyn_cast<ConstantSDNode>(RHS); + const ConstantSDNode *CRHS = isConstOrConstSplat(RHS); SelectionDAG &DAG = DCI.DAG; EVT VT = N->getValueType(0); @@ -13602,6 +13923,23 @@ SDValue SITargetLowering::performXorCombine(SDNode *N, return Split; } + // v2i32 (xor (vselect cc, x, y), K) -> + // (v2i32 svelect cc, (xor x, K), (xor y, K)) This enables the xor to be + // replaced with source modifiers when the select is lowered to CNDMASK. + unsigned Opc = LHS.getOpcode(); + if (((Opc == ISD::VSELECT && VT == MVT::v2i32) || + (Opc == ISD::SELECT && VT == MVT::i64)) && + CRHS && CRHS->getAPIntValue().isSignMask()) { + SDValue CC = LHS->getOperand(0); + SDValue TRUE = LHS->getOperand(1); + SDValue FALSE = LHS->getOperand(2); + SDValue XTrue = DAG.getNode(ISD::XOR, SDLoc(N), VT, TRUE, RHS); + SDValue XFalse = DAG.getNode(ISD::XOR, SDLoc(N), VT, FALSE, RHS); + SDValue XSelect = + DAG.getNode(ISD::VSELECT, SDLoc(N), VT, CC, XTrue, XFalse); + return XSelect; + } + // Make sure to apply the 64-bit constant splitting fold before trying to fold // fneg-like xors into 64-bit select. if (LHS.getOpcode() == ISD::SELECT && VT == MVT::i32) { @@ -14592,6 +14930,27 @@ SITargetLowering::performExtractVectorEltCombine(SDNode *N, return DAG.getNode(Vec.getOpcode(), SL, ResVT, Elt); } + // (extract_vector_element (and {y0, y1}, (build_vector 0x1f, 0x1f)), index) + // -> (and (extract_vector_element {y0, y1}, index), 0x1f) + // There are optimisations to transform 64-bit shifts into 32-bit shifts + // depending on the shift operand. See e.g. performSraCombine(). + // This combine ensures that the optimisation is compatible with v2i32 + // legalised AND. + if (VecVT == MVT::v2i32 && Vec->getOpcode() == ISD::AND && + Vec->getOperand(1)->getOpcode() == ISD::BUILD_VECTOR) { + + const ConstantSDNode *C = isConstOrConstSplat(Vec.getOperand(1)); + if (!C || C->getZExtValue() != 0x1f) + return SDValue(); + + SDLoc SL(N); + SDValue AndMask = DAG.getConstant(0x1f, SL, MVT::i32); + SDValue EVE = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, SL, MVT::i32, + Vec->getOperand(0), N->getOperand(1)); + SDValue A = DAG.getNode(ISD::AND, SL, MVT::i32, EVE, AndMask); + DAG.ReplaceAllUsesWith(N, A.getNode()); + } + // ScalarRes = EXTRACT_VECTOR_ELT ((vector-BINOP Vec1, Vec2), Idx) // => // Vec1Elt = EXTRACT_VECTOR_ELT(Vec1, Idx) diff --git a/llvm/lib/Target/AMDGPU/SIISelLowering.h b/llvm/lib/Target/AMDGPU/SIISelLowering.h index 728c6490bdfd..4886fcf9fd01 100644 --- a/llvm/lib/Target/AMDGPU/SIISelLowering.h +++ b/llvm/lib/Target/AMDGPU/SIISelLowering.h @@ -444,6 +444,7 @@ public: SDValue lowerFP_EXTEND(SDValue Op, SelectionDAG &DAG) const; SDValue lowerGET_FPENV(SDValue Op, SelectionDAG &DAG) const; SDValue lowerSET_FPENV(SDValue Op, SelectionDAG &DAG) const; + SDValue lowerROTR(SDValue Op, SelectionDAG &DAG) const; Register getRegisterByName(const char* RegName, LLT VT, const MachineFunction &MF) const override; diff --git a/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp b/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp index 5c958dfe6954..23a124fecdda 100644 --- a/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp +++ b/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp @@ -5975,12 +5975,10 @@ SIInstrInfo::getWholeWaveFunctionSetup(MachineFunction &MF) const { static const TargetRegisterClass * adjustAllocatableRegClass(const GCNSubtarget &ST, const SIRegisterInfo &RI, - const MCInstrDesc &TID, unsigned RCID, - bool IsAllocatable) { - if ((IsAllocatable || !ST.hasGFX90AInsts()) && - (((TID.mayLoad() || TID.mayStore()) && - !(TID.TSFlags & SIInstrFlags::Spill)) || - (TID.TSFlags & SIInstrFlags::MIMG))) { + const MCInstrDesc &TID, unsigned RCID) { + if (!ST.hasGFX90AInsts() && (((TID.mayLoad() || TID.mayStore()) && + !(TID.TSFlags & SIInstrFlags::Spill)) || + (TID.TSFlags & SIInstrFlags::MIMG))) { switch (RCID) { case AMDGPU::AV_32RegClassID: RCID = AMDGPU::VGPR_32RegClassID; @@ -6020,7 +6018,7 @@ const TargetRegisterClass *SIInstrInfo::getRegClass(const MCInstrDesc &TID, return RI.getRegClass(RegClass); } - return adjustAllocatableRegClass(ST, RI, TID, RegClass, false); + return adjustAllocatableRegClass(ST, RI, TID, RegClass); } const TargetRegisterClass *SIInstrInfo::getOpRegClass(const MachineInstr &MI, @@ -6039,7 +6037,7 @@ const TargetRegisterClass *SIInstrInfo::getOpRegClass(const MachineInstr &MI, } unsigned RCID = Desc.operands()[OpNo].RegClass; - return adjustAllocatableRegClass(ST, RI, Desc, RCID, true); + return adjustAllocatableRegClass(ST, RI, Desc, RCID); } void SIInstrInfo::legalizeOpWithMove(MachineInstr &MI, unsigned OpIdx) const { @@ -6127,12 +6125,10 @@ bool SIInstrInfo::isLegalRegOperand(const MachineRegisterInfo &MRI, const TargetRegisterClass *SuperRC = RI.getLargestLegalSuperClass(RC, *MF); if (!SuperRC) return false; - - DRC = RI.getMatchingSuperRegClass(SuperRC, DRC, MO.getSubReg()); - if (!DRC) - return false; + return RI.getMatchingSuperRegClass(SuperRC, DRC, MO.getSubReg()) != nullptr; } - return RC->hasSuperClassEq(DRC); + + return RI.getCommonSubClass(DRC, RC) != nullptr; } bool SIInstrInfo::isLegalRegOperand(const MachineInstr &MI, unsigned OpIdx, @@ -10281,7 +10277,7 @@ unsigned SIInstrInfo::getInstrLatency(const InstrItineraryData *ItinData, InstructionUniformity SIInstrInfo::getGenericInstructionUniformity(const MachineInstr &MI) const { const MachineRegisterInfo &MRI = MI.getMF()->getRegInfo(); - unsigned opcode = MI.getOpcode(); + unsigned Opcode = MI.getOpcode(); auto HandleAddrSpaceCast = [this, &MRI](const MachineInstr &MI) { Register Dst = MI.getOperand(0).getReg(); @@ -10301,7 +10297,7 @@ SIInstrInfo::getGenericInstructionUniformity(const MachineInstr &MI) const { // If the target supports globally addressable scratch, the mapping from // scratch memory to the flat aperture changes therefore an address space cast // is no longer uniform. - if (opcode == TargetOpcode::G_ADDRSPACE_CAST) + if (Opcode == TargetOpcode::G_ADDRSPACE_CAST) return HandleAddrSpaceCast(MI); if (auto *GI = dyn_cast<GIntrinsic>(&MI)) { @@ -10329,7 +10325,8 @@ SIInstrInfo::getGenericInstructionUniformity(const MachineInstr &MI) const { // // All other loads are not divergent, because if threads issue loads with the // same arguments, they will always get the same result. - if (opcode == AMDGPU::G_LOAD) { + if (Opcode == AMDGPU::G_LOAD || Opcode == AMDGPU::G_ZEXTLOAD || + Opcode == AMDGPU::G_SEXTLOAD) { if (MI.memoperands_empty()) return InstructionUniformity::NeverUniform; // conservative assumption @@ -10343,10 +10340,10 @@ SIInstrInfo::getGenericInstructionUniformity(const MachineInstr &MI) const { return InstructionUniformity::Default; } - if (SIInstrInfo::isGenericAtomicRMWOpcode(opcode) || - opcode == AMDGPU::G_ATOMIC_CMPXCHG || - opcode == AMDGPU::G_ATOMIC_CMPXCHG_WITH_SUCCESS || - AMDGPU::isGenericAtomic(opcode)) { + if (SIInstrInfo::isGenericAtomicRMWOpcode(Opcode) || + Opcode == AMDGPU::G_ATOMIC_CMPXCHG || + Opcode == AMDGPU::G_ATOMIC_CMPXCHG_WITH_SUCCESS || + AMDGPU::isGenericAtomic(Opcode)) { return InstructionUniformity::NeverUniform; } return InstructionUniformity::Default; diff --git a/llvm/lib/Target/AMDGPU/SIInstrInfo.h b/llvm/lib/Target/AMDGPU/SIInstrInfo.h index c964d02ee2b9..f7dde2b90b68 100644 --- a/llvm/lib/Target/AMDGPU/SIInstrInfo.h +++ b/llvm/lib/Target/AMDGPU/SIInstrInfo.h @@ -1055,6 +1055,8 @@ public: return AMDGPU::S_WAIT_DSCNT; case AMDGPU::S_WAIT_KMCNT_soft: return AMDGPU::S_WAIT_KMCNT; + case AMDGPU::S_WAIT_XCNT_soft: + return AMDGPU::S_WAIT_XCNT; default: return Opcode; } diff --git a/llvm/lib/Target/AMDGPU/SIInstrInfo.td b/llvm/lib/Target/AMDGPU/SIInstrInfo.td index 562023cde44a..c8231b470aba 100644 --- a/llvm/lib/Target/AMDGPU/SIInstrInfo.td +++ b/llvm/lib/Target/AMDGPU/SIInstrInfo.td @@ -1898,7 +1898,7 @@ class getVregSrcForVT<ValueType VT, bit IsTrue16 = 0, bit IsFake16 = 1> { !eq(VT.Size, 64) : RegisterOperand<VReg_64>, !eq(VT.Size, 48) : RegisterOperand<VReg_64>, !eq(VT.Size, 16) : !if(IsTrue16, - !if(IsFake16, VGPRSrc_32_Lo128, VGPRSrc_16_Lo128), + !if(IsFake16, VGPROp_32_Lo128, VGPROp_16_Lo128), RegisterOperand<VGPR_32>), 1 : RegisterOperand<VGPR_32>); } @@ -1950,6 +1950,20 @@ class getVOP3VRegSrcForVT<ValueType VT> { 1 : VRegSrc_32); } +// VGPR only VOP3 src with 8 bit encoding e.g. VOP3DPP src0. +class getVGPRSrcForVT<ValueType VT, bit IsTrue16 = 0, bit IsFake16 = 0> { + RegisterOperand ret = + !cond(!eq(VT.Size, 128) : VGPROp_128, + !eq(VT.Size, 96) : VGPROp_96, + !eq(VT.Size, 64) : VGPROp_64, + !eq(VT.Size, 48) : VGPROp_64, + !eq(VT.Size, 16) : !if(IsTrue16, + !if(IsFake16, VGPROp_32, + VGPROp_16), + VGPROp_32), + 1 : VGPROp_32); +} + // Src2 of VOP3 DPP instructions cannot be a literal class getVOP3DPPSrcForVT<ValueType VT, bit IsFake16 = 1> { RegisterOperand ret = @@ -2582,34 +2596,42 @@ class getAlign2RegOp<RegisterOperand RC> { RegisterOperand ret = !cond(!eq(RC, VGPROp_16) : VGPROp_16, !eq(RC, VGPROp_32) : VGPROp_32, + !eq(RC, VGPROp_64) : VGPROp_64_Align2, !eq(RC, VGPROp_64_Align1) : VGPROp_64_Align2, + !eq(RC, VGPROp_64_Align2) : VGPROp_64_Align2, + !eq(RC, VGPROp_96) : VGPROp_96_Align2, !eq(RC, VGPROp_96_Align1) : VGPROp_96_Align2, + !eq(RC, VGPROp_96_Align2) : VGPROp_96_Align2, + !eq(RC, VGPROp_128) : VGPROp_128_Align2, !eq(RC, VGPROp_128_Align1) : VGPROp_128_Align2, + !eq(RC, VGPROp_128_Align2) : VGPROp_128_Align2, + !eq(RC, VGPROp_160) : VGPROp_160_Align2, !eq(RC, VGPROp_160_Align1) : VGPROp_160_Align2, + !eq(RC, VGPROp_160_Align2) : VGPROp_160_Align2, + !eq(RC, VGPROp_1024) : VGPROp_1024_Align2, !eq(RC, VGPROp_1024_Align1) : VGPROp_1024_Align2, + !eq(RC, VGPROp_1024_Align2) : VGPROp_1024_Align2, + !eq(RC, AVLdSt_32) : AVLdSt_32, + !eq(RC, AVLdSt_64_Align1) : AVLdSt_64_Align2, !eq(RC, AVLdSt_64) : AVLdSt_64_Align2, + !eq(RC, AVLdSt_96) : AVLdSt_96_Align2, + !eq(RC, AVLdSt_96_Align1) : AVLdSt_96_Align1, !eq(RC, AVLdSt_96_Align1) : AVLdSt_96_Align2, + !eq(RC, AVLdSt_128) : AVLdSt_128_Align2, !eq(RC, AVLdSt_128_Align1) : AVLdSt_128_Align2, - !eq(RC, AVLdSt_160) : AVLdSt_160_Align2, - !eq(RC, AVLdSt_160_Align1) : AVLdSt_160_Align2); -} + !eq(RC, AVLdSt_128_Align2) : AVLdSt_128_Align2, -class getEquivalentAGPRClass<RegisterClass RC> { - RegisterClass ret = - !cond(!eq(RC.Size, 32) : AGPR_32, - !eq(RC.Size, 64) : AReg_64, - !eq(RC.Size, 96) : AReg_96, - !eq(RC.Size, 128) : AReg_128, - !eq(RC.Size, 160) : AReg_160, - !eq(RC.Size, 1024) : AReg_1024); + !eq(RC, AVLdSt_160) : AVLdSt_160_Align2, + !eq(RC, AVLdSt_160_Align1) : AVLdSt_160_Align2, + !eq(RC, AVLdSt_160_Align2) : AVLdSt_160_Align2); } class getEquivalentAGPROperand<RegisterOperand RC> { @@ -2681,7 +2703,7 @@ class VOPProfile <list<ValueType> _ArgVT, bit _EnableClamp = 0> { field RegisterOperand Src0DPP = getVregSrcForVT<Src0VT>.ret; field RegisterOperand Src1DPP = getVregSrcForVT<Src1VT>.ret; field RegisterOperand Src2DPP = getVregSrcForVT<Src2VT>.ret; - field RegisterOperand Src0VOP3DPP = VGPRSrc_32; + field RegisterOperand Src0VOP3DPP = getVGPRSrcForVT<Src0VT>.ret; field RegisterOperand Src1VOP3DPP = getVOP3DPPSrcForVT<Src1VT>.ret; field RegisterOperand Src2VOP3DPP = getVOP3DPPSrcForVT<Src2VT>.ret; field RegisterOperand Src0SDWA = getSDWASrcForVT<Src0VT>.ret; @@ -2897,7 +2919,7 @@ class VOPProfile_True16<VOPProfile P> : VOPProfile<P.ArgVT> { let Src0ModDPP = getSrcModDPP_t16<Src0VT, 0 /*IsFake16*/>.ret; let Src1ModDPP = getSrcModDPP_t16<Src1VT, 0 /*IsFake16*/>.ret; let Src2ModDPP = getSrcModDPP_t16<Src2VT, 0 /*IsFake16*/>.ret; - let Src0VOP3DPP = !if (!eq(Src0VT.Size, 16), VGPRSrc_16, VGPRSrc_32); + let Src0VOP3DPP = !if (!eq(Src0VT.Size, 16), VGPROp_16, VGPROp_32); let Src1VOP3DPP = getVOP3DPPSrcForVT<Src1VT, 0 /*IsFake16*/>.ret; let Src2VOP3DPP = getVOP3DPPSrcForVT<Src2VT, 0 /*IsFake16*/>.ret; let Src0ModVOP3DPP = getSrc0ModVOP3DPP<Src0VT, DstVT, 0/*IsFake16*/>.ret; diff --git a/llvm/lib/Target/AMDGPU/SIInstructions.td b/llvm/lib/Target/AMDGPU/SIInstructions.td index b6e697d34c3d..d4c1bc6d8438 100644 --- a/llvm/lib/Target/AMDGPU/SIInstructions.td +++ b/llvm/lib/Target/AMDGPU/SIInstructions.td @@ -326,28 +326,57 @@ def : GCNPat<(i32 (int_amdgcn_set_inactive_chain_arg i32:$src, i32:$inactive)), (V_SET_INACTIVE_B32 0, VGPR_32:$src, 0, VGPR_32:$inactive, (IMPLICIT_DEF))>; // clang-format off -defvar int_amdgcn_wave_reduce_ = "int_amdgcn_wave_reduce_"; + multiclass - AMDGPUWaveReducePseudoGenerator<string Op, string DataType> { + AMDGPUWaveReducePseudoGenerator<string Op, string DataType, ValueType ty, RegisterClass RetReg, SrcRegOrImm9 Reg> { let usesCustomInserter = 1, hasSideEffects = 0, mayLoad = 0, mayStore = 0, Uses = [EXEC] in { def !toupper(Op) #"_PSEUDO_" #DataType - : VPseudoInstSI<(outs SGPR_32 : $sdst), - (ins VSrc_b32 : $src, VSrc_b32 : $strategy), - [(set i32 : $sdst, (!cast<AMDGPUWaveReduce>(int_amdgcn_wave_reduce_ #Op) i32 : $src, i32 : $strategy))]> {} + : VPseudoInstSI<(outs RetReg : $sdst), + (ins Reg : $src, VSrc_b32 : $strategy), + [(set ty : $sdst, (!cast<AMDGPUWaveReduce>("int_amdgcn_wave_reduce_" #Op) ty : $src, i32 : $strategy))]> {} } } // clang-format on +class WaveReduceOp<string OpName, string TypeStr, ValueType Ty, + RegisterClass ReturnRegisterClass, SrcRegOrImm9 RC> { + string Name = OpName; + string TypeString = TypeStr; + ValueType VT = Ty; + RegisterClass RetReg = ReturnRegisterClass; + SrcRegOrImm9 Reg = RC; +} + // Input list : [Operation_name, -// type - Signed(I)/Unsigned(U)/Float(F)/Bitwise(B)] +// type - Signed(I)/Unsigned(U)/Float(F)/Bitwise(B), +// bit-width +// output register class, +// input register class] defvar Operations = [ - ["umin", "U32"], ["min", "I32"], ["umax", "U32"], ["max", "I32"], - ["add", "I32"], ["sub", "I32"], ["and", "B32"], ["or", "B32"], - ["xor", "B32"] + WaveReduceOp<"umin", "U32", i32, SGPR_32, VSrc_b32>, + WaveReduceOp<"min", "I32", i32, SGPR_32, VSrc_b32>, + WaveReduceOp<"umax", "U32", i32, SGPR_32, VSrc_b32>, + WaveReduceOp<"max", "I32", i32, SGPR_32, VSrc_b32>, + WaveReduceOp<"add", "I32", i32, SGPR_32, VSrc_b32>, + WaveReduceOp<"sub", "I32", i32, SGPR_32, VSrc_b32>, + WaveReduceOp<"and", "B32", i32, SGPR_32, VSrc_b32>, + WaveReduceOp<"or", "B32", i32, SGPR_32, VSrc_b32>, + WaveReduceOp<"xor", "B32", i32, SGPR_32, VSrc_b32>, + + WaveReduceOp<"umin", "U64", i64, SGPR_64, VSrc_b64>, + WaveReduceOp<"min", "I64", i64, SGPR_64, VSrc_b64>, + WaveReduceOp<"umax", "U64", i64, SGPR_64, VSrc_b64>, + WaveReduceOp<"max", "I64", i64, SGPR_64, VSrc_b64>, + WaveReduceOp<"add", "U64", i64, SGPR_64, VSrc_b64>, + WaveReduceOp<"sub", "U64", i64, SGPR_64, VSrc_b64>, + WaveReduceOp<"and", "B64", i64, SGPR_64, VSrc_b64>, + WaveReduceOp<"or", "B64", i64, SGPR_64, VSrc_b64>, + WaveReduceOp<"xor", "B64", i64, SGPR_64, VSrc_b64>, ]; foreach Op = Operations in { - defm WAVE_REDUCE_ : AMDGPUWaveReducePseudoGenerator<Op[0], Op[1]>; + defm WAVE_REDUCE_ : AMDGPUWaveReducePseudoGenerator<Op.Name, Op.TypeString, + Op.VT, Op.RetReg, Op.Reg>; } let usesCustomInserter = 1, Defs = [VCC] in { @@ -2201,6 +2230,8 @@ def : GCNPat < } foreach fp16vt = [f16, bf16] in { +foreach p = [NotHasTrue16BitInsts, UseFakeTrue16Insts] in +let True16Predicate = p in { def : GCNPat < (fcopysign fp16vt:$src0, fp16vt:$src1), (V_BFI_B32_e64 (S_MOV_B32 (i32 0x00007fff)), $src0, $src1) @@ -2231,6 +2262,42 @@ def : GCNPat < (V_BFI_B32_e64 (S_MOV_B32 (i32 0x00007fff)), $src0, (V_LSHRREV_B32_e64 (i32 16), (EXTRACT_SUBREG $src1, sub1))) >; +} +let True16Predicate = UseRealTrue16Insts in { +def : GCNPat < + (fcopysign fp16vt:$src0, fp16vt:$src1), + (EXTRACT_SUBREG (V_BFI_B32_e64 (S_MOV_B32 (i32 0x00007fff)), + (REG_SEQUENCE VGPR_32, $src0, lo16, (i16 (IMPLICIT_DEF)), hi16), + (REG_SEQUENCE VGPR_32, $src1, lo16, (i16 (IMPLICIT_DEF)), hi16)), lo16) +>; + +def : GCNPat < + (fcopysign f32:$src0, fp16vt:$src1), + (V_BFI_B32_e64 (S_MOV_B32 (i32 0x7fffffff)), $src0, + (REG_SEQUENCE VGPR_32, (i16 (IMPLICIT_DEF)), lo16, $src1, hi16)) +>; + +def : GCNPat < + (fcopysign f64:$src0, fp16vt:$src1), + (REG_SEQUENCE VReg_64, + (i32 (EXTRACT_SUBREG $src0, sub0)), sub0, + (V_BFI_B32_e64 (S_MOV_B32 (i32 0x7fffffff)), (i32 (EXTRACT_SUBREG $src0, sub1)), + (REG_SEQUENCE VGPR_32, (i16 (IMPLICIT_DEF)), lo16, $src1, hi16)), sub1) +>; + +def : GCNPat < + (fcopysign fp16vt:$src0, f32:$src1), + (EXTRACT_SUBREG (V_BFI_B32_e64 (S_MOV_B32 (i32 0x7fff0000)), + (REG_SEQUENCE VGPR_32, (i16 (IMPLICIT_DEF)), lo16, $src0, hi16), $src1), hi16) +>; + +def : GCNPat < + (fcopysign fp16vt:$src0, f64:$src1), + (V_BFI_B32_e64 (S_MOV_B32 (i32 0x00007fff)), + (REG_SEQUENCE VGPR_32, $src0, lo16, (i16 (IMPLICIT_DEF)), hi16), + (V_LSHRREV_B32_e64 (i32 16), (EXTRACT_SUBREG $src1, sub1))) +>; +} } // End foreach fp16vt = [f16, bf16] @@ -2547,9 +2614,9 @@ def : AMDGPUPatIgnoreCopies < (COPY_TO_REGCLASS VSrc_b32:$z, VGPR_32)) >; -// 64-bit version +foreach vt = [i64, v2i32] in { def : AMDGPUPatIgnoreCopies < - (DivergentBinFrag<xor> i64:$z, (and i64:$x, (xor i64:$y, i64:$z))), + (DivergentBinFrag<xor> vt:$z, (and vt:$x, (xor vt:$y, vt:$z))), (REG_SEQUENCE VReg_64, (V_BFI_B32_e64 (i32 (EXTRACT_SUBREG VReg_64:$x, sub0)), (i32 (EXTRACT_SUBREG VReg_64:$y, sub0)), @@ -2558,6 +2625,7 @@ def : AMDGPUPatIgnoreCopies < (i32 (EXTRACT_SUBREG VReg_64:$y, sub1)), (i32 (EXTRACT_SUBREG VReg_64:$z, sub1))), sub1) >; +} def : AMDGPUPat < (fcopysign f32:$src0, f32:$src1), @@ -3154,6 +3222,11 @@ def : GCNPat< (i32 (zext (i16 (bitconvert fp16_zeros_high_16bits:$src)))), (COPY VSrc_b16:$src) >; + +def : GCNPat < + (i1 (DivergentUnaryFrag<trunc> i16:$a)), + (V_CMP_EQ_U32_e64 (V_AND_B32_e64 (i32 1), $a), (i32 1)) +>; } let True16Predicate = UseRealTrue16Insts in { @@ -3171,6 +3244,11 @@ def : GCNPat< (i32 (zext (i16 (bitconvert fp16_zeros_high_16bits:$src)))), (REG_SEQUENCE VGPR_32, VGPR_16:$src, lo16, (V_MOV_B16_t16_e64 0, (i16 0), 0), hi16) >; + +def : GCNPat < + (i1 (DivergentUnaryFrag<trunc> i16:$a)), + (V_CMP_EQ_U16_t16_e64 (i32 0), (V_AND_B16_t16_e64 (i32 0), (i16 1), (i32 0), $a), (i32 0), (i16 1), (i32 0)) +>; } def : GCNPat < @@ -3199,11 +3277,6 @@ def : GCNPat < (V_CMP_EQ_U32_e64 (V_AND_B32_e64 (i32 1), $a), (i32 1)) >; -def : GCNPat < - (i1 (DivergentUnaryFrag<trunc> i16:$a)), - (V_CMP_EQ_U32_e64 (V_AND_B32_e64 (i32 1), $a), (i32 1)) ->; - def IMMBitSelConst : SDNodeXForm<imm, [{ return CurDAG->getTargetConstant(1ULL << N->getZExtValue(), SDLoc(N), MVT::i32); @@ -3807,7 +3880,8 @@ def : GCNPat < (v2i16 (S_PACK_HL_B32_B16 SReg_32:$src0, SReg_32:$src1)) >; - +foreach p = [NotHasTrue16BitInsts, UseFakeTrue16Insts] in +let True16Predicate = p in { def : GCNPat < (v2f16 (scalar_to_vector f16:$src0)), (COPY $src0) @@ -3827,6 +3901,29 @@ def : GCNPat < (v4f16 (scalar_to_vector f16:$src0)), (INSERT_SUBREG (IMPLICIT_DEF), $src0, sub0) >; +} + +let True16Predicate = UseRealTrue16Insts in { +def : GCNPat < + (v2f16 (scalar_to_vector f16:$src0)), + (REG_SEQUENCE VGPR_32, $src0, lo16, (i16 (IMPLICIT_DEF)), hi16) +>; + +def : GCNPat < + (v2i16 (scalar_to_vector i16:$src0)), + (REG_SEQUENCE VGPR_32, $src0, lo16, (i16 (IMPLICIT_DEF)), hi16) +>; + +def : GCNPat < + (v4i16 (scalar_to_vector i16:$src0)), + (REG_SEQUENCE VGPR_32, $src0, lo16, (i16 (IMPLICIT_DEF)), hi16, (i32 (IMPLICIT_DEF)), sub1) +>; + +def : GCNPat < + (v4f16 (scalar_to_vector f16:$src0)), + (REG_SEQUENCE VGPR_32, $src0, lo16, (i16 (IMPLICIT_DEF)), hi16, (i32 (IMPLICIT_DEF)), sub1) +>; +} def : GCNPat < (i64 (int_amdgcn_mov_dpp i64:$src, timm:$dpp_ctrl, timm:$row_mask, @@ -4306,6 +4403,21 @@ def G_AMDGPU_BUFFER_LOAD_FORMAT_D16 : BufferLoadGenericInstruction; def G_AMDGPU_TBUFFER_LOAD_FORMAT : TBufferLoadGenericInstruction; def G_AMDGPU_TBUFFER_LOAD_FORMAT_D16 : TBufferLoadGenericInstruction; +class D16LoadGenericInstruction : AMDGPUGenericInstruction { + let OutOperandList = (outs type0:$dst); + let InOperandList = (ins ptype1:$addr, type0:$src); + let hasSideEffects = 0; + let mayLoad = 1; +} + +def G_AMDGPU_LOAD_D16_LO : D16LoadGenericInstruction; +def G_AMDGPU_LOAD_D16_LO_U8 : D16LoadGenericInstruction; +def G_AMDGPU_LOAD_D16_LO_I8 : D16LoadGenericInstruction; +def G_AMDGPU_LOAD_D16_HI : D16LoadGenericInstruction; +def G_AMDGPU_LOAD_D16_HI_U8 : D16LoadGenericInstruction; +def G_AMDGPU_LOAD_D16_HI_I8 : D16LoadGenericInstruction; + + class BufferStoreGenericInstruction : AMDGPUGenericInstruction { let OutOperandList = (outs); let InOperandList = (ins type0:$vdata, type1:$rsrc, type2:$vindex, type2:$voffset, diff --git a/llvm/lib/Target/AMDGPU/SIMachineFunctionInfo.cpp b/llvm/lib/Target/AMDGPU/SIMachineFunctionInfo.cpp index 8a1120321af9..54426d33d347 100644 --- a/llvm/lib/Target/AMDGPU/SIMachineFunctionInfo.cpp +++ b/llvm/lib/Target/AMDGPU/SIMachineFunctionInfo.cpp @@ -86,7 +86,7 @@ SIMachineFunctionInfo::SIMachineFunctionInfo(const Function &F, // FIXME: MayNeedAGPRs is a misnomer for how this is used. MFMA selection // should be separated from availability of AGPRs if (MFMAVGPRForm || - (ST.getMaxNumVGPRs(F) <= AMDGPU::VGPR_32RegClass.getNumRegs() && + (ST.getMaxNumVGPRs(F) <= ST.getAddressableNumArchVGPRs() && !mayUseAGPRs(F))) MayNeedAGPRs = false; // We will select all MAI with VGPR operands. } diff --git a/llvm/lib/Target/AMDGPU/SIMemoryLegalizer.cpp b/llvm/lib/Target/AMDGPU/SIMemoryLegalizer.cpp index 6ab4eb4bde97..1637c06936f9 100644 --- a/llvm/lib/Target/AMDGPU/SIMemoryLegalizer.cpp +++ b/llvm/lib/Target/AMDGPU/SIMemoryLegalizer.cpp @@ -63,6 +63,7 @@ enum class SIAtomicScope { SINGLETHREAD, WAVEFRONT, WORKGROUP, + CLUSTER, // Promoted to AGENT on targets without workgroup clusters. AGENT, SYSTEM }; @@ -106,6 +107,7 @@ private: bool IsCooperative = false; SIMemOpInfo( + const GCNSubtarget &ST, AtomicOrdering Ordering = AtomicOrdering::SequentiallyConsistent, SIAtomicScope Scope = SIAtomicScope::SYSTEM, SIAtomicAddrSpace OrderingAddrSpace = SIAtomicAddrSpace::ATOMIC, @@ -156,6 +158,11 @@ private: SIAtomicAddrSpace::GDS)) == SIAtomicAddrSpace::NONE) { this->Scope = std::min(Scope, SIAtomicScope::AGENT); } + + // On targets that have no concept of a workgroup cluster, use + // AGENT scope as a conservatively correct alternative. + if (this->Scope == SIAtomicScope::CLUSTER && !ST.hasClusters()) + this->Scope = SIAtomicScope::AGENT; } public: @@ -225,6 +232,7 @@ public: class SIMemOpAccess final { private: const AMDGPUMachineModuleInfo *MMI = nullptr; + const GCNSubtarget &ST; /// Reports unsupported message \p Msg for \p MI to LLVM context. void reportUnsupported(const MachineBasicBlock::iterator &MI, @@ -248,7 +256,7 @@ private: public: /// Construct class to support accessing the machine memory operands /// of instructions in the machine function \p MF. - SIMemOpAccess(const AMDGPUMachineModuleInfo &MMI); + SIMemOpAccess(const AMDGPUMachineModuleInfo &MMI, const GCNSubtarget &ST); /// \returns Load info if \p MI is a load operation, "std::nullopt" otherwise. std::optional<SIMemOpInfo> @@ -606,7 +614,11 @@ protected: SIAtomicScope Scope, SIAtomicAddrSpace AddrSpace) const; public: - SIGfx12CacheControl(const GCNSubtarget &ST) : SIGfx11CacheControl(ST) {} + SIGfx12CacheControl(const GCNSubtarget &ST) : SIGfx11CacheControl(ST) { + // GFX12.0 and GFX12.5 memory models greatly overlap, and in some cases + // the behavior is the same if assuming GFX12.0 in CU mode. + assert(!ST.hasGFX1250Insts() || ST.isCuModeEnabled()); + } bool insertWait(MachineBasicBlock::iterator &MI, SIAtomicScope Scope, SIAtomicAddrSpace AddrSpace, SIMemOp Op, @@ -769,6 +781,8 @@ SIMemOpAccess::toSIAtomicScope(SyncScope::ID SSID, return std::tuple(SIAtomicScope::SYSTEM, SIAtomicAddrSpace::ATOMIC, true); if (SSID == MMI->getAgentSSID()) return std::tuple(SIAtomicScope::AGENT, SIAtomicAddrSpace::ATOMIC, true); + if (SSID == MMI->getClusterSSID()) + return std::tuple(SIAtomicScope::CLUSTER, SIAtomicAddrSpace::ATOMIC, true); if (SSID == MMI->getWorkgroupSSID()) return std::tuple(SIAtomicScope::WORKGROUP, SIAtomicAddrSpace::ATOMIC, true); @@ -784,6 +798,9 @@ SIMemOpAccess::toSIAtomicScope(SyncScope::ID SSID, if (SSID == MMI->getAgentOneAddressSpaceSSID()) return std::tuple(SIAtomicScope::AGENT, SIAtomicAddrSpace::ATOMIC & InstrAddrSpace, false); + if (SSID == MMI->getClusterOneAddressSpaceSSID()) + return std::tuple(SIAtomicScope::CLUSTER, + SIAtomicAddrSpace::ATOMIC & InstrAddrSpace, false); if (SSID == MMI->getWorkgroupOneAddressSpaceSSID()) return std::tuple(SIAtomicScope::WORKGROUP, SIAtomicAddrSpace::ATOMIC & InstrAddrSpace, false); @@ -811,8 +828,9 @@ SIAtomicAddrSpace SIMemOpAccess::toSIAtomicAddrSpace(unsigned AS) const { return SIAtomicAddrSpace::OTHER; } -SIMemOpAccess::SIMemOpAccess(const AMDGPUMachineModuleInfo &MMI_) - : MMI(&MMI_) {} +SIMemOpAccess::SIMemOpAccess(const AMDGPUMachineModuleInfo &MMI_, + const GCNSubtarget &ST) + : MMI(&MMI_), ST(ST) {} std::optional<SIMemOpInfo> SIMemOpAccess::constructFromMIWithMMO( const MachineBasicBlock::iterator &MI) const { @@ -873,7 +891,7 @@ std::optional<SIMemOpInfo> SIMemOpAccess::constructFromMIWithMMO( return std::nullopt; } } - return SIMemOpInfo(Ordering, Scope, OrderingAddrSpace, InstrAddrSpace, + return SIMemOpInfo(ST, Ordering, Scope, OrderingAddrSpace, InstrAddrSpace, IsCrossAddressSpaceOrdering, FailureOrdering, IsVolatile, IsNonTemporal, IsLastUse, IsCooperative); } @@ -887,7 +905,7 @@ SIMemOpAccess::getLoadInfo(const MachineBasicBlock::iterator &MI) const { // Be conservative if there are no memory operands. if (MI->getNumMemOperands() == 0) - return SIMemOpInfo(); + return SIMemOpInfo(ST); return constructFromMIWithMMO(MI); } @@ -901,7 +919,7 @@ SIMemOpAccess::getStoreInfo(const MachineBasicBlock::iterator &MI) const { // Be conservative if there are no memory operands. if (MI->getNumMemOperands() == 0) - return SIMemOpInfo(); + return SIMemOpInfo(ST); return constructFromMIWithMMO(MI); } @@ -942,8 +960,9 @@ SIMemOpAccess::getAtomicFenceInfo(const MachineBasicBlock::iterator &MI) const { if (SynchronizeAS) OrderingAddrSpace = *SynchronizeAS; - return SIMemOpInfo(Ordering, Scope, OrderingAddrSpace, SIAtomicAddrSpace::ATOMIC, - IsCrossAddressSpaceOrdering, AtomicOrdering::NotAtomic); + return SIMemOpInfo(ST, Ordering, Scope, OrderingAddrSpace, + SIAtomicAddrSpace::ATOMIC, IsCrossAddressSpaceOrdering, + AtomicOrdering::NotAtomic); } std::optional<SIMemOpInfo> SIMemOpAccess::getAtomicCmpxchgOrRmwInfo( @@ -955,7 +974,7 @@ std::optional<SIMemOpInfo> SIMemOpAccess::getAtomicCmpxchgOrRmwInfo( // Be conservative if there are no memory operands. if (MI->getNumMemOperands() == 0) - return SIMemOpInfo(); + return SIMemOpInfo(ST); return constructFromMIWithMMO(MI); } @@ -2198,7 +2217,8 @@ bool SIGfx10CacheControl::insertBarrierStart( // mode. This is because a CU mode release fence does not emit any wait, which // is fine when only dealing with vmem, but isn't sufficient in the presence // of barriers which do not go through vmem. - if (!ST.isCuModeEnabled()) + // GFX12.5 does not require this additional wait. + if (!ST.isCuModeEnabled() || ST.hasGFX1250Insts()) return false; BuildMI(*MI->getParent(), MI, MI->getDebugLoc(), @@ -2372,18 +2392,23 @@ bool SIGfx12CacheControl::insertWait(MachineBasicBlock::iterator &MI, switch (Scope) { case SIAtomicScope::SYSTEM: case SIAtomicScope::AGENT: + case SIAtomicScope::CLUSTER: if ((Op & SIMemOp::LOAD) != SIMemOp::NONE) LOADCnt |= true; if ((Op & SIMemOp::STORE) != SIMemOp::NONE) STORECnt |= true; break; case SIAtomicScope::WORKGROUP: - // In WGP mode the waves of a work-group can be executing on either CU of - // the WGP. Therefore need to wait for operations to complete to ensure - // they are visible to waves in the other CU as the L0 is per CU. - // Otherwise in CU mode and all waves of a work-group are on the same CU - // which shares the same L0. - if (!ST.isCuModeEnabled()) { + // GFX12.0: + // In WGP mode the waves of a work-group can be executing on either CU + // of the WGP. Therefore need to wait for operations to complete to + // ensure they are visible to waves in the other CU as the L0 is per CU. + // Otherwise in CU mode and all waves of a work-group are on the same CU + // which shares the same L0. + // + // GFX12.5: + // TODO DOCS + if (!ST.isCuModeEnabled() || ST.hasGFX1250Insts()) { if ((Op & SIMemOp::LOAD) != SIMemOp::NONE) LOADCnt |= true; if ((Op & SIMemOp::STORE) != SIMemOp::NONE) @@ -2404,6 +2429,7 @@ bool SIGfx12CacheControl::insertWait(MachineBasicBlock::iterator &MI, switch (Scope) { case SIAtomicScope::SYSTEM: case SIAtomicScope::AGENT: + case SIAtomicScope::CLUSTER: case SIAtomicScope::WORKGROUP: // If no cross address space ordering then an "S_WAITCNT lgkmcnt(0)" is // not needed as LDS operations for all waves are executed in a total @@ -2435,7 +2461,7 @@ bool SIGfx12CacheControl::insertWait(MachineBasicBlock::iterator &MI, // // This also applies to fences. Fences cannot pair with an instruction // tracked with bvh/samplecnt as we don't have any atomics that do that. - if (Order != AtomicOrdering::Acquire) { + if (Order != AtomicOrdering::Acquire && ST.hasImageInsts()) { BuildMI(MBB, MI, DL, TII->get(AMDGPU::S_WAIT_BVHCNT_soft)).addImm(0); BuildMI(MBB, MI, DL, TII->get(AMDGPU::S_WAIT_SAMPLECNT_soft)).addImm(0); } @@ -2486,11 +2512,18 @@ bool SIGfx12CacheControl::insertAcquire(MachineBasicBlock::iterator &MI, case SIAtomicScope::AGENT: ScopeImm = AMDGPU::CPol::SCOPE_DEV; break; + case SIAtomicScope::CLUSTER: + ScopeImm = AMDGPU::CPol::SCOPE_SE; + break; case SIAtomicScope::WORKGROUP: - // In WGP mode the waves of a work-group can be executing on either CU of - // the WGP. Therefore we need to invalidate the L0 which is per CU. - // Otherwise in CU mode all waves of a work-group are on the same CU, and so - // the L0 does not need to be invalidated. + // GFX12.0: + // In WGP mode the waves of a work-group can be executing on either CU of + // the WGP. Therefore we need to invalidate the L0 which is per CU. + // Otherwise in CU mode all waves of a work-group are on the same CU, and + // so the L0 does not need to be invalidated. + // + // GFX12.5 + // TODO DOCS if (ST.isCuModeEnabled()) return false; @@ -2535,7 +2568,8 @@ bool SIGfx12CacheControl::insertRelease(MachineBasicBlock::iterator &MI, if (Pos == Position::AFTER) ++MI; - // global_wb is only necessary at system scope for gfx120x targets. + // global_wb is only necessary at system scope for GFX12.0, + // they're also necessary at device scope for GFX12.5. // // Emitting it for lower scopes is a slow no-op, so we omit it // for performance. @@ -2545,6 +2579,13 @@ bool SIGfx12CacheControl::insertRelease(MachineBasicBlock::iterator &MI, .addImm(AMDGPU::CPol::SCOPE_SYS); break; case SIAtomicScope::AGENT: + // TODO DOCS + if (ST.hasGFX1250Insts()) { + BuildMI(MBB, MI, DL, TII->get(AMDGPU::GLOBAL_WB)) + .addImm(AMDGPU::CPol::SCOPE_DEV); + } + break; + case SIAtomicScope::CLUSTER: case SIAtomicScope::WORKGROUP: // No WB necessary, but we still have to wait. break; @@ -2607,27 +2648,32 @@ bool SIGfx12CacheControl::enableVolatileAndOrNonTemporal( } bool SIGfx12CacheControl::finalizeStore(MachineInstr &MI, bool Atomic) const { - MachineOperand *CPol = TII->getNamedOperand(MI, OpName::cpol); - if (!CPol) - return false; + assert(MI.mayStore() && "Not a Store inst"); + const bool IsRMW = (MI.mayLoad() && MI.mayStore()); + bool Changed = false; + + // GFX12.5 only: xcnt wait is needed before flat and global atomics + // stores/rmw. + if (Atomic && ST.requiresWaitXCntBeforeAtomicStores() && TII->isFLAT(MI)) { + MachineBasicBlock &MBB = *MI.getParent(); + BuildMI(MBB, MI, MI.getDebugLoc(), TII->get(S_WAIT_XCNT_soft)).addImm(0); + Changed = true; + } + + // Remaining fixes do not apply to RMWs. + if (IsRMW) + return Changed; + MachineOperand *CPol = TII->getNamedOperand(MI, OpName::cpol); + if (!CPol) // Some vmem operations do not have a scope and are not concerned. + return Changed; const unsigned Scope = CPol->getImm() & CPol::SCOPE; // GFX12.0 only: Extra waits needed before system scope stores. - if (!ST.hasGFX1250Insts()) { - if (!Atomic && Scope == CPol::SCOPE_SYS) - return insertWaitsBeforeSystemScopeStore(MI); - return false; - } + if (!ST.hasGFX1250Insts() && !Atomic && Scope == CPol::SCOPE_SYS) + Changed |= insertWaitsBeforeSystemScopeStore(MI.getIterator()); - // GFX12.5 only: Require SCOPE_SE on stores that may hit the scratch address - // space. - // We also require SCOPE_SE minimum if we not have the "cu-stores" feature. - if (Scope == CPol::SCOPE_CU && - (!ST.hasCUStores() || TII->mayAccessScratchThroughFlat(MI))) - return setScope(MI, CPol::SCOPE_SE); - - return false; + return Changed; } bool SIGfx12CacheControl::handleCooperativeAtomic(MachineInstr &MI) const { @@ -2656,6 +2702,9 @@ bool SIGfx12CacheControl::setAtomicScope(const MachineBasicBlock::iterator &MI, case SIAtomicScope::AGENT: Changed |= setScope(MI, AMDGPU::CPol::SCOPE_DEV); break; + case SIAtomicScope::CLUSTER: + Changed |= setScope(MI, AMDGPU::CPol::SCOPE_SE); + break; case SIAtomicScope::WORKGROUP: // In workgroup mode, SCOPE_SE is needed as waves can executes on // different CUs that access different L0s. @@ -2839,6 +2888,7 @@ bool SIMemoryLegalizer::expandAtomicCmpxchgOrRmw(const SIMemOpInfo &MOI, assert(MI->mayLoad() && MI->mayStore()); bool Changed = false; + MachineInstr &RMWMI = *MI; if (MOI.isAtomic()) { const AtomicOrdering Order = MOI.getOrdering(); @@ -2873,6 +2923,7 @@ bool SIMemoryLegalizer::expandAtomicCmpxchgOrRmw(const SIMemOpInfo &MOI, Position::AFTER); } + Changed |= CC->finalizeStore(RMWMI, /*Atomic=*/true); return Changed; } @@ -2900,8 +2951,8 @@ SIMemoryLegalizerPass::run(MachineFunction &MF, bool SIMemoryLegalizer::run(MachineFunction &MF) { bool Changed = false; - SIMemOpAccess MOA(MMI.getObjFileInfo<AMDGPUMachineModuleInfo>()); const GCNSubtarget &ST = MF.getSubtarget<GCNSubtarget>(); + SIMemOpAccess MOA(MMI.getObjFileInfo<AMDGPUMachineModuleInfo>(), ST); CC = SICacheControl::create(ST); for (auto &MBB : MF) { diff --git a/llvm/lib/Target/AMDGPU/SIRegisterInfo.td b/llvm/lib/Target/AMDGPU/SIRegisterInfo.td index 162ef647be7e..31dd6b9e8d84 100644 --- a/llvm/lib/Target/AMDGPU/SIRegisterInfo.td +++ b/llvm/lib/Target/AMDGPU/SIRegisterInfo.td @@ -1307,51 +1307,41 @@ def VRegSrc_fake16: SrcReg9<VGPR_32> { let EncoderMethod = "getMachineOpValueT16"; } //===----------------------------------------------------------------------===// -// VGPRSrc_* +// VGPROp_* An 8-bit RegisterOperand wrapper for a VGPR //===----------------------------------------------------------------------===// -// An 8-bit RegisterOperand wrapper for a VGPR -def VGPRSrc_32 : RegisterOperand<VGPR_32> { - let DecoderMethod = "DecodeVGPR_32RegisterClass"; +class VGPROp<RegisterClass regClass> : RegisterOperand<regClass> { + let DecoderMethod = "Decode" # regClass # "RegisterClass"; } -def VGPRSrc_32_Lo128 : RegisterOperand<VGPR_32_Lo128> { - let DecoderMethod = "DecodeVGPR_32RegisterClass"; +class VGPROp_Align2<RegisterClass regClass> : RegisterOperand<!cast<RegisterClass>(regClass#_Align2)> { + let DecoderMethod = "Decode" # regClass # "RegisterClass"; } -def VGPRSrc_64 : RegisterOperand<VReg_64> { - let DecoderMethod = "DecodeVReg_64RegisterClass"; +multiclass VGPROp_Aligned<RegisterClass regClass> { + def _Align1 : VGPROp<regClass>; + def _Align2 : VGPROp_Align2<regClass>; } -def VGPRSrc_96 : RegisterOperand<VReg_96> { - let DecoderMethod = "DecodeVReg_96RegisterClass"; +// TODO: These cases should use default target alignment +def VGPROp_16 : VGPROp<VGPR_16> { + let EncoderMethod = "getMachineOpValueT16"; } +def VGPROp_32 : VGPROp<VGPR_32>; -def VGPRSrc_128 : RegisterOperand<VReg_128> { - let DecoderMethod = "DecodeVReg_128RegisterClass"; +foreach size = ["64", "96", "128", "160", "192", "224", "256", "288", "320", "352", "384", "512", "1024"] in { + def VGPROp_#size : VGPROp<!cast<RegisterClass>("VReg_"#size)>; } -def VGPRSrc_192 : RegisterOperand<VReg_192> { - let DecoderMethod = "DecodeVReg_192RegisterClass"; +foreach size = ["64", "96", "128", "160", "256", "1024"] in { + defm VGPROp_#size : VGPROp_Aligned<!cast<RegisterClass>("VReg_"#size)>; } -def VGPRSrc_16_Lo128 : RegisterOperand<VGPR_16_Lo128> { +def VGPROp_16_Lo128 : RegisterOperand<VGPR_16_Lo128> { let DecoderMethod = "DecodeVGPR_16_Lo128RegisterClass"; let EncoderMethod = "getMachineOpValueT16Lo128"; } -// True 16 operands. -def VGPRSrc_16 : RegisterOperand<VGPR_16> { - let DecoderMethod = "DecodeVGPR_16RegisterClass"; - let EncoderMethod = "getMachineOpValueT16"; -} - -// TODO: These cases should use default target alignment -def VGPROp_16 : RegisterOperand<VGPR_16>; -def VGPROp_32 : RegisterOperand<VGPR_32>; - -foreach size = ["64", "96", "128", "160", "256", "1024" ] in { - def VGPROp_#size : RegisterOperand<!cast<RegisterClass>("VReg_"#size)>; - def VGPROp_#size#_Align1 : RegisterOperand<!cast<RegisterClass>("VReg_"#size)>; - def VGPROp_#size#_Align2 : RegisterOperand<!cast<RegisterClass>("VReg_"#size#_Align2)>; +def VGPROp_32_Lo128 : RegisterOperand<VGPR_32_Lo128> { + let DecoderMethod = "DecodeVGPR_32RegisterClass"; } //===----------------------------------------------------------------------===// diff --git a/llvm/lib/Target/AMDGPU/SOPInstructions.td b/llvm/lib/Target/AMDGPU/SOPInstructions.td index dbe0b8c496fe..fe94887cdff9 100644 --- a/llvm/lib/Target/AMDGPU/SOPInstructions.td +++ b/llvm/lib/Target/AMDGPU/SOPInstructions.td @@ -1653,6 +1653,11 @@ let OtherPredicates = [HasImageInsts] in { def S_WAIT_KMCNT_soft : SOPP_Pseudo <"s_soft_wait_kmcnt", (ins s16imm:$simm16), "$simm16">; } + +let SubtargetPredicate = HasWaitXcnt in { + def S_WAIT_XCNT_soft : SOPP_Pseudo<"", (ins s16imm:$simm16), "$simm16">; +} + // Represents the point at which a wave must wait for all outstanding direct loads to LDS. // Typically inserted by the memory legalizer and consumed by SIInsertWaitcnts. @@ -1882,6 +1887,21 @@ def : GCNPat < (S_MOV_B32 imm:$imm) >; +def : GCNPat < + (v2i32 (UniformBinFrag<and> v2i32:$x, v2i32:$y)), + (S_AND_B64 SReg_64:$x, SReg_64:$y) +>; + +def : GCNPat < + (v2i32 (UniformBinFrag<or> v2i32:$x, v2i32:$y)), + (S_OR_B64 SReg_64:$x, SReg_64:$y) +>; + +def : GCNPat < + (v2i32 (UniformBinFrag<xor> v2i32:$x, v2i32:$y)), + (S_XOR_B64 SReg_64:$x, SReg_64:$y) +>; + // Same as a 32-bit inreg def : GCNPat< (i32 (UniformUnaryFrag<sext> i16:$src)), diff --git a/llvm/lib/Target/AMDGPU/Utils/AMDGPUBaseInfo.cpp b/llvm/lib/Target/AMDGPU/Utils/AMDGPUBaseInfo.cpp index 9f4f42185d9a..40da4f96aefd 100644 --- a/llvm/lib/Target/AMDGPU/Utils/AMDGPUBaseInfo.cpp +++ b/llvm/lib/Target/AMDGPU/Utils/AMDGPUBaseInfo.cpp @@ -1399,13 +1399,16 @@ unsigned getTotalNumVGPRs(const MCSubtargetInfo *STI) { return IsWave32 ? 1024 : 512; } -unsigned getAddressableNumArchVGPRs(const MCSubtargetInfo *STI) { return 256; } +unsigned getAddressableNumArchVGPRs(const MCSubtargetInfo *STI) { + const auto &Features = STI->getFeatureBits(); + if (Features.test(Feature1024AddressableVGPRs)) + return Features.test(FeatureWavefrontSize32) ? 1024 : 512; + return 256; +} unsigned getAddressableNumVGPRs(const MCSubtargetInfo *STI, unsigned DynamicVGPRBlockSize) { const auto &Features = STI->getFeatureBits(); - if (Features.test(FeatureGFX1250Insts)) - return Features.test(FeatureWavefrontSize32) ? 1024 : 512; if (Features.test(FeatureGFX90AInsts)) return 512; diff --git a/llvm/lib/Target/AMDGPU/VOP2Instructions.td b/llvm/lib/Target/AMDGPU/VOP2Instructions.td index 4f6b7c592339..46a1a4bf1ab4 100644 --- a/llvm/lib/Target/AMDGPU/VOP2Instructions.td +++ b/llvm/lib/Target/AMDGPU/VOP2Instructions.td @@ -416,12 +416,12 @@ def VOP_MADAK_F16_t16 : VOP_MADAK <f16> { let IsTrue16 = 1; let IsRealTrue16 = 1; let DstRC = getVALUDstForVT<DstVT, 1/*IsTrue16*/, 0/*IsVOP3Encoding*/>.ret; - let Ins32 = (ins VSrcT_f16_Lo128:$src0, VGPRSrc_16_Lo128:$src1, ImmOpType:$imm); + let Ins32 = (ins VSrcT_f16_Lo128:$src0, VGPROp_16_Lo128:$src1, ImmOpType:$imm); } def VOP_MADAK_F16_fake16 : VOP_MADAK <f16> { let IsTrue16 = 1; let DstRC = getVALUDstForVT_fake16<DstVT>.ret; - let Ins32 = (ins VSrcFake16_f16_Lo128:$src0, VGPRSrc_32_Lo128:$src1, ImmOpType:$imm); + let Ins32 = (ins VSrcFake16_f16_Lo128:$src0, VGPROp_32_Lo128:$src1, ImmOpType:$imm); } def VOP_MADAK_F32 : VOP_MADAK <f32>; def VOP_MADAK_F64 : VOP_MADAK <f64>; @@ -452,12 +452,12 @@ def VOP_MADMK_F16_t16 : VOP_MADMK <f16> { let IsTrue16 = 1; let IsRealTrue16 = 1; let DstRC = getVALUDstForVT<DstVT, 1/*IsTrue16*/, 0/*IsVOP3Encoding*/>.ret; - let Ins32 = (ins VSrcT_f16_Lo128:$src0, ImmOpType:$imm, VGPRSrc_16_Lo128:$src1); + let Ins32 = (ins VSrcT_f16_Lo128:$src0, ImmOpType:$imm, VGPROp_16_Lo128:$src1); } def VOP_MADMK_F16_fake16 : VOP_MADMK <f16> { let IsTrue16 = 1; let DstRC = getVALUDstForVT_fake16<DstVT>.ret; - let Ins32 = (ins VSrcFake16_f16_Lo128:$src0, ImmOpType:$imm, VGPRSrc_32_Lo128:$src1); + let Ins32 = (ins VSrcFake16_f16_Lo128:$src0, ImmOpType:$imm, VGPROp_32_Lo128:$src1); } def VOP_MADMK_F32 : VOP_MADMK <f32>; def VOP_MADMK_F64 : VOP_MADMK <f64>; @@ -496,14 +496,14 @@ class VOP_MAC <ValueType vt0, ValueType vt1=vt0> : VOPProfile <[vt0, vt1, vt1, v HasClamp, HasModifiers, HasModifiers, HasOMod, Src0ModVOP3DPP, Src1ModVOP3DPP, Src2Mod, HasOpSel>.ret; // We need a dummy src2 tied to dst to track the use of that register for s_delay_alu - let InsVOPDX = (ins Src0RC32:$src0X, Src1RC32:$vsrc1X, VGPRSrc_32:$src2X); - let InsVOPDY = (ins Src0RC32:$src0Y, Src1RC32:$vsrc1Y, VGPRSrc_32:$src2Y); + let InsVOPDX = (ins Src0RC32:$src0X, Src1RC32:$vsrc1X, VGPROp_32:$src2X); + let InsVOPDY = (ins Src0RC32:$src0Y, Src1RC32:$vsrc1Y, VGPROp_32:$src2Y); let InsVOPD3X = (ins Src0ModVOPD3:$src0X_modifiers, Src0VOPD3:$src0X, Src1ModVOPD3:$vsrc1X_modifiers, Src1RC32:$vsrc1X, - VGPRSrc_32:$src2X); + VGPROp_32:$src2X); let InsVOPD3Y = (ins Src0ModVOPD3:$src0Y_modifiers, Src0VOPD3:$src0Y, Src1ModVOPD3:$vsrc1Y_modifiers, Src1RC32:$vsrc1Y, - VGPRSrc_32:$src2Y); + VGPROp_32:$src2Y); let InsDPP8 = (ins Src0ModDPP:$src0_modifiers, Src0DPP:$src0, Src1ModDPP:$src1_modifiers, Src1DPP:$src1, @@ -565,7 +565,7 @@ def VOP_MAC_F16_t16 : VOP_MAC <f16> { let DstRC64 = getVALUDstForVT<DstVT, 1/*IsTrue*/, 1/*IsVOP3Encoding*/>.ret; let Src0RC64 = getVOP3SrcForVT<Src0VT, 1/*IsTrue16*/>.ret; let Src1RC64 = getVOP3SrcForVT<Src1VT, 1/*IsTrue16*/>.ret; - let Src0VOP3DPP = VGPRSrc_16; + let Src0VOP3DPP = VGPROp_16; let Src1VOP3DPP = getVOP3DPPSrcForVT<Src1VT, 0/*IsFake16*/>.ret; let Src2VOP3DPP = getVOP3DPPSrcForVT<Src2VT, 0/*IsFake16*/>.ret; let Src0ModVOP3DPP = getSrc0ModVOP3DPP<Src0VT, DstVT, 0/*IsFake16*/>.ret; @@ -597,7 +597,7 @@ def VOP_MAC_F16_fake16 : VOP_MAC <f16> { getVregSrcForVT<Src2VT, 1/*IsTrue16*/, 1/*IsFake16*/>.ret:$src2, // stub argument dpp8:$dpp8, Dpp8FI:$fi); let DstRC64 = getVALUDstForVT<DstVT>.ret; - let Src0VOP3DPP = VGPRSrc_32; + let Src0VOP3DPP = VGPROp_32; let Src1VOP3DPP = getVOP3DPPSrcForVT<Src1VT, 1/*IsFake16*/>.ret; let Src2VOP3DPP = getVOP3DPPSrcForVT<Src2VT, 1/*IsFake16*/>.ret; let Src0ModVOP3DPP = getSrc0ModVOP3DPP<Src0VT, DstVT, 1/*IsFake16*/>.ret; @@ -796,7 +796,7 @@ def VOP2e_I16_I16_I16_I1_true16 : VOP2e_SGPR<[i16, i16, i16, i1]> { Src2RC64, NumSrcArgs, HasClamp, 1/*HasModifiers*/, 0/*HasSrc2Mods*/, HasOMod, Src0Mod, Src1Mod, Src2Mod, 1/*HasOpSel*/>.ret; - let Src0VOP3DPP = VGPRSrc_16; + let Src0VOP3DPP = VGPROp_16; let Src1VOP3DPP = getVOP3DPPSrcForVT<Src1VT, 0/*IsFake16*/>.ret; let Src0ModVOP3DPP = getSrc0ModVOP3DPP<f16, DstVT, 0/*IsFake16*/>.ret; let Src1ModVOP3DPP = getSrcModVOP3VC<f16, 0/*IsFake16*/>.ret; @@ -808,7 +808,7 @@ def VOP2e_I16_I16_I16_I1_fake16 : VOP2e_SGPR<[i16, i16, i16, i1]> { let Src0Mod = getSrc0Mod<f16, DstVT, 1/*IsTrue16*/, 1/*IsFake16*/>.ret; let Src1Mod = getSrcMod<f16, 1/*IsTrue16*/, 1/*IsFake16*/>.ret; - let Src0VOP3DPP = VGPRSrc_32; + let Src0VOP3DPP = VGPROp_32; let Src1VOP3DPP = getVOP3DPPSrcForVT<Src1VT, 1/*IsFake16*/>.ret; let Src0ModVOP3DPP = getSrc0ModVOP3DPP<f16, DstVT, 1/*IsFake16*/>.ret; let Src1ModVOP3DPP = getSrcModVOP3VC<f16, 1/*IsFake16*/>.ret; @@ -1024,9 +1024,9 @@ def : DivergentClampingBinOp<sub, V_SUB_CO_U32_e64>; def : DivergentBinOp<adde, V_ADDC_U32_e32>; def : DivergentBinOp<sube, V_SUBB_U32_e32>; -class divergent_i64_BinOp <SDPatternOperator Op, Instruction Inst> : +class divergent_i64_BinOp <SDPatternOperator Op, Instruction Inst, ValueType vt = i64> : GCNPat< - (DivergentBinFrag<Op> i64:$src0, i64:$src1), + (DivergentBinFrag<Op> vt:$src0, vt:$src1), (REG_SEQUENCE VReg_64, (Inst (i32 (EXTRACT_SUBREG $src0, sub0)), @@ -1043,6 +1043,10 @@ def : divergent_i64_BinOp <and, V_AND_B32_e64>; def : divergent_i64_BinOp <or, V_OR_B32_e64>; def : divergent_i64_BinOp <xor, V_XOR_B32_e64>; +def : divergent_i64_BinOp <and, V_AND_B32_e64, v2i32>; +def : divergent_i64_BinOp <or, V_OR_B32_e64, v2i32>; +def : divergent_i64_BinOp <xor, V_XOR_B32_e64, v2i32>; + // mul24 w/ 64 bit output. class mul24_64_Pat<SDPatternOperator Op, Instruction InstLo, Instruction InstHi> : GCNPat< (i64 (Op i32:$src0, i32:$src1)), diff --git a/llvm/lib/Target/AMDGPU/VOP3PInstructions.td b/llvm/lib/Target/AMDGPU/VOP3PInstructions.td index cd17382d8d30..f7279b664ed2 100644 --- a/llvm/lib/Target/AMDGPU/VOP3PInstructions.td +++ b/llvm/lib/Target/AMDGPU/VOP3PInstructions.td @@ -44,7 +44,7 @@ class VOP3P_Mix_Profile<VOPProfile P, VOP3Features Features = VOP3_REGULAR, FP16InputMods:$src1_modifiers, Src1RC:$src1, FP16InputMods:$src2_modifiers, Src2RC:$src2); dag dpp_srcs = - (ins FPVRegInputMods:$src0_modifiers, VGPRSrc_32:$src0, + (ins FPVRegInputMods:$src0_modifiers, VGPROp_32:$src0, FPVRegInputMods:$src1_modifiers, VRegSrc_32:$src1, FP16InputMods:$src2_modifiers, Src2RC:$src2); @@ -966,9 +966,14 @@ class MAIInst<string OpName, VOPProfile P, SDPatternOperator node, bit Scaled = class ScaledMAIInst<string OpName, MAIInst BaseInst, SDPatternOperator node> : MAIInst<OpName, BaseInst.Pfl, node, /*Scaled=*/true> { // Append operands from V_MFMA_LD_SCALE_B32, but we need to rename them. + // Restrict to VGPR only (VRegSrc_32) for the scale operands to workaround a + // hardware design defect: For all Inline/SGPR constants, SP HW use bits + // [30:23] as the scale. + // TODO: We may still be able to allow Inline Constants/SGPR, with a proper + // shift, to obtain a potentially better performance. let InOperandList = !con(BaseInst.InOperandList, - (ins VSrc_b32:$scale_src0, - VSrc_b32:$scale_src1, + (ins VRegSrc_32:$scale_src0, + VRegSrc_32:$scale_src1, op_sel0:$src0_modifiers, op_sel_hi0:$src1_modifiers)); let AsmOperands = @@ -1857,8 +1862,8 @@ defm V_WMMA_SCALE_F32_32X16X128_F4_w32 : WMMAInstGFX12<"v_wmma_scale_f32_32x16 defm V_WMMA_SCALE16_F32_32X16X128_F4_w32 : WMMAInstGFX12<"v_wmma_scale16_f32_32x16x128_f4", F32_32X16X128_F4_SCALE16_w32, "_w32">; } // End is_wmma_xdl = 1. -defm V_WMMA_LD_SCALE_PAIRED_B32 : VOP3PInst<"v_wmma_ld_scale_paired_b32", VOP_WMMA_LD_SCALE<i32, VCSrc_b32>>; -defm V_WMMA_LD_SCALE16_PAIRED_B64 : VOP3PInst<"v_wmma_ld_scale16_paired_b64", VOP_WMMA_LD_SCALE<i64, VCSrc_b64>>; +defm V_WMMA_LD_SCALE_PAIRED_B32 : VOP3PInst<"v_wmma_ld_scale_paired_b32", VOP_WMMA_LD_SCALE<i32, VCSrc_b32_Lo256>>; +defm V_WMMA_LD_SCALE16_PAIRED_B64 : VOP3PInst<"v_wmma_ld_scale16_paired_b64", VOP_WMMA_LD_SCALE<i64, VCSrc_b64_Lo256>>; } // End SubtargetPredicate = isGFX125xOnly } // End WaveSizePredicate = isWave32 diff --git a/llvm/lib/Target/AMDGPU/VOPCInstructions.td b/llvm/lib/Target/AMDGPU/VOPCInstructions.td index 450c88b6bf60..2730ec52294e 100644 --- a/llvm/lib/Target/AMDGPU/VOPCInstructions.td +++ b/llvm/lib/Target/AMDGPU/VOPCInstructions.td @@ -100,7 +100,7 @@ multiclass VOPC_Profile_t16<list<SchedReadWrite> sched, ValueType vt0, ValueType let Src0ModDPP = getSrcModDPP_t16<Src0VT, 0/*IsFake16*/>.ret; let Src1ModDPP = getSrcModDPP_t16<Src1VT, 0/*IsFake16*/>.ret; let Src2ModDPP = getSrcModDPP_t16<Src2VT, 0/*IsFake16*/>.ret; - let Src0VOP3DPP = VGPRSrc_16; + let Src0VOP3DPP = VGPROp_16; let Src1VOP3DPP = getVOP3DPPSrcForVT<Src1VT, 0/*IsFake16*/>.ret; let Src2VOP3DPP = getVOP3DPPSrcForVT<Src2VT, 0/*IsFake16*/>.ret; @@ -126,7 +126,7 @@ multiclass VOPC_Profile_t16<list<SchedReadWrite> sched, ValueType vt0, ValueType let Src0ModDPP = getSrcModDPP_t16<Src0VT, 1/*IsFake16*/>.ret; let Src1ModDPP = getSrcModDPP_t16<Src1VT, 1/*IsFake16*/>.ret; let Src2ModDPP = getSrcModDPP_t16<Src2VT, 1/*IsFake16*/>.ret; - let Src0VOP3DPP = VGPRSrc_32; + let Src0VOP3DPP = VGPROp_32; let Src1VOP3DPP = getVOP3DPPSrcForVT<Src1VT, 1/*IsFake16*/>.ret; let Src2VOP3DPP = getVOP3DPPSrcForVT<Src2VT, 1/*IsFake16*/>.ret; @@ -173,7 +173,7 @@ multiclass VOPC_NoSdst_Profile_t16<list<SchedReadWrite> sched, ValueType vt0, Va let Src0ModDPP = getSrcModDPP_t16<Src0VT, 0/*IsFake16*/>.ret; let Src1ModDPP = getSrcModDPP_t16<Src1VT, 0/*IsFake16*/>.ret; let Src2ModDPP = getSrcModDPP_t16<Src2VT, 0/*IsFake16*/>.ret; - let Src0VOP3DPP = VGPRSrc_16; + let Src0VOP3DPP = VGPROp_16; let Src1VOP3DPP = getVOP3DPPSrcForVT<Src1VT, 0/*IsFake16*/>.ret; let Src2VOP3DPP = getVOP3DPPSrcForVT<Src2VT, 0/*IsFake16*/>.ret; @@ -197,7 +197,7 @@ multiclass VOPC_NoSdst_Profile_t16<list<SchedReadWrite> sched, ValueType vt0, Va let Src0ModDPP = getSrcModDPP_t16<Src0VT, 1/*IsFake16*/>.ret; let Src1ModDPP = getSrcModDPP_t16<Src1VT, 1/*IsFake16*/>.ret; let Src2ModDPP = getSrcModDPP_t16<Src2VT, 1/*IsFake16*/>.ret; - let Src0VOP3DPP = VGPRSrc_32; + let Src0VOP3DPP = VGPROp_32; let Src1VOP3DPP = getVOP3DPPSrcForVT<Src1VT, 1/*IsFake16*/>.ret; let Src2VOP3DPP = getVOP3DPPSrcForVT<Src2VT, 1/*IsFake16*/>.ret; @@ -892,7 +892,7 @@ class VOPC_Class_Profile<list<SchedReadWrite> sched, ValueType src0VT, ValueType // DPP8 forbids modifiers and can inherit from VOPC_Profile let Ins64 = (ins Src0Mod:$src0_modifiers, Src0RC64:$src0, Src1RC64:$src1); - dag InsPartVOP3DPP = (ins FPVRegInputMods:$src0_modifiers, VGPRSrc_32:$src0, VCSrc_b32:$src1); + dag InsPartVOP3DPP = (ins FPVRegInputMods:$src0_modifiers, VGPROp_32:$src0, VCSrc_b32:$src1); let InsVOP3Base = !con(InsPartVOP3DPP, !if(HasOpSel, (ins op_sel0:$op_sel), (ins))); let AsmVOP3Base = "$sdst, $src0_modifiers, $src1"; @@ -915,7 +915,7 @@ multiclass VOPC_Class_Profile_t16<list<SchedReadWrite> sched> { let Src0ModDPP = getSrcModDPP_t16<Src0VT, 0/*IsFake16*/>.ret; let Src1ModDPP = getSrcModDPP_t16<Src1VT, 0/*IsFake16*/>.ret; let Src2ModDPP = getSrcModDPP_t16<Src2VT, 0/*IsFake16*/>.ret; - let Src0VOP3DPP = VGPRSrc_16; + let Src0VOP3DPP = VGPROp_16; let Src1VOP3DPP = getVOP3DPPSrcForVT<Src1VT, 0/*IsFake16*/>.ret; let Src2VOP3DPP = getVOP3DPPSrcForVT<Src2VT, 0/*IsFake16*/>.ret; @@ -941,7 +941,7 @@ multiclass VOPC_Class_Profile_t16<list<SchedReadWrite> sched> { let Src0ModDPP = getSrcModDPP_t16<Src0VT, 1/*IsFake16*/>.ret; let Src1ModDPP = getSrcModDPP_t16<Src1VT, 1/*IsFake16*/>.ret; let Src2ModDPP = getSrcModDPP_t16<Src2VT, 1/*IsFake16*/>.ret; - let Src0VOP3DPP = VGPRSrc_32; + let Src0VOP3DPP = VGPROp_32; let Src1VOP3DPP = getVOP3DPPSrcForVT<Src1VT, 1/*IsFake16*/>.ret; let Src2VOP3DPP = getVOP3DPPSrcForVT<Src2VT, 1/*IsFake16*/>.ret; @@ -985,7 +985,7 @@ multiclass VOPC_Class_NoSdst_Profile_t16<list<SchedReadWrite> sched> { let Src0ModDPP = getSrcModDPP_t16<Src0VT, 0/*IsFake16*/>.ret; let Src1ModDPP = getSrcModDPP_t16<Src1VT, 0/*IsFake16*/>.ret; let Src2ModDPP = getSrcModDPP_t16<Src2VT, 0/*IsFake16*/>.ret; - let Src0VOP3DPP = VGPRSrc_16; + let Src0VOP3DPP = VGPROp_16; let Src1VOP3DPP = getVOP3DPPSrcForVT<Src1VT, 0/*IsFake16*/>.ret; let Src2VOP3DPP = getVOP3DPPSrcForVT<Src2VT, 0/*IsFake16*/>.ret; @@ -1009,7 +1009,7 @@ multiclass VOPC_Class_NoSdst_Profile_t16<list<SchedReadWrite> sched> { let Src0ModDPP = getSrcModDPP_t16<Src0VT, 1/*IsFake16*/>.ret; let Src1ModDPP = getSrcModDPP_t16<Src1VT, 1/*IsFake16*/>.ret; let Src2ModDPP = getSrcModDPP_t16<Src2VT, 1/*IsFake16*/>.ret; - let Src0VOP3DPP = VGPRSrc_32; + let Src0VOP3DPP = VGPROp_32; let Src1VOP3DPP = getVOP3DPPSrcForVT<Src1VT, 1/*IsFake16*/>.ret; let Src2VOP3DPP = getVOP3DPPSrcForVT<Src2VT, 1/*IsFake16*/>.ret; diff --git a/llvm/lib/Target/AMDGPU/VOPDInstructions.td b/llvm/lib/Target/AMDGPU/VOPDInstructions.td index 4d97b9c81ea8..f416c0654048 100644 --- a/llvm/lib/Target/AMDGPU/VOPDInstructions.td +++ b/llvm/lib/Target/AMDGPU/VOPDInstructions.td @@ -224,7 +224,7 @@ foreach Gen = [GFX11GenD, GFX12GenD, GFX1250GenD] in { defvar isOpXMADK = !or(!eq(x, "V_FMAAK_F32"), !eq(x, "V_FMAMK_F32")); defvar isOpYMADK = !or(!eq(y, "V_FMAAK_F32"), !eq(y, "V_FMAMK_F32")); defvar OpName = "V_DUAL_" # !substr(x,2) # "_X_" # !substr(y,2) # Gen.Suffix; - defvar outs = (outs VGPRSrc_32:$vdstX, VOPDDstYOperand:$vdstY); + defvar outs = (outs VGPROp_32:$vdstX, VOPDDstYOperand:$vdstY); if !or(isOpXMADK, isOpYMADK) then { // If Both X and Y are MADK, the mandatory literal of X additionally must // use an alternate operand format which defers to the 'real' Y literal. diff --git a/llvm/lib/Target/ARC/ARCTargetMachine.cpp b/llvm/lib/Target/ARC/ARCTargetMachine.cpp index 370336394ba7..8e1944062a2c 100644 --- a/llvm/lib/Target/ARC/ARCTargetMachine.cpp +++ b/llvm/lib/Target/ARC/ARCTargetMachine.cpp @@ -33,12 +33,9 @@ ARCTargetMachine::ARCTargetMachine(const Target &T, const Triple &TT, std::optional<Reloc::Model> RM, std::optional<CodeModel::Model> CM, CodeGenOptLevel OL, bool JIT) - : CodeGenTargetMachineImpl( - T, - "e-m:e-p:32:32-i1:8:32-i8:8:32-i16:16:32-i32:32:32-" - "f32:32:32-i64:32-f64:32-a:0:32-n32", - TT, CPU, FS, Options, getRelocModel(RM), - getEffectiveCodeModel(CM, CodeModel::Small), OL), + : CodeGenTargetMachineImpl(T, TT.computeDataLayout(), TT, CPU, FS, Options, + getRelocModel(RM), + getEffectiveCodeModel(CM, CodeModel::Small), OL), TLOF(std::make_unique<TargetLoweringObjectFileELF>()), Subtarget(TT, std::string(CPU), std::string(FS), *this) { initAsmInfo(); diff --git a/llvm/lib/Target/ARM/ARMISelLowering.cpp b/llvm/lib/Target/ARM/ARMISelLowering.cpp index d4d3c7009527..4af2721562d7 100644 --- a/llvm/lib/Target/ARM/ARMISelLowering.cpp +++ b/llvm/lib/Target/ARM/ARMISelLowering.cpp @@ -20304,9 +20304,11 @@ ARMTargetLowering::getSingleConstraintMatchWeight( static bool isIncompatibleReg(const MCPhysReg &PR, MVT VT) { if (PR == 0 || VT == MVT::Other) return false; - return (ARM::SPRRegClass.contains(PR) && VT != MVT::f32 && VT != MVT::i32) || - (ARM::DPRRegClass.contains(PR) && VT != MVT::f64 && - !VT.is64BitVector()); + if (ARM::SPRRegClass.contains(PR)) + return VT != MVT::f32 && VT != MVT::f16 && VT != MVT::i32; + if (ARM::DPRRegClass.contains(PR)) + return VT != MVT::f64 && !VT.is64BitVector(); + return false; } using RCPair = std::pair<unsigned, const TargetRegisterClass *>; diff --git a/llvm/lib/Target/ARM/ARMTargetMachine.cpp b/llvm/lib/Target/ARM/ARMTargetMachine.cpp index fedf9e2cf34b..346776e0c4b2 100644 --- a/llvm/lib/Target/ARM/ARMTargetMachine.cpp +++ b/llvm/lib/Target/ARM/ARMTargetMachine.cpp @@ -121,62 +121,6 @@ static std::unique_ptr<TargetLoweringObjectFile> createTLOF(const Triple &TT) { return std::make_unique<ARMElfTargetObjectFile>(); } -static std::string computeDataLayout(const Triple &TT, - const TargetOptions &Options, - bool isLittle) { - auto ABI = ARM::computeTargetABI(TT, Options.MCOptions.ABIName); - std::string Ret; - - if (isLittle) - // Little endian. - Ret += "e"; - else - // Big endian. - Ret += "E"; - - Ret += DataLayout::getManglingComponent(TT); - - // Pointers are 32 bits and aligned to 32 bits. - Ret += "-p:32:32"; - - // Function pointers are aligned to 8 bits (because the LSB stores the - // ARM/Thumb state). - Ret += "-Fi8"; - - // ABIs other than APCS have 64 bit integers with natural alignment. - if (ABI != ARM::ARM_ABI_APCS) - Ret += "-i64:64"; - - // We have 64 bits floats. The APCS ABI requires them to be aligned to 32 - // bits, others to 64 bits. We always try to align to 64 bits. - if (ABI == ARM::ARM_ABI_APCS) - Ret += "-f64:32:64"; - - // We have 128 and 64 bit vectors. The APCS ABI aligns them to 32 bits, others - // to 64. We always ty to give them natural alignment. - if (ABI == ARM::ARM_ABI_APCS) - Ret += "-v64:32:64-v128:32:128"; - else if (ABI != ARM::ARM_ABI_AAPCS16) - Ret += "-v128:64:128"; - - // Try to align aggregates to 32 bits (the default is 64 bits, which has no - // particular hardware support on 32-bit ARM). - Ret += "-a:0:32"; - - // Integer registers are 32 bits. - Ret += "-n32"; - - // The stack is 64 bit aligned on AAPCS and 32 bit aligned everywhere else. - if (ABI == ARM::ARM_ABI_AAPCS16) - Ret += "-S128"; - else if (ABI == ARM::ARM_ABI_AAPCS) - Ret += "-S64"; - else - Ret += "-S32"; - - return Ret; -} - static Reloc::Model getEffectiveRelocModel(const Triple &TT, std::optional<Reloc::Model> RM) { if (!RM) @@ -201,12 +145,13 @@ ARMBaseTargetMachine::ARMBaseTargetMachine(const Target &T, const Triple &TT, const TargetOptions &Options, std::optional<Reloc::Model> RM, std::optional<CodeModel::Model> CM, - CodeGenOptLevel OL, bool isLittle) - : CodeGenTargetMachineImpl(T, computeDataLayout(TT, Options, isLittle), TT, - CPU, FS, Options, getEffectiveRelocModel(TT, RM), - getEffectiveCodeModel(CM, CodeModel::Small), OL), + CodeGenOptLevel OL) + : CodeGenTargetMachineImpl( + T, TT.computeDataLayout(Options.MCOptions.ABIName), TT, CPU, FS, + Options, getEffectiveRelocModel(TT, RM), + getEffectiveCodeModel(CM, CodeModel::Small), OL), TargetABI(ARM::computeTargetABI(TT, Options.MCOptions.ABIName)), - TLOF(createTLOF(getTargetTriple())), isLittle(isLittle) { + TLOF(createTLOF(getTargetTriple())), isLittle(TT.isLittleEndian()) { // Default to triple-appropriate float ABI if (Options.FloatABIType == FloatABI::Default) { @@ -334,7 +279,7 @@ ARMLETargetMachine::ARMLETargetMachine(const Target &T, const Triple &TT, std::optional<Reloc::Model> RM, std::optional<CodeModel::Model> CM, CodeGenOptLevel OL, bool JIT) - : ARMBaseTargetMachine(T, TT, CPU, FS, Options, RM, CM, OL, true) {} + : ARMBaseTargetMachine(T, TT, CPU, FS, Options, RM, CM, OL) {} ARMBETargetMachine::ARMBETargetMachine(const Target &T, const Triple &TT, StringRef CPU, StringRef FS, @@ -342,7 +287,7 @@ ARMBETargetMachine::ARMBETargetMachine(const Target &T, const Triple &TT, std::optional<Reloc::Model> RM, std::optional<CodeModel::Model> CM, CodeGenOptLevel OL, bool JIT) - : ARMBaseTargetMachine(T, TT, CPU, FS, Options, RM, CM, OL, false) {} + : ARMBaseTargetMachine(T, TT, CPU, FS, Options, RM, CM, OL) {} namespace { diff --git a/llvm/lib/Target/ARM/ARMTargetMachine.h b/llvm/lib/Target/ARM/ARMTargetMachine.h index 1d73af1da6d0..c417c4c8bae6 100644 --- a/llvm/lib/Target/ARM/ARMTargetMachine.h +++ b/llvm/lib/Target/ARM/ARMTargetMachine.h @@ -42,8 +42,7 @@ public: ARMBaseTargetMachine(const Target &T, const Triple &TT, StringRef CPU, StringRef FS, const TargetOptions &Options, std::optional<Reloc::Model> RM, - std::optional<CodeModel::Model> CM, CodeGenOptLevel OL, - bool isLittle); + std::optional<CodeModel::Model> CM, CodeGenOptLevel OL); ~ARMBaseTargetMachine() override; const ARMSubtarget *getSubtargetImpl(const Function &F) const override; diff --git a/llvm/lib/Target/AVR/AVRTargetMachine.cpp b/llvm/lib/Target/AVR/AVRTargetMachine.cpp index fbd148478c89..f001d7974669 100644 --- a/llvm/lib/Target/AVR/AVRTargetMachine.cpp +++ b/llvm/lib/Target/AVR/AVRTargetMachine.cpp @@ -28,9 +28,6 @@ namespace llvm { -static const char *AVRDataLayout = - "e-P1-p:16:8-i8:8-i16:8-i32:8-i64:8-f32:8-f64:8-n8:16-a:8"; - /// Processes a CPU name. static StringRef getCPU(StringRef CPU) { if (CPU.empty() || CPU == "generic") { @@ -50,8 +47,8 @@ AVRTargetMachine::AVRTargetMachine(const Target &T, const Triple &TT, std::optional<Reloc::Model> RM, std::optional<CodeModel::Model> CM, CodeGenOptLevel OL, bool JIT) - : CodeGenTargetMachineImpl(T, AVRDataLayout, TT, getCPU(CPU), FS, Options, - getEffectiveRelocModel(RM), + : CodeGenTargetMachineImpl(T, TT.computeDataLayout(), TT, getCPU(CPU), FS, + Options, getEffectiveRelocModel(RM), getEffectiveCodeModel(CM, CodeModel::Small), OL), SubTarget(TT, std::string(getCPU(CPU)), std::string(FS), *this) { this->TLOF = std::make_unique<AVRTargetObjectFile>(); diff --git a/llvm/lib/Target/BPF/BPFTargetMachine.cpp b/llvm/lib/Target/BPF/BPFTargetMachine.cpp index 527a48035457..10b758647c73 100644 --- a/llvm/lib/Target/BPF/BPFTargetMachine.cpp +++ b/llvm/lib/Target/BPF/BPFTargetMachine.cpp @@ -59,14 +59,6 @@ extern "C" LLVM_ABI LLVM_EXTERNAL_VISIBILITY void LLVMInitializeBPFTarget() { initializeBPFMIPreEmitCheckingPass(PR); } -// DataLayout: little or big endian -static std::string computeDataLayout(const Triple &TT) { - if (TT.getArch() == Triple::bpfeb) - return "E-m:e-p:64:64-i64:64-i128:128-n32:64-S128"; - else - return "e-m:e-p:64:64-i64:64-i128:128-n32:64-S128"; -} - static Reloc::Model getEffectiveRelocModel(std::optional<Reloc::Model> RM) { return RM.value_or(Reloc::PIC_); } @@ -77,7 +69,7 @@ BPFTargetMachine::BPFTargetMachine(const Target &T, const Triple &TT, std::optional<Reloc::Model> RM, std::optional<CodeModel::Model> CM, CodeGenOptLevel OL, bool JIT) - : CodeGenTargetMachineImpl(T, computeDataLayout(TT), TT, CPU, FS, Options, + : CodeGenTargetMachineImpl(T, TT.computeDataLayout(), TT, CPU, FS, Options, getEffectiveRelocModel(RM), getEffectiveCodeModel(CM, CodeModel::Small), OL), TLOF(std::make_unique<TargetLoweringObjectFileELF>()), diff --git a/llvm/lib/Target/CSKY/CSKYTargetMachine.cpp b/llvm/lib/Target/CSKY/CSKYTargetMachine.cpp index ae6ef89fdcd0..d0058b9af14b 100644 --- a/llvm/lib/Target/CSKY/CSKYTargetMachine.cpp +++ b/llvm/lib/Target/CSKY/CSKYTargetMachine.cpp @@ -33,28 +33,13 @@ extern "C" LLVM_EXTERNAL_VISIBILITY void LLVMInitializeCSKYTarget() { initializeCSKYDAGToDAGISelLegacyPass(*Registry); } -static std::string computeDataLayout(const Triple &TT) { - std::string Ret; - - // Only support little endian for now. - // TODO: Add support for big endian. - Ret += "e"; - - // CSKY is always 32-bit target with the CSKYv2 ABI as prefer now. - // It's a 4-byte aligned stack with ELF mangling only. - Ret += "-m:e-S32-p:32:32-i32:32:32-i64:32:32-f32:32:32-f64:32:32-v64:32:32" - "-v128:32:32-a:0:32-Fi32-n32"; - - return Ret; -} - CSKYTargetMachine::CSKYTargetMachine(const Target &T, const Triple &TT, StringRef CPU, StringRef FS, const TargetOptions &Options, std::optional<Reloc::Model> RM, std::optional<CodeModel::Model> CM, CodeGenOptLevel OL, bool JIT) - : CodeGenTargetMachineImpl(T, computeDataLayout(TT), TT, CPU, FS, Options, + : CodeGenTargetMachineImpl(T, TT.computeDataLayout(), TT, CPU, FS, Options, RM.value_or(Reloc::Static), getEffectiveCodeModel(CM, CodeModel::Small), OL), TLOF(std::make_unique<CSKYELFTargetObjectFile>()) { diff --git a/llvm/lib/Target/DirectX/DXContainerGlobals.cpp b/llvm/lib/Target/DirectX/DXContainerGlobals.cpp index a1ef2578f00a..ca81d30473c0 100644 --- a/llvm/lib/Target/DirectX/DXContainerGlobals.cpp +++ b/llvm/lib/Target/DirectX/DXContainerGlobals.cpp @@ -158,12 +158,15 @@ void DXContainerGlobals::addRootSignature(Module &M, if (MMI.ShaderProfile == llvm::Triple::Library) return; - assert(MMI.EntryPropertyVec.size() == 1); - auto &RSA = getAnalysis<RootSignatureAnalysisWrapper>().getRSInfo(); - const Function *EntryFunction = MMI.EntryPropertyVec[0].Entry; - const mcdxbc::RootSignatureDesc *RS = RSA.getDescForFunction(EntryFunction); + const Function *EntryFunction = nullptr; + if (MMI.ShaderProfile != llvm::Triple::RootSignature) { + assert(MMI.EntryPropertyVec.size() == 1); + EntryFunction = MMI.EntryPropertyVec[0].Entry; + } + + const mcdxbc::RootSignatureDesc *RS = RSA.getDescForFunction(EntryFunction); if (!RS) return; @@ -258,7 +261,8 @@ void DXContainerGlobals::addPipelineStateValidationInfo( dxil::ModuleMetadataInfo &MMI = getAnalysis<DXILMetadataAnalysisWrapperPass>().getModuleMetadata(); assert(MMI.EntryPropertyVec.size() == 1 || - MMI.ShaderProfile == Triple::Library); + MMI.ShaderProfile == Triple::Library || + MMI.ShaderProfile == Triple::RootSignature); PSV.BaseData.ShaderStage = static_cast<uint8_t>(MMI.ShaderProfile - Triple::Pixel); @@ -279,7 +283,8 @@ void DXContainerGlobals::addPipelineStateValidationInfo( break; } - if (MMI.ShaderProfile != Triple::Library) + if (MMI.ShaderProfile != Triple::Library && + MMI.ShaderProfile != Triple::RootSignature) PSV.EntryName = MMI.EntryPropertyVec[0].Entry->getName(); PSV.finalize(MMI.ShaderProfile); diff --git a/llvm/lib/Target/DirectX/DXILPostOptimizationValidation.cpp b/llvm/lib/Target/DirectX/DXILPostOptimizationValidation.cpp index a139167685ec..7e93474e7311 100644 --- a/llvm/lib/Target/DirectX/DXILPostOptimizationValidation.cpp +++ b/llvm/lib/Target/DirectX/DXILPostOptimizationValidation.cpp @@ -25,21 +25,6 @@ using namespace llvm; using namespace llvm::dxil; -static ResourceClass toResourceClass(dxbc::DescriptorRangeType RangeType) { - using namespace dxbc; - switch (RangeType) { - case DescriptorRangeType::SRV: - return ResourceClass::SRV; - case DescriptorRangeType::UAV: - return ResourceClass::UAV; - case DescriptorRangeType::CBV: - return ResourceClass::CBuffer; - case DescriptorRangeType::Sampler: - return ResourceClass::Sampler; - } - llvm_unreachable("Unknown DescriptorRangeType"); -} - static ResourceClass toResourceClass(dxbc::RootParameterType Type) { using namespace dxbc; switch (Type) { @@ -118,6 +103,17 @@ static void reportOverlappingBinding(Module &M, DXILResourceMap &DRM) { "true, yet no overlapping binding was found"); } +static void reportInvalidHandleTyError(Module &M, ResourceClass RC, + ResourceInfo::ResourceBinding Binding) { + SmallString<160> Message; + raw_svector_ostream OS(Message); + StringRef RCName = getResourceClassName(RC); + OS << RCName << " at register " << Binding.LowerBound << " and space " + << Binding.Space << " is bound to a texture or typed buffer. " << RCName + << " root descriptors can only be Raw or Structured buffers."; + M.getContext().diagnose(DiagnosticInfoGeneric(Message)); +} + static void reportOverlappingRegisters(Module &M, const llvm::hlsl::Binding &R1, const llvm::hlsl::Binding &R2) { SmallString<128> Message; @@ -205,23 +201,20 @@ static void validateRootSignature(Module &M, const mcdxbc::DescriptorTable &Table = RSD.ParametersContainer.getDescriptorTable(ParamInfo.Location); - for (const dxbc::RTS0::v2::DescriptorRange &Range : Table.Ranges) { + for (const mcdxbc::DescriptorRange &Range : Table.Ranges) { uint32_t UpperBound = Range.NumDescriptors == ~0U ? Range.BaseShaderRegister : Range.BaseShaderRegister + Range.NumDescriptors - 1; - Builder.trackBinding( - toResourceClass( - static_cast<dxbc::DescriptorRangeType>(Range.RangeType)), - Range.RegisterSpace, Range.BaseShaderRegister, UpperBound, - &ParamInfo); + Builder.trackBinding(Range.RangeType, Range.RegisterSpace, + Range.BaseShaderRegister, UpperBound, &ParamInfo); } break; } } } - for (const dxbc::RTS0::v1::StaticSampler &S : RSD.StaticSamplers) + for (const mcdxbc::StaticSampler &S : RSD.StaticSamplers) Builder.trackBinding(dxil::ResourceClass::Sampler, S.RegisterSpace, S.ShaderRegister, S.ShaderRegister, &S); @@ -235,10 +228,29 @@ static void validateRootSignature(Module &M, const hlsl::BoundRegs &BoundRegs = Builder.takeBoundRegs(); for (const ResourceInfo &RI : DRM) { const ResourceInfo::ResourceBinding &Binding = RI.getBinding(); - ResourceClass RC = DRTM[RI.getHandleTy()].getResourceClass(); - if (!BoundRegs.isBound(RC, Binding.Space, Binding.LowerBound, - Binding.LowerBound + Binding.Size - 1)) + const dxil::ResourceTypeInfo &RTI = DRTM[RI.getHandleTy()]; + dxil::ResourceClass RC = RTI.getResourceClass(); + dxil::ResourceKind RK = RTI.getResourceKind(); + + const llvm::hlsl::Binding *Reg = + BoundRegs.findBoundReg(RC, Binding.Space, Binding.LowerBound, + Binding.LowerBound + Binding.Size - 1); + + if (Reg != nullptr) { + const auto *ParamInfo = + static_cast<const mcdxbc::RootParameterInfo *>(Reg->Cookie); + + if (RC != ResourceClass::SRV && RC != ResourceClass::UAV) + continue; + + if (ParamInfo->Type == dxbc::RootParameterType::DescriptorTable) + continue; + + if (RK != ResourceKind::RawBuffer && RK != ResourceKind::StructuredBuffer) + reportInvalidHandleTyError(M, RC, Binding); + } else { reportRegNotBound(M, RC, Binding); + } } } diff --git a/llvm/lib/Target/DirectX/DXILRootSignature.cpp b/llvm/lib/Target/DirectX/DXILRootSignature.cpp index 2436d3869464..ac3c7dde6b89 100644 --- a/llvm/lib/Target/DirectX/DXILRootSignature.cpp +++ b/llvm/lib/Target/DirectX/DXILRootSignature.cpp @@ -72,6 +72,13 @@ analyzeModule(Module &M) { if (RootSignatureNode == nullptr) return RSDMap; + bool AllowNullFunctions = false; + if (M.getTargetTriple().getEnvironment() == + Triple::EnvironmentType::RootSignature) { + assert(RootSignatureNode->getNumOperands() == 1); + AllowNullFunctions = true; + } + for (const auto &RSDefNode : RootSignatureNode->operands()) { if (RSDefNode->getNumOperands() != 3) { reportError(Ctx, "Invalid Root Signature metadata - expected function, " @@ -80,24 +87,28 @@ analyzeModule(Module &M) { } // Function was pruned during compilation. - const MDOperand &FunctionPointerMdNode = RSDefNode->getOperand(0); - if (FunctionPointerMdNode == nullptr) { - reportError( - Ctx, "Function associated with Root Signature definition is null."); - continue; - } + Function *F = nullptr; + + if (!AllowNullFunctions) { + const MDOperand &FunctionPointerMdNode = RSDefNode->getOperand(0); + if (FunctionPointerMdNode == nullptr) { + reportError( + Ctx, "Function associated with Root Signature definition is null."); + continue; + } - ValueAsMetadata *VAM = - llvm::dyn_cast<ValueAsMetadata>(FunctionPointerMdNode.get()); - if (VAM == nullptr) { - reportError(Ctx, "First element of root signature is not a Value"); - continue; - } + ValueAsMetadata *VAM = + llvm::dyn_cast<ValueAsMetadata>(FunctionPointerMdNode.get()); + if (VAM == nullptr) { + reportError(Ctx, "First element of root signature is not a Value"); + continue; + } - Function *F = dyn_cast<Function>(VAM->getValue()); - if (F == nullptr) { - reportError(Ctx, "First element of root signature is not a Function"); - continue; + F = dyn_cast<Function>(VAM->getValue()); + if (F == nullptr) { + reportError(Ctx, "First element of root signature is not a Function"); + continue; + } } Metadata *RootElementListOperand = RSDefNode->getOperand(1).get(); @@ -205,8 +216,9 @@ PreservedAnalyses RootSignatureAnalysisPrinter::run(Module &M, RS.ParametersContainer.getDescriptorTable(Info.Location); OS << " NumRanges: " << Table.Ranges.size() << "\n"; - for (const dxbc::RTS0::v2::DescriptorRange Range : Table) { - OS << " - Range Type: " << Range.RangeType << "\n" + for (const mcdxbc::DescriptorRange &Range : Table) { + OS << " - Range Type: " + << dxil::getResourceClassName(Range.RangeType) << "\n" << " Register Space: " << Range.RegisterSpace << "\n" << " Base Shader Register: " << Range.BaseShaderRegister << "\n" << " Num Descriptors: " << Range.NumDescriptors << "\n" diff --git a/llvm/lib/Target/DirectX/DXILTranslateMetadata.cpp b/llvm/lib/Target/DirectX/DXILTranslateMetadata.cpp index 82bcacee7a6d..9eebcc9b1306 100644 --- a/llvm/lib/Target/DirectX/DXILTranslateMetadata.cpp +++ b/llvm/lib/Target/DirectX/DXILTranslateMetadata.cpp @@ -127,6 +127,8 @@ static StringRef getShortShaderStage(Triple::EnvironmentType Env) { return "ms"; case Triple::Amplification: return "as"; + case Triple::RootSignature: + return "rootsig"; default: break; } diff --git a/llvm/lib/Target/DirectX/DirectXTargetMachine.cpp b/llvm/lib/Target/DirectX/DirectXTargetMachine.cpp index f5d5a73c926e..bcf84403b2c0 100644 --- a/llvm/lib/Target/DirectX/DirectXTargetMachine.cpp +++ b/llvm/lib/Target/DirectX/DirectXTargetMachine.cpp @@ -134,11 +134,8 @@ DirectXTargetMachine::DirectXTargetMachine(const Target &T, const Triple &TT, std::optional<Reloc::Model> RM, std::optional<CodeModel::Model> CM, CodeGenOptLevel OL, bool JIT) - : CodeGenTargetMachineImpl( - T, - "e-m:e-p:32:32-i1:32-i8:8-i16:16-i32:32-i64:64-f16:16-" - "f32:32-f64:64-n8:16:32:64", - TT, CPU, FS, Options, Reloc::Static, CodeModel::Small, OL), + : CodeGenTargetMachineImpl(T, TT.computeDataLayout(), TT, CPU, FS, Options, + Reloc::Static, CodeModel::Small, OL), TLOF(std::make_unique<DXILTargetObjectFile>()), Subtarget(std::make_unique<DirectXSubtarget>(TT, CPU, FS, *this)) { initAsmInfo(); diff --git a/llvm/lib/Target/Hexagon/Hexagon.td b/llvm/lib/Target/Hexagon/Hexagon.td index 0dbe743d13ed..6d0529fb4277 100644 --- a/llvm/lib/Target/Hexagon/Hexagon.td +++ b/llvm/lib/Target/Hexagon/Hexagon.td @@ -176,8 +176,11 @@ def UseSmallData : Predicate<"HST->useSmallData()">; def UseCabac : Predicate<"HST->useCabac()">, AssemblerPredicate<(any_of FeatureCabac)>; -def Hvx64: HwMode<"+hvx-length64b", [UseHVX64B]>; -def Hvx128: HwMode<"+hvx-length128b", [UseHVX128B]>; +def : HwModePredicateProlog<[{ + const auto *HST = static_cast<const HexagonSubtarget *>(this); +}]>; +def Hvx64: HwMode<[UseHVX64B]>; +def Hvx128: HwMode<[UseHVX128B]>; //===----------------------------------------------------------------------===// // Classes used for relation maps. diff --git a/llvm/lib/Target/Hexagon/HexagonTargetMachine.cpp b/llvm/lib/Target/Hexagon/HexagonTargetMachine.cpp index 66508fd76779..0afa04ab57e8 100644 --- a/llvm/lib/Target/Hexagon/HexagonTargetMachine.cpp +++ b/llvm/lib/Target/Hexagon/HexagonTargetMachine.cpp @@ -231,14 +231,10 @@ HexagonTargetMachine::HexagonTargetMachine(const Target &T, const Triple &TT, // Specify the vector alignment explicitly. For v512x1, the calculated // alignment would be 512*alignment(i1), which is 512 bytes, instead of // the required minimum of 64 bytes. - : CodeGenTargetMachineImpl( - T, - "e-m:e-p:32:32:32-a:0-n16:32-" - "i64:64:64-i32:32:32-i16:16:16-i1:8:8-f32:32:32-f64:64:64-" - "v32:32:32-v64:64:64-v512:512:512-v1024:1024:1024-v2048:2048:2048", - TT, CPU, FS, Options, getEffectiveRelocModel(RM), - getEffectiveCodeModel(CM, CodeModel::Small), - (HexagonNoOpt ? CodeGenOptLevel::None : OL)), + : CodeGenTargetMachineImpl(T, TT.computeDataLayout(), TT, CPU, FS, Options, + getEffectiveRelocModel(RM), + getEffectiveCodeModel(CM, CodeModel::Small), + (HexagonNoOpt ? CodeGenOptLevel::None : OL)), TLOF(std::make_unique<HexagonTargetObjectFile>()), Subtarget(Triple(TT), CPU, FS, *this) { initAsmInfo(); diff --git a/llvm/lib/Target/Lanai/LanaiTargetMachine.cpp b/llvm/lib/Target/Lanai/LanaiTargetMachine.cpp index 3d6ba9ecc55e..df56f9ae39fe 100644 --- a/llvm/lib/Target/Lanai/LanaiTargetMachine.cpp +++ b/llvm/lib/Target/Lanai/LanaiTargetMachine.cpp @@ -37,17 +37,6 @@ extern "C" LLVM_ABI LLVM_EXTERNAL_VISIBILITY void LLVMInitializeLanaiTarget() { initializeLanaiMemAluCombinerPass(PR); } -static std::string computeDataLayout() { - // Data layout (keep in sync with clang/lib/Basic/Targets.cpp) - return "E" // Big endian - "-m:e" // ELF name manging - "-p:32:32" // 32-bit pointers, 32 bit aligned - "-i64:64" // 64 bit integers, 64 bit aligned - "-a:0:32" // 32 bit alignment of objects of aggregate type - "-n32" // 32 bit native integer width - "-S64"; // 64 bit natural stack alignment -} - static Reloc::Model getEffectiveRelocModel(std::optional<Reloc::Model> RM) { return RM.value_or(Reloc::PIC_); } @@ -58,7 +47,7 @@ LanaiTargetMachine::LanaiTargetMachine( std::optional<CodeModel::Model> CodeModel, CodeGenOptLevel OptLevel, bool JIT) : CodeGenTargetMachineImpl( - T, computeDataLayout(), TT, Cpu, FeatureString, Options, + T, TT.computeDataLayout(), TT, Cpu, FeatureString, Options, getEffectiveRelocModel(RM), getEffectiveCodeModel(CodeModel, CodeModel::Medium), OptLevel), Subtarget(TT, Cpu, FeatureString, *this, Options, getCodeModel(), diff --git a/llvm/lib/Target/LoongArch/LoongArch.td b/llvm/lib/Target/LoongArch/LoongArch.td index 39948b31fb9b..6497ff999f6f 100644 --- a/llvm/lib/Target/LoongArch/LoongArch.td +++ b/llvm/lib/Target/LoongArch/LoongArch.td @@ -39,7 +39,7 @@ def IsLA32 "LA32 Basic Integer and Privilege Instruction Set">; defvar LA32 = DefaultMode; -def LA64 : HwMode<"+64bit", [IsLA64]>; +def LA64 : HwMode<[IsLA64]>; // Single Precision floating point def FeatureBasicF diff --git a/llvm/lib/Target/LoongArch/LoongArchTargetMachine.cpp b/llvm/lib/Target/LoongArch/LoongArchTargetMachine.cpp index c36db9c75dd3..d0a8ababe8e5 100644 --- a/llvm/lib/Target/LoongArch/LoongArchTargetMachine.cpp +++ b/llvm/lib/Target/LoongArch/LoongArchTargetMachine.cpp @@ -57,13 +57,6 @@ static cl::opt<bool> cl::desc("Enable the loop data prefetch pass"), cl::init(false)); -static std::string computeDataLayout(const Triple &TT) { - if (TT.isArch64Bit()) - return "e-m:e-p:64:64-i64:64-i128:128-n32:64-S128"; - assert(TT.isArch32Bit() && "only LA32 and LA64 are currently supported"); - return "e-m:e-p:32:32-i64:64-n32-S128"; -} - static Reloc::Model getEffectiveRelocModel(const Triple &TT, std::optional<Reloc::Model> RM) { return RM.value_or(Reloc::Static); @@ -93,7 +86,7 @@ LoongArchTargetMachine::LoongArchTargetMachine( const Target &T, const Triple &TT, StringRef CPU, StringRef FS, const TargetOptions &Options, std::optional<Reloc::Model> RM, std::optional<CodeModel::Model> CM, CodeGenOptLevel OL, bool JIT) - : CodeGenTargetMachineImpl(T, computeDataLayout(TT), TT, CPU, FS, Options, + : CodeGenTargetMachineImpl(T, TT.computeDataLayout(), TT, CPU, FS, Options, getEffectiveRelocModel(TT, RM), getEffectiveLoongArchCodeModel(TT, CM), OL), TLOF(std::make_unique<TargetLoweringObjectFileELF>()) { diff --git a/llvm/lib/Target/M68k/M68kTargetMachine.cpp b/llvm/lib/Target/M68k/M68kTargetMachine.cpp index ce15ee635e21..847c27bac2cb 100644 --- a/llvm/lib/Target/M68k/M68kTargetMachine.cpp +++ b/llvm/lib/Target/M68k/M68kTargetMachine.cpp @@ -46,35 +46,6 @@ extern "C" LLVM_EXTERNAL_VISIBILITY void LLVMInitializeM68kTarget() { namespace { -std::string computeDataLayout(const Triple &TT, StringRef CPU, - const TargetOptions &Options) { - std::string Ret = ""; - // M68k is Big Endian - Ret += "E"; - - // FIXME how to wire it with the used object format? - Ret += "-m:e"; - - // M68k pointers are always 32 bit wide even for 16-bit CPUs. - // The ABI only specifies 16-bit alignment. - // On at least the 68020+ with a 32-bit bus, there is a performance benefit - // to having 32-bit alignment. - Ret += "-p:32:16:32"; - - // Bytes do not require special alignment, words are word aligned and - // long words are word aligned at minimum. - Ret += "-i8:8:8-i16:16:16-i32:16:32"; - - // FIXME no floats at the moment - - // The registers can hold 8, 16, 32 bits - Ret += "-n8:16:32"; - - Ret += "-a:0:16-S16"; - - return Ret; -} - Reloc::Model getEffectiveRelocModel(const Triple &TT, std::optional<Reloc::Model> RM) { // If not defined we default to static @@ -101,8 +72,8 @@ M68kTargetMachine::M68kTargetMachine(const Target &T, const Triple &TT, std::optional<Reloc::Model> RM, std::optional<CodeModel::Model> CM, CodeGenOptLevel OL, bool JIT) - : CodeGenTargetMachineImpl(T, computeDataLayout(TT, CPU, Options), TT, CPU, - FS, Options, getEffectiveRelocModel(TT, RM), + : CodeGenTargetMachineImpl(T, TT.computeDataLayout(), TT, CPU, FS, Options, + getEffectiveRelocModel(TT, RM), ::getEffectiveCodeModel(CM, JIT), OL), TLOF(std::make_unique<M68kELFTargetObjectFile>()), Subtarget(TT, CPU, FS, *this) { diff --git a/llvm/lib/Target/MSP430/CMakeLists.txt b/llvm/lib/Target/MSP430/CMakeLists.txt index 4081d3472fd7..bcf9fd288dbd 100644 --- a/llvm/lib/Target/MSP430/CMakeLists.txt +++ b/llvm/lib/Target/MSP430/CMakeLists.txt @@ -40,6 +40,7 @@ add_llvm_target(MSP430CodeGen SelectionDAG Support Target + TargetParser ADD_TO_COMPONENT MSP430 diff --git a/llvm/lib/Target/MSP430/MSP430TargetMachine.cpp b/llvm/lib/Target/MSP430/MSP430TargetMachine.cpp index e6024f4a6218..988bcae120f9 100644 --- a/llvm/lib/Target/MSP430/MSP430TargetMachine.cpp +++ b/llvm/lib/Target/MSP430/MSP430TargetMachine.cpp @@ -34,19 +34,14 @@ static Reloc::Model getEffectiveRelocModel(std::optional<Reloc::Model> RM) { return RM.value_or(Reloc::Static); } -static std::string computeDataLayout(const Triple &TT, StringRef CPU, - const TargetOptions &Options) { - return "e-m:e-p:16:16-i32:16-i64:16-f32:16-f64:16-a:8-n8:16-S16"; -} - MSP430TargetMachine::MSP430TargetMachine(const Target &T, const Triple &TT, StringRef CPU, StringRef FS, const TargetOptions &Options, std::optional<Reloc::Model> RM, std::optional<CodeModel::Model> CM, CodeGenOptLevel OL, bool JIT) - : CodeGenTargetMachineImpl(T, computeDataLayout(TT, CPU, Options), TT, CPU, - FS, Options, getEffectiveRelocModel(RM), + : CodeGenTargetMachineImpl(T, TT.computeDataLayout(), TT, CPU, FS, Options, + getEffectiveRelocModel(RM), getEffectiveCodeModel(CM, CodeModel::Small), OL), TLOF(std::make_unique<TargetLoweringObjectFileELF>()), Subtarget(TT, std::string(CPU), std::string(FS), *this) { diff --git a/llvm/lib/Target/Mips/AsmParser/MipsAsmParser.cpp b/llvm/lib/Target/Mips/AsmParser/MipsAsmParser.cpp index 7b2ee832ae7d..8a5cb517c94c 100644 --- a/llvm/lib/Target/Mips/AsmParser/MipsAsmParser.cpp +++ b/llvm/lib/Target/Mips/AsmParser/MipsAsmParser.cpp @@ -524,8 +524,8 @@ public: MipsAsmParser(const MCSubtargetInfo &sti, MCAsmParser &parser, const MCInstrInfo &MII, const MCTargetOptions &Options) : MCTargetAsmParser(Options, sti, MII), - ABI(MipsABIInfo::computeTargetABI(sti.getTargetTriple(), sti.getCPU(), - Options)) { + ABI(MipsABIInfo::computeTargetABI(sti.getTargetTriple(), + Options.getABIName())) { MCAsmParserExtension::Initialize(parser); parser.addAliasForDirective(".asciiz", ".asciz"); diff --git a/llvm/lib/Target/Mips/MCTargetDesc/MipsABIInfo.cpp b/llvm/lib/Target/Mips/MCTargetDesc/MipsABIInfo.cpp index 1be29cf3c94b..d7809e27e23f 100644 --- a/llvm/lib/Target/Mips/MCTargetDesc/MipsABIInfo.cpp +++ b/llvm/lib/Target/Mips/MCTargetDesc/MipsABIInfo.cpp @@ -57,17 +57,16 @@ unsigned MipsABIInfo::GetCalleeAllocdArgSizeInBytes(CallingConv::ID CC) const { llvm_unreachable("Unhandled ABI"); } -MipsABIInfo MipsABIInfo::computeTargetABI(const Triple &TT, StringRef CPU, - const MCTargetOptions &Options) { - if (Options.getABIName().starts_with("o32")) +MipsABIInfo MipsABIInfo::computeTargetABI(const Triple &TT, StringRef ABIName) { + if (ABIName.starts_with("o32")) return MipsABIInfo::O32(); - if (Options.getABIName().starts_with("n32")) + if (ABIName.starts_with("n32")) return MipsABIInfo::N32(); - if (Options.getABIName().starts_with("n64")) + if (ABIName.starts_with("n64")) return MipsABIInfo::N64(); if (TT.isABIN32()) return MipsABIInfo::N32(); - assert(Options.getABIName().empty() && "Unknown ABI option for MIPS"); + assert(ABIName.empty() && "Unknown ABI option for MIPS"); if (TT.isMIPS64()) return MipsABIInfo::N64(); diff --git a/llvm/lib/Target/Mips/MCTargetDesc/MipsABIInfo.h b/llvm/lib/Target/Mips/MCTargetDesc/MipsABIInfo.h index 44b023c7c3ef..d8003d2fcc16 100644 --- a/llvm/lib/Target/Mips/MCTargetDesc/MipsABIInfo.h +++ b/llvm/lib/Target/Mips/MCTargetDesc/MipsABIInfo.h @@ -33,8 +33,7 @@ public: static MipsABIInfo O32() { return MipsABIInfo(ABI::O32); } static MipsABIInfo N32() { return MipsABIInfo(ABI::N32); } static MipsABIInfo N64() { return MipsABIInfo(ABI::N64); } - static MipsABIInfo computeTargetABI(const Triple &TT, StringRef CPU, - const MCTargetOptions &Options); + static MipsABIInfo computeTargetABI(const Triple &TT, StringRef ABIName); bool IsKnown() const { return ThisABI != ABI::Unknown; } bool IsO32() const { return ThisABI == ABI::O32; } diff --git a/llvm/lib/Target/Mips/MCTargetDesc/MipsAsmBackend.cpp b/llvm/lib/Target/Mips/MCTargetDesc/MipsAsmBackend.cpp index 33aab71044b0..74e7baf1db29 100644 --- a/llvm/lib/Target/Mips/MCTargetDesc/MipsAsmBackend.cpp +++ b/llvm/lib/Target/Mips/MCTargetDesc/MipsAsmBackend.cpp @@ -619,7 +619,7 @@ MCAsmBackend *llvm::createMipsAsmBackend(const Target &T, return new WindowsMipsAsmBackend(T, MRI, STI); MipsABIInfo ABI = MipsABIInfo::computeTargetABI(STI.getTargetTriple(), - STI.getCPU(), Options); + Options.getABIName()); return new MipsAsmBackend(T, MRI, STI.getTargetTriple(), STI.getCPU(), ABI.IsN32()); } diff --git a/llvm/lib/Target/Mips/MCTargetDesc/MipsMCAsmInfo.cpp b/llvm/lib/Target/Mips/MCTargetDesc/MipsMCAsmInfo.cpp index 8b28ee62b878..e1c9954c19cc 100644 --- a/llvm/lib/Target/Mips/MCTargetDesc/MipsMCAsmInfo.cpp +++ b/llvm/lib/Target/Mips/MCTargetDesc/MipsMCAsmInfo.cpp @@ -24,7 +24,8 @@ MipsELFMCAsmInfo::MipsELFMCAsmInfo(const Triple &TheTriple, const MCTargetOptions &Options) { IsLittleEndian = TheTriple.isLittleEndian(); - MipsABIInfo ABI = MipsABIInfo::computeTargetABI(TheTriple, "", Options); + MipsABIInfo ABI = + MipsABIInfo::computeTargetABI(TheTriple, Options.getABIName()); if (TheTriple.isMIPS64() && !ABI.IsN32()) CodePointerSize = CalleeSaveStackSlotSize = 8; diff --git a/llvm/lib/Target/Mips/MipsSubtarget.cpp b/llvm/lib/Target/Mips/MipsSubtarget.cpp index 8b9ee7fca208..8c4bb15a7e61 100644 --- a/llvm/lib/Target/Mips/MipsSubtarget.cpp +++ b/llvm/lib/Target/Mips/MipsSubtarget.cpp @@ -245,20 +245,10 @@ CodeGenOptLevel MipsSubtarget::getOptLevelToEnablePostRAScheduler() const { MipsSubtarget & MipsSubtarget::initializeSubtargetDependencies(StringRef CPU, StringRef FS, const TargetMachine &TM) { - const Triple &TT = TM.getTargetTriple(); - StringRef CPUName = MIPS_MC::selectMipsCPU(TT, CPU); - - std::string FullFS; - if (getABI().ArePtrs64bit()) { - FullFS = "+ptr64"; - if (!FS.empty()) - FullFS = (Twine(FullFS) + "," + FS).str(); - } else { - FullFS = FS.str(); - } + StringRef CPUName = MIPS_MC::selectMipsCPU(TM.getTargetTriple(), CPU); // Parse features string. - ParseSubtargetFeatures(CPUName, /*TuneCPU=*/CPUName, FullFS); + ParseSubtargetFeatures(CPUName, /*TuneCPU*/ CPUName, FS); // Initialize scheduling itinerary for the specified CPU. InstrItins = getInstrItineraryForCPU(CPUName); diff --git a/llvm/lib/Target/Mips/MipsTargetMachine.cpp b/llvm/lib/Target/Mips/MipsTargetMachine.cpp index 8c519fa379dd..03bedc5b15c4 100644 --- a/llvm/lib/Target/Mips/MipsTargetMachine.cpp +++ b/llvm/lib/Target/Mips/MipsTargetMachine.cpp @@ -77,42 +77,6 @@ static std::unique_ptr<TargetLoweringObjectFile> createTLOF(const Triple &TT) { return std::make_unique<MipsTargetObjectFile>(); } -static std::string computeDataLayout(const Triple &TT, StringRef CPU, - const TargetOptions &Options, - bool isLittle) { - std::string Ret; - MipsABIInfo ABI = MipsABIInfo::computeTargetABI(TT, CPU, Options.MCOptions); - - // There are both little and big endian mips. - if (isLittle) - Ret += "e"; - else - Ret += "E"; - - if (ABI.IsO32()) - Ret += "-m:m"; - else - Ret += "-m:e"; - - // Pointers are 32 bit on some ABIs. - if (!ABI.IsN64()) - Ret += "-p:32:32"; - - // 8 and 16 bit integers only need to have natural alignment, but try to - // align them to 32 bits. 64 bit integers have natural alignment. - Ret += "-i8:8:32-i16:16:32-i64:64"; - - // 32 bit registers are always available and the stack is at least 64 bit - // aligned. On N64 64 bit registers are also available and the stack is - // 128 bit aligned. - if (ABI.IsN64() || ABI.IsN32()) - Ret += "-i128:128-n32:64-S128"; - else - Ret += "-n32-S64"; - - return Ret; -} - static Reloc::Model getEffectiveRelocModel(bool JIT, std::optional<Reloc::Model> RM) { if (!RM || JIT) @@ -132,12 +96,12 @@ MipsTargetMachine::MipsTargetMachine(const Target &T, const Triple &TT, std::optional<CodeModel::Model> CM, CodeGenOptLevel OL, bool JIT, bool isLittle) - : CodeGenTargetMachineImpl(T, computeDataLayout(TT, CPU, Options, isLittle), - TT, CPU, FS, Options, - getEffectiveRelocModel(JIT, RM), - getEffectiveCodeModel(CM, CodeModel::Small), OL), + : CodeGenTargetMachineImpl( + T, TT.computeDataLayout(Options.MCOptions.getABIName()), TT, CPU, FS, + Options, getEffectiveRelocModel(JIT, RM), + getEffectiveCodeModel(CM, CodeModel::Small), OL), isLittle(isLittle), TLOF(createTLOF(getTargetTriple())), - ABI(MipsABIInfo::computeTargetABI(TT, CPU, Options.MCOptions)), + ABI(MipsABIInfo::computeTargetABI(TT, Options.MCOptions.getABIName())), Subtarget(nullptr), DefaultSubtarget(TT, CPU, FS, isLittle, *this, std::nullopt), NoMips16Subtarget(TT, CPU, FS.empty() ? "-mips16" : FS.str() + ",-mips16", diff --git a/llvm/lib/Target/NVPTX/NVPTXTargetMachine.cpp b/llvm/lib/Target/NVPTX/NVPTXTargetMachine.cpp index 833f014a4c87..a6837a482608 100644 --- a/llvm/lib/Target/NVPTX/NVPTXTargetMachine.cpp +++ b/llvm/lib/Target/NVPTX/NVPTXTargetMachine.cpp @@ -118,24 +118,6 @@ extern "C" LLVM_ABI LLVM_EXTERNAL_VISIBILITY void LLVMInitializeNVPTXTarget() { initializeNVPTXPrologEpilogPassPass(PR); } -static std::string computeDataLayout(bool is64Bit, bool UseShortPointers) { - std::string Ret = "e"; - - // Tensor Memory (addrspace:6) is always 32-bits. - // Distributed Shared Memory (addrspace:7) follows shared memory - // (addrspace:3). - if (!is64Bit) - Ret += "-p:32:32-p6:32:32-p7:32:32"; - else if (UseShortPointers) - Ret += "-p3:32:32-p4:32:32-p5:32:32-p6:32:32-p7:32:32"; - else - Ret += "-p6:32:32"; - - Ret += "-i64:64-i128:128-i256:256-v16:16-v32:32-n16:32:64"; - - return Ret; -} - NVPTXTargetMachine::NVPTXTargetMachine(const Target &T, const Triple &TT, StringRef CPU, StringRef FS, const TargetOptions &Options, @@ -144,10 +126,10 @@ NVPTXTargetMachine::NVPTXTargetMachine(const Target &T, const Triple &TT, CodeGenOptLevel OL, bool is64bit) // The pic relocation model is used regardless of what the client has // specified, as it is the only relocation model currently supported. - : CodeGenTargetMachineImpl(T, - computeDataLayout(is64bit, UseShortPointersOpt), - TT, CPU, FS, Options, Reloc::PIC_, - getEffectiveCodeModel(CM, CodeModel::Small), OL), + : CodeGenTargetMachineImpl( + T, TT.computeDataLayout(UseShortPointersOpt ? "shortptr" : ""), TT, + CPU, FS, Options, Reloc::PIC_, + getEffectiveCodeModel(CM, CodeModel::Small), OL), is64bit(is64bit), TLOF(std::make_unique<NVPTXTargetObjectFile>()), Subtarget(TT, std::string(CPU), std::string(FS), *this), StrPool(StrAlloc) { diff --git a/llvm/lib/Target/PowerPC/PPCAsmPrinter.cpp b/llvm/lib/Target/PowerPC/PPCAsmPrinter.cpp index 2ab2c147be0e..023fd147535e 100644 --- a/llvm/lib/Target/PowerPC/PPCAsmPrinter.cpp +++ b/llvm/lib/Target/PowerPC/PPCAsmPrinter.cpp @@ -920,10 +920,6 @@ void PPCAsmPrinter::emitInstruction(const MachineInstr *MI) { case TargetOpcode::PATCHABLE_FUNCTION_ENTER: { assert(!Subtarget->isAIXABI() && "AIX does not support patchable function entry!"); - // PATCHABLE_FUNCTION_ENTER on little endian is for XRAY support which is - // handled in PPCLinuxAsmPrinter. - if (MAI->isLittleEndian()) - return; const Function &F = MF->getFunction(); unsigned Num = 0; (void)F.getFnAttribute("patchable-function-entry") @@ -1789,7 +1785,13 @@ void PPCLinuxAsmPrinter::emitInstruction(const MachineInstr *MI) { // Update compiler-rt/lib/xray/xray_powerpc64.cc accordingly when number // of instructions change. // XRAY is only supported on PPC Linux little endian. - if (!MAI->isLittleEndian()) + const Function &F = MF->getFunction(); + unsigned Num = 0; + (void)F.getFnAttribute("patchable-function-entry") + .getValueAsString() + .getAsInteger(10, Num); + + if (!MAI->isLittleEndian() || Num) break; MCSymbol *BeginOfSled = OutContext.createTempSymbol(); MCSymbol *EndOfSled = OutContext.createTempSymbol(); diff --git a/llvm/lib/Target/PowerPC/PPCTargetMachine.cpp b/llvm/lib/Target/PowerPC/PPCTargetMachine.cpp index ae92d5eab20c..000d29610678 100644 --- a/llvm/lib/Target/PowerPC/PPCTargetMachine.cpp +++ b/llvm/lib/Target/PowerPC/PPCTargetMachine.cpp @@ -149,58 +149,6 @@ LLVMInitializePowerPCTarget() { initializePPCAIXAsmPrinterPass(PR); } -static bool isLittleEndianTriple(const Triple &T) { - return T.getArch() == Triple::ppc64le || T.getArch() == Triple::ppcle; -} - -/// Return the datalayout string of a subtarget. -static std::string getDataLayoutString(const Triple &T) { - bool is64Bit = T.getArch() == Triple::ppc64 || T.getArch() == Triple::ppc64le; - std::string Ret; - - // Most PPC* platforms are big endian, PPC(64)LE is little endian. - if (isLittleEndianTriple(T)) - Ret = "e"; - else - Ret = "E"; - - Ret += DataLayout::getManglingComponent(T); - - // PPC32 has 32 bit pointers. The PS3 (OS Lv2) is a PPC64 machine with 32 bit - // pointers. - if (!is64Bit || T.getOS() == Triple::Lv2) - Ret += "-p:32:32"; - - // If the target ABI uses function descriptors, then the alignment of function - // pointers depends on the alignment used to emit the descriptor. Otherwise, - // function pointers are aligned to 32 bits because the instructions must be. - if ((T.getArch() == Triple::ppc64 && !T.isPPC64ELFv2ABI())) { - Ret += "-Fi64"; - } else if (T.isOSAIX()) { - Ret += is64Bit ? "-Fi64" : "-Fi32"; - } else { - Ret += "-Fn32"; - } - - // Note, the alignment values for f64 and i64 on ppc64 in Darwin - // documentation are wrong; these are correct (i.e. "what gcc does"). - Ret += "-i64:64"; - - // PPC64 has 32 and 64 bit registers, PPC32 has only 32 bit ones. - if (is64Bit) - Ret += "-i128:128-n32:64"; - else - Ret += "-n32"; - - // Specify the vector alignment explicitly. For v256i1 and v512i1, the - // calculated alignment would be 256*alignment(i1) and 512*alignment(i1), - // which is 256 and 512 bytes - way over aligned. - if (is64Bit && (T.isOSAIX() || T.isOSLinux())) - Ret += "-S128-v256:256:256-v512:512:512"; - - return Ret; -} - static std::string computeFSAdditions(StringRef FS, CodeGenOptLevel OL, const Triple &TT) { std::string FullFS = std::string(FS); @@ -348,13 +296,13 @@ PPCTargetMachine::PPCTargetMachine(const Target &T, const Triple &TT, std::optional<Reloc::Model> RM, std::optional<CodeModel::Model> CM, CodeGenOptLevel OL, bool JIT) - : CodeGenTargetMachineImpl(T, getDataLayoutString(TT), TT, CPU, + : CodeGenTargetMachineImpl(T, TT.computeDataLayout(), TT, CPU, computeFSAdditions(FS, OL, TT), Options, getEffectiveRelocModel(TT, RM), getEffectivePPCCodeModel(TT, CM, JIT), OL), TLOF(createTLOF(getTargetTriple())), TargetABI(computeTargetABI(TT, Options)), - Endianness(isLittleEndianTriple(TT) ? Endian::LITTLE : Endian::BIG) { + Endianness(TT.isLittleEndian() ? Endian::LITTLE : Endian::BIG) { initAsmInfo(); } diff --git a/llvm/lib/Target/RISCV/Disassembler/RISCVDisassembler.cpp b/llvm/lib/Target/RISCV/Disassembler/RISCVDisassembler.cpp index 89df9d82f878..61b86abdc4ca 100644 --- a/llvm/lib/Target/RISCV/Disassembler/RISCVDisassembler.cpp +++ b/llvm/lib/Target/RISCV/Disassembler/RISCVDisassembler.cpp @@ -255,6 +255,15 @@ static DecodeStatus DecodeGPRPairRegisterClass(MCInst &Inst, uint32_t RegNo, return MCDisassembler::Success; } +static DecodeStatus +DecodeGPRPairNoX0RegisterClass(MCInst &Inst, uint32_t RegNo, uint64_t Address, + const MCDisassembler *Decoder) { + if (RegNo == 0) + return MCDisassembler::Fail; + + return DecodeGPRPairRegisterClass(Inst, RegNo, Address, Decoder); +} + static DecodeStatus DecodeGPRPairCRegisterClass(MCInst &Inst, uint32_t RegNo, uint64_t Address, const MCDisassembler *Decoder) { diff --git a/llvm/lib/Target/RISCV/GISel/RISCVLegalizerInfo.cpp b/llvm/lib/Target/RISCV/GISel/RISCVLegalizerInfo.cpp index ab5c9e17b9a3..16f34a89a52e 100644 --- a/llvm/lib/Target/RISCV/GISel/RISCVLegalizerInfo.cpp +++ b/llvm/lib/Target/RISCV/GISel/RISCVLegalizerInfo.cpp @@ -151,7 +151,9 @@ RISCVLegalizerInfo::RISCVLegalizerInfo(const RISCVSubtarget &ST) getActionDefinitionsBuilder( {G_UADDE, G_UADDO, G_USUBE, G_USUBO}).lower(); - getActionDefinitionsBuilder({G_SADDO, G_SSUBO}).minScalar(0, sXLen).lower(); + getActionDefinitionsBuilder({G_SADDO, G_SADDE, G_SSUBO}) + .minScalar(0, sXLen) + .lower(); // TODO: Use Vector Single-Width Saturating Instructions for vector types. getActionDefinitionsBuilder( @@ -488,7 +490,9 @@ RISCVLegalizerInfo::RISCVLegalizerInfo(const RISCVSubtarget &ST) .minScalar(ST.hasStdExtZbb(), 0, sXLen) .lower(); - getActionDefinitionsBuilder({G_ABDS, G_ABDU}).lower(); + getActionDefinitionsBuilder({G_ABDS, G_ABDU}) + .minScalar(ST.hasStdExtZbb(), 0, sXLen) + .lower(); getActionDefinitionsBuilder({G_UMAX, G_UMIN, G_SMAX, G_SMIN}) .legalFor(ST.hasStdExtZbb(), {sXLen}) @@ -507,8 +511,9 @@ RISCVLegalizerInfo::RISCVLegalizerInfo(const RISCVSubtarget &ST) // FP Operations // FIXME: Support s128 for rv32 when libcall handling is able to use sret. - getActionDefinitionsBuilder( - {G_FADD, G_FSUB, G_FMUL, G_FDIV, G_FMA, G_FSQRT, G_FMAXNUM, G_FMINNUM}) + getActionDefinitionsBuilder({G_FADD, G_FSUB, G_FMUL, G_FDIV, G_FMA, G_FSQRT, + G_FMAXNUM, G_FMINNUM, G_FMAXIMUMNUM, + G_FMINIMUMNUM}) .legalFor(ST.hasStdExtF(), {s32}) .legalFor(ST.hasStdExtD(), {s64}) .legalFor(ST.hasStdExtZfh(), {s16}) @@ -697,7 +702,10 @@ RISCVLegalizerInfo::RISCVLegalizerInfo(const RISCVSubtarget &ST) .customIf(all(typeIsLegalIntOrFPVec(0, IntOrFPVecTys, ST), typeIsLegalIntOrFPVec(1, IntOrFPVecTys, ST))); - getActionDefinitionsBuilder(G_ATOMICRMW_ADD) + getActionDefinitionsBuilder(G_ATOMIC_CMPXCHG_WITH_SUCCESS) + .lowerIf(all(typeInSet(0, {s8, s16, s32, s64}), typeIs(2, p0))); + + getActionDefinitionsBuilder({G_ATOMIC_CMPXCHG, G_ATOMICRMW_ADD}) .legalFor(ST.hasStdExtA(), {{sXLen, p0}}) .libcallFor(!ST.hasStdExtA(), {{s8, p0}, {s16, p0}, {s32, p0}, {s64, p0}}) .clampScalar(0, sXLen, sXLen); @@ -714,6 +722,10 @@ RISCVLegalizerInfo::RISCVLegalizerInfo(const RISCVSubtarget &ST) bool RISCVLegalizerInfo::legalizeIntrinsic(LegalizerHelper &Helper, MachineInstr &MI) const { Intrinsic::ID IntrinsicID = cast<GIntrinsic>(MI).getIntrinsicID(); + + if (RISCVVIntrinsicsTable::getRISCVVIntrinsicInfo(IntrinsicID)) + return true; + switch (IntrinsicID) { default: return false; @@ -746,6 +758,7 @@ bool RISCVLegalizerInfo::legalizeIntrinsic(LegalizerHelper &Helper, } case Intrinsic::riscv_masked_atomicrmw_add: case Intrinsic::riscv_masked_atomicrmw_sub: + case Intrinsic::riscv_masked_cmpxchg: return true; } } diff --git a/llvm/lib/Target/RISCV/RISCVFeatures.td b/llvm/lib/Target/RISCV/RISCVFeatures.td index bf5dca481cd2..063963d4ec36 100644 --- a/llvm/lib/Target/RISCV/RISCVFeatures.td +++ b/llvm/lib/Target/RISCV/RISCVFeatures.td @@ -1682,7 +1682,7 @@ def IsRV32 : Predicate<"!Subtarget->is64Bit()">, "RV32I Base Instruction Set">; defvar RV32 = DefaultMode; -def RV64 : HwMode<"+64bit", [IsRV64]>; +def RV64 : HwMode<[IsRV64]>; def FeatureRelax : SubtargetFeature<"relax", "EnableLinkerRelax", "true", diff --git a/llvm/lib/Target/RISCV/RISCVFrameLowering.cpp b/llvm/lib/Target/RISCV/RISCVFrameLowering.cpp index 9fc0d815ceee..06ce91771c9e 100644 --- a/llvm/lib/Target/RISCV/RISCVFrameLowering.cpp +++ b/llvm/lib/Target/RISCV/RISCVFrameLowering.cpp @@ -106,8 +106,14 @@ static void emitSCSPrologue(MachineFunction &MF, MachineBasicBlock &MBB, MachineBasicBlock::iterator MI, const DebugLoc &DL) { const auto &STI = MF.getSubtarget<RISCVSubtarget>(); + // We check Zimop instead of (Zimop || Zcmop) to determine whether HW shadow + // stack is available despite the fact that sspush/sspopchk both have a + // compressed form, because if only Zcmop is available, we would need to + // reserve X5 due to c.sspopchk only takes X5 and we currently do not support + // using X5 as the return address register. + // However, we can still aggressively use c.sspush x1 if zcmop is available. bool HasHWShadowStack = MF.getFunction().hasFnAttribute("hw-shadow-stack") && - STI.hasStdExtZicfiss(); + STI.hasStdExtZimop(); bool HasSWShadowStack = MF.getFunction().hasFnAttribute(Attribute::ShadowCallStack); if (!HasHWShadowStack && !HasSWShadowStack) @@ -124,7 +130,12 @@ static void emitSCSPrologue(MachineFunction &MF, MachineBasicBlock &MBB, const RISCVInstrInfo *TII = STI.getInstrInfo(); if (HasHWShadowStack) { - BuildMI(MBB, MI, DL, TII->get(RISCV::SSPUSH)).addReg(RAReg); + if (STI.hasStdExtZcmop()) { + static_assert(RAReg == RISCV::X1, "C.SSPUSH only accepts X1"); + BuildMI(MBB, MI, DL, TII->get(RISCV::PseudoMOP_C_SSPUSH)); + } else { + BuildMI(MBB, MI, DL, TII->get(RISCV::PseudoMOP_SSPUSH)).addReg(RAReg); + } return; } @@ -172,7 +183,7 @@ static void emitSCSEpilogue(MachineFunction &MF, MachineBasicBlock &MBB, const DebugLoc &DL) { const auto &STI = MF.getSubtarget<RISCVSubtarget>(); bool HasHWShadowStack = MF.getFunction().hasFnAttribute("hw-shadow-stack") && - STI.hasStdExtZicfiss(); + STI.hasStdExtZimop(); bool HasSWShadowStack = MF.getFunction().hasFnAttribute(Attribute::ShadowCallStack); if (!HasHWShadowStack && !HasSWShadowStack) @@ -186,7 +197,7 @@ static void emitSCSEpilogue(MachineFunction &MF, MachineBasicBlock &MBB, const RISCVInstrInfo *TII = STI.getInstrInfo(); if (HasHWShadowStack) { - BuildMI(MBB, MI, DL, TII->get(RISCV::SSPOPCHK)).addReg(RAReg); + BuildMI(MBB, MI, DL, TII->get(RISCV::PseudoMOP_SSPOPCHK)).addReg(RAReg); return; } diff --git a/llvm/lib/Target/RISCV/RISCVISelDAGToDAG.cpp b/llvm/lib/Target/RISCV/RISCVISelDAGToDAG.cpp index e84bf0eab08d..c7f15415ebb9 100644 --- a/llvm/lib/Target/RISCV/RISCVISelDAGToDAG.cpp +++ b/llvm/lib/Target/RISCV/RISCVISelDAGToDAG.cpp @@ -1052,7 +1052,7 @@ void RISCVDAGToDAGISel::Select(SDNode *Node) { SDLoc DL(Node); MVT VT = Node->getSimpleValueType(0); - bool HasBitTest = Subtarget->hasStdExtZbs() || Subtarget->hasVendorXTHeadBs(); + bool HasBitTest = Subtarget->hasBEXTILike(); switch (Opcode) { case ISD::Constant: { @@ -2905,6 +2905,65 @@ static bool isWorthFoldingAdd(SDValue Add) { return true; } +bool isRegImmLoadOrStore(SDNode *User, SDValue Add) { + switch (User->getOpcode()) { + default: + return false; + case ISD::LOAD: + case RISCVISD::LD_RV32: + case ISD::ATOMIC_LOAD: + break; + case ISD::STORE: + // Don't allow stores of Add. It must only be used as the address. + if (cast<StoreSDNode>(User)->getValue() == Add) + return false; + break; + case RISCVISD::SD_RV32: + // Don't allow stores of Add. It must only be used as the address. + if (User->getOperand(0) == Add || User->getOperand(1) == Add) + return false; + break; + case ISD::ATOMIC_STORE: + // Don't allow stores of Add. It must only be used as the address. + if (cast<AtomicSDNode>(User)->getVal() == Add) + return false; + break; + } + + return true; +} + +// To prevent SelectAddrRegImm from folding offsets that conflict with the +// fusion of PseudoMovAddr, check if the offset of every use of a given address +// is within the alignment. +bool RISCVDAGToDAGISel::areOffsetsWithinAlignment(SDValue Addr, + Align Alignment) { + assert(Addr->getOpcode() == RISCVISD::ADD_LO); + for (auto *User : Addr->users()) { + // If the user is a load or store, then the offset is 0 which is always + // within alignment. + if (isRegImmLoadOrStore(User, Addr)) + continue; + + if (CurDAG->isBaseWithConstantOffset(SDValue(User, 0))) { + int64_t CVal = cast<ConstantSDNode>(User->getOperand(1))->getSExtValue(); + if (!isInt<12>(CVal) || Alignment <= CVal) + return false; + + // Make sure all uses are foldable load/stores. + for (auto *AddUser : User->users()) + if (!isRegImmLoadOrStore(AddUser, SDValue(User, 0))) + return false; + + continue; + } + + return false; + } + + return true; +} + bool RISCVDAGToDAGISel::SelectAddrRegImm(SDValue Addr, SDValue &Base, SDValue &Offset) { if (SelectAddrFrameIndex(Addr, Base, Offset)) @@ -2914,9 +2973,21 @@ bool RISCVDAGToDAGISel::SelectAddrRegImm(SDValue Addr, SDValue &Base, MVT VT = Addr.getSimpleValueType(); if (Addr.getOpcode() == RISCVISD::ADD_LO) { - Base = Addr.getOperand(0); - Offset = Addr.getOperand(1); - return true; + bool CanFold = true; + // Unconditionally fold if operand 1 is not a global address (e.g. + // externsymbol) + if (auto *GA = dyn_cast<GlobalAddressSDNode>(Addr.getOperand(1))) { + const DataLayout &DL = CurDAG->getDataLayout(); + Align Alignment = commonAlignment( + GA->getGlobal()->getPointerAlignment(DL), GA->getOffset()); + if (!areOffsetsWithinAlignment(Addr, Alignment)) + CanFold = false; + } + if (CanFold) { + Base = Addr.getOperand(0); + Offset = Addr.getOperand(1); + return true; + } } if (CurDAG->isBaseWithConstantOffset(Addr)) { @@ -2934,7 +3005,8 @@ bool RISCVDAGToDAGISel::SelectAddrRegImm(SDValue Addr, SDValue &Base, const DataLayout &DL = CurDAG->getDataLayout(); Align Alignment = commonAlignment( GA->getGlobal()->getPointerAlignment(DL), GA->getOffset()); - if ((CVal == 0 || Alignment > CVal)) { + if ((CVal == 0 || Alignment > CVal) && + areOffsetsWithinAlignment(Base, Alignment)) { int64_t CombinedOffset = CVal + GA->getOffset(); Base = Base.getOperand(0); Offset = CurDAG->getTargetGlobalAddress( diff --git a/llvm/lib/Target/RISCV/RISCVISelDAGToDAG.h b/llvm/lib/Target/RISCV/RISCVISelDAGToDAG.h index ba129457d628..cf2f763abc06 100644 --- a/llvm/lib/Target/RISCV/RISCVISelDAGToDAG.h +++ b/llvm/lib/Target/RISCV/RISCVISelDAGToDAG.h @@ -45,6 +45,8 @@ public: InlineAsm::ConstraintCode ConstraintID, std::vector<SDValue> &OutOps) override; + bool areOffsetsWithinAlignment(SDValue Addr, Align Alignment); + bool SelectAddrFrameIndex(SDValue Addr, SDValue &Base, SDValue &Offset); bool SelectAddrRegImm(SDValue Addr, SDValue &Base, SDValue &Offset); bool SelectAddrRegImm9(SDValue Addr, SDValue &Base, SDValue &Offset); diff --git a/llvm/lib/Target/RISCV/RISCVISelLowering.cpp b/llvm/lib/Target/RISCV/RISCVISelLowering.cpp index 46f544c0d4df..4f137756d2f4 100644 --- a/llvm/lib/Target/RISCV/RISCVISelLowering.cpp +++ b/llvm/lib/Target/RISCV/RISCVISelLowering.cpp @@ -38,6 +38,7 @@ #include "llvm/IR/DiagnosticPrinter.h" #include "llvm/IR/IRBuilder.h" #include "llvm/IR/Instructions.h" +#include "llvm/IR/IntrinsicInst.h" #include "llvm/IR/IntrinsicsRISCV.h" #include "llvm/MC/MCCodeEmitter.h" #include "llvm/MC/MCInstBuilder.h" @@ -402,12 +403,14 @@ RISCVTargetLowering::RISCVTargetLowering(const TargetMachine &TM, Legal); } - if (Subtarget.hasStdExtZbb() || - (Subtarget.hasVendorXCVbitmanip() && !Subtarget.is64Bit())) { + if (Subtarget.hasCTZLike()) { if (Subtarget.is64Bit()) setOperationAction({ISD::CTTZ, ISD::CTTZ_ZERO_UNDEF}, MVT::i32, Custom); } else { setOperationAction(ISD::CTTZ, XLenVT, Expand); + } + + if (!Subtarget.hasCPOPLike()) { // TODO: These should be set to LibCall, but this currently breaks // the Linux kernel build. See #101786. Lacks i128 tests, too. if (Subtarget.is64Bit()) @@ -417,11 +420,11 @@ RISCVTargetLowering::RISCVTargetLowering(const TargetMachine &TM, setOperationAction(ISD::CTPOP, MVT::i64, Expand); } - if (Subtarget.hasStdExtZbb() || Subtarget.hasVendorXTHeadBb() || - (Subtarget.hasVendorXCVbitmanip() && !Subtarget.is64Bit())) { + if (Subtarget.hasCLZLike()) { // We need the custom lowering to make sure that the resulting sequence // for the 32bit case is efficient on 64bit targets. - if (Subtarget.is64Bit()) + // Use default promotion for i32 without Zbb. + if (Subtarget.is64Bit() && Subtarget.hasStdExtZbb()) setOperationAction({ISD::CTLZ, ISD::CTLZ_ZERO_UNDEF}, MVT::i32, Custom); } else { setOperationAction(ISD::CTLZ, XLenVT, Expand); @@ -2156,13 +2159,11 @@ bool RISCVTargetLowering::signExtendConstant(const ConstantInt *CI) const { } bool RISCVTargetLowering::isCheapToSpeculateCttz(Type *Ty) const { - return Subtarget.hasStdExtZbb() || - (Subtarget.hasVendorXCVbitmanip() && !Subtarget.is64Bit()); + return Subtarget.hasCTZLike(); } bool RISCVTargetLowering::isCheapToSpeculateCtlz(Type *Ty) const { - return Subtarget.hasStdExtZbb() || Subtarget.hasVendorXTHeadBb() || - (Subtarget.hasVendorXCVbitmanip() && !Subtarget.is64Bit()); + return Subtarget.hasCLZLike(); } bool RISCVTargetLowering::isMaskAndCmp0FoldingBeneficial( @@ -2173,7 +2174,7 @@ bool RISCVTargetLowering::isMaskAndCmp0FoldingBeneficial( // on the basis that it's possible the sinking+duplication of the AND in // CodeGenPrepare triggered by this hook wouldn't decrease the instruction // count and would increase code size (e.g. ANDI+BNEZ => BEXTI+BNEZ). - if (!Subtarget.hasStdExtZbs() && !Subtarget.hasVendorXTHeadBs()) + if (!Subtarget.hasBEXTILike()) return false; ConstantInt *Mask = dyn_cast<ConstantInt>(AndI.getOperand(1)); if (!Mask) @@ -9141,9 +9142,7 @@ static SDValue lowerSelectToBinOp(SDNode *N, SelectionDAG &DAG, return DAG.getNode(ISD::OR, DL, VT, Neg, DAG.getFreeze(TrueV)); } - const bool HasCZero = - VT.isScalarInteger() && - (Subtarget.hasStdExtZicond() || Subtarget.hasVendorXVentanaCondOps()); + const bool HasCZero = VT.isScalarInteger() && Subtarget.hasCZEROLike(); // (select c, 0, y) -> (c-1) & y if (isNullConstant(TrueV) && (!HasCZero || isSimm12Constant(FalseV))) { @@ -9281,8 +9280,7 @@ SDValue RISCVTargetLowering::lowerSELECT(SDValue Op, SelectionDAG &DAG) const { // nodes to implement the SELECT. Performing the lowering here allows for // greater control over when CZERO_{EQZ/NEZ} are used vs another branchless // sequence or RISCVISD::SELECT_CC node (branch-based select). - if ((Subtarget.hasStdExtZicond() || Subtarget.hasVendorXVentanaCondOps()) && - VT.isScalarInteger()) { + if (Subtarget.hasCZEROLike() && VT.isScalarInteger()) { // (select c, t, 0) -> (czero_eqz t, c) if (isNullConstant(FalseV)) @@ -15487,9 +15485,7 @@ static SDValue combineSelectAndUse(SDNode *N, SDValue Slct, SDValue OtherOp, if (!Subtarget.hasConditionalMoveFusion()) { // (select cond, x, (and x, c)) has custom lowering with Zicond. - if ((!Subtarget.hasStdExtZicond() && - !Subtarget.hasVendorXVentanaCondOps()) || - N->getOpcode() != ISD::AND) + if (!Subtarget.hasCZEROLike() || N->getOpcode() != ISD::AND) return SDValue(); // Maybe harmful when condition code has multiple use. @@ -16257,8 +16253,8 @@ static SDValue performXORCombine(SDNode *N, SelectionDAG &DAG, SDValue Op0 = DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i64, N0.getOperand(0)); SDValue Op1 = DAG.getNode(ISD::ZERO_EXTEND, DL, MVT::i64, N0.getOperand(1)); SDValue Shl = DAG.getNode(ISD::SHL, DL, MVT::i64, Op0, Op1); - SDValue And = DAG.getNOT(DL, Shl, MVT::i64); - return DAG.getNode(ISD::TRUNCATE, DL, MVT::i32, And); + SDValue Not = DAG.getNOT(DL, Shl, MVT::i64); + return DAG.getNode(ISD::TRUNCATE, DL, MVT::i32, Not); } // fold (xor (sllw 1, x), -1) -> (rolw ~1, x) @@ -16761,29 +16757,37 @@ static SDValue performSETCCCombine(SDNode *N, combineVectorSizedSetCCEquality(VT, N0, N1, Cond, dl, DAG, Subtarget)) return V; - if (DCI.isAfterLegalizeDAG() && isNullConstant(N1) && + if (DCI.isAfterLegalizeDAG() && isa<ConstantSDNode>(N1) && N0.getOpcode() == ISD::AND && N0.hasOneUse() && isa<ConstantSDNode>(N0.getOperand(1))) { - const APInt &AndRHSC = - cast<ConstantSDNode>(N0.getOperand(1))->getAPIntValue(); + const APInt &AndRHSC = N0.getConstantOperandAPInt(1); // (X & -(1 << C)) == 0 -> (X >> C) == 0 if the AND constant can't use ANDI. - if (!isInt<12>(AndRHSC.getSExtValue()) && AndRHSC.isNegatedPowerOf2()) { + if (isNullConstant(N1) && !isInt<12>(AndRHSC.getSExtValue()) && + AndRHSC.isNegatedPowerOf2()) { unsigned ShiftBits = AndRHSC.countr_zero(); SDValue Shift = DAG.getNode(ISD::SRL, dl, OpVT, N0.getOperand(0), DAG.getConstant(ShiftBits, dl, OpVT)); return DAG.getSetCC(dl, VT, Shift, N1, Cond); } - // Similar to above but handling the lower 32 bits by using srliw. - // FIXME: Handle the case where N1 is non-zero. - if (OpVT == MVT::i64 && AndRHSC.getZExtValue() <= 0xffffffff && - isPowerOf2_32(-uint32_t(AndRHSC.getZExtValue()))) { - unsigned ShiftBits = llvm::countr_zero(AndRHSC.getZExtValue()); - SDValue And = DAG.getNode(ISD::AND, dl, OpVT, N0.getOperand(0), - DAG.getConstant(0xffffffff, dl, OpVT)); - SDValue Shift = DAG.getNode(ISD::SRL, dl, OpVT, And, - DAG.getConstant(ShiftBits, dl, OpVT)); - return DAG.getSetCC(dl, VT, Shift, N1, Cond); + // Similar to above but handling the lower 32 bits by using sraiw. Allow + // comparing with constants other than 0 if the constant can be folded into + // addi or xori after shifting. + uint64_t N1Int = cast<ConstantSDNode>(N1)->getZExtValue(); + uint64_t AndRHSInt = AndRHSC.getZExtValue(); + if (OpVT == MVT::i64 && AndRHSInt <= 0xffffffff && + isPowerOf2_32(-uint32_t(AndRHSInt)) && (N1Int & AndRHSInt) == N1Int) { + unsigned ShiftBits = llvm::countr_zero(AndRHSInt); + int64_t NewC = SignExtend64<32>(N1Int) >> ShiftBits; + if (NewC >= -2048 && NewC <= 2048) { + SDValue SExt = + DAG.getNode(ISD::SIGN_EXTEND_INREG, dl, OpVT, N0.getOperand(0), + DAG.getValueType(MVT::i32)); + SDValue Shift = DAG.getNode(ISD::SRA, dl, OpVT, SExt, + DAG.getConstant(ShiftBits, dl, OpVT)); + return DAG.getSetCC(dl, VT, Shift, + DAG.getSignedConstant(NewC, dl, OpVT), Cond); + } } } @@ -18945,7 +18949,7 @@ static SDValue useInversedSetcc(SDNode *N, SelectionDAG &DAG, // Replace (setcc eq (and x, C)) with (setcc ne (and x, C))) to generate // BEXTI, where C is power of 2. if (Subtarget.hasStdExtZbs() && VT.isScalarInteger() && - (Subtarget.hasStdExtZicond() || Subtarget.hasVendorXVentanaCondOps())) { + (Subtarget.hasCZEROLike() || Subtarget.hasVendorXTHeadCondMov())) { SDValue LHS = Cond.getOperand(0); SDValue RHS = Cond.getOperand(1); ISD::CondCode CC = cast<CondCodeSDNode>(Cond.getOperand(2))->get(); @@ -19120,6 +19124,7 @@ static SDValue foldReduceOperandViaVQDOT(SDValue InVec, const SDLoc &DL, SelectionDAG &DAG, const RISCVSubtarget &Subtarget, const RISCVTargetLowering &TLI) { + using namespace SDPatternMatch; // Note: We intentionally do not check the legality of the reduction type. // We want to handle the m4/m8 *src* types, and thus need to let illegal // intermediate types flow through here. @@ -19127,11 +19132,10 @@ static SDValue foldReduceOperandViaVQDOT(SDValue InVec, const SDLoc &DL, !InVec.getValueType().getVectorElementCount().isKnownMultipleOf(4)) return SDValue(); - // Recurse through adds (since generic dag canonicalizes to that - // form). TODO: Handle disjoint or here. - if (InVec->getOpcode() == ISD::ADD) { - SDValue A = InVec.getOperand(0); - SDValue B = InVec.getOperand(1); + // Recurse through adds/disjoint ors (since generic dag canonicalizes to that + // form). + SDValue A, B; + if (sd_match(InVec, m_AddLike(m_Value(A), m_Value(B)))) { SDValue AOpt = foldReduceOperandViaVQDOT(A, DL, DAG, Subtarget, TLI); SDValue BOpt = foldReduceOperandViaVQDOT(B, DL, DAG, Subtarget, TLI); if (AOpt || BOpt) { @@ -19168,12 +19172,9 @@ static SDValue foldReduceOperandViaVQDOT(SDValue InVec, const SDLoc &DL, // mul (zext a, zext b) -> partial_reduce_umla 0, a, b // mul (sext a, zext b) -> partial_reduce_ssmla 0, a, b // mul (zext a, sext b) -> partial_reduce_smla 0, b, a (swapped) - if (InVec.getOpcode() != ISD::MUL) + if (!sd_match(InVec, m_Mul(m_Value(A), m_Value(B)))) return SDValue(); - SDValue A = InVec.getOperand(0); - SDValue B = InVec.getOperand(1); - if (!ISD::isExtOpcode(A.getOpcode())) return SDValue(); @@ -24841,6 +24842,7 @@ bool RISCVTargetLowering::isCtpopFast(EVT VT) const { return isTypeLegal(VT) && Subtarget.hasStdExtZvbb(); if (VT.isFixedLengthVector() && Subtarget.hasStdExtZvbb()) return true; + // FIXME: Should use hasCPOPLike here. return Subtarget.hasStdExtZbb() && (VT == MVT::i32 || VT == MVT::i64 || VT.isFixedLengthVector()); } @@ -24890,6 +24892,12 @@ bool RISCVTargetLowering::fallBackToDAGISel(const Instruction &Inst) const { Op == Instruction::Freeze || Op == Instruction::Store) return false; + if (auto *II = dyn_cast<IntrinsicInst>(&Inst)) { + // Mark RVV intrinsic as supported. + if (RISCVVIntrinsicsTable::getRISCVVIntrinsicInfo(II->getIntrinsicID())) + return false; + } + if (Inst.getType()->isScalableTy()) return true; @@ -24929,7 +24937,7 @@ RISCVTargetLowering::BuildSDIVPow2(SDNode *N, const APInt &Divisor, bool RISCVTargetLowering::shouldFoldSelectWithSingleBitTest( EVT VT, const APInt &AndMask) const { - if (Subtarget.hasStdExtZicond() || Subtarget.hasVendorXVentanaCondOps()) + if (Subtarget.hasCZEROLike()) return !Subtarget.hasStdExtZbs() && AndMask.ugt(1024); return TargetLowering::shouldFoldSelectWithSingleBitTest(VT, AndMask); } diff --git a/llvm/lib/Target/RISCV/RISCVInstrInfo.cpp b/llvm/lib/Target/RISCV/RISCVInstrInfo.cpp index f2e6e8959a72..d0bb57a3eaa1 100644 --- a/llvm/lib/Target/RISCV/RISCVInstrInfo.cpp +++ b/llvm/lib/Target/RISCV/RISCVInstrInfo.cpp @@ -3511,6 +3511,9 @@ RISCVInstrInfo::getOutliningTypeImpl(const MachineModuleInfo &MMI, return outliner::InstrType::Illegal; } + if (isLPAD(MI)) + return outliner::InstrType::Illegal; + return outliner::InstrType::Legal; } diff --git a/llvm/lib/Target/RISCV/RISCVInstrInfo.td b/llvm/lib/Target/RISCV/RISCVInstrInfo.td index 4b6870554ab6..92552b36aa0b 100644 --- a/llvm/lib/Target/RISCV/RISCVInstrInfo.td +++ b/llvm/lib/Target/RISCV/RISCVInstrInfo.td @@ -2328,7 +2328,6 @@ include "RISCVInstrInfoZalasr.td" include "RISCVInstrInfoZimop.td" include "RISCVInstrInfoZicbo.td" include "RISCVInstrInfoZicond.td" -include "RISCVInstrInfoZicfiss.td" include "RISCVInstrInfoZilsd.td" // Scalar FP @@ -2357,6 +2356,9 @@ include "RISCVInstrInfoZc.td" include "RISCVInstrInfoZcmop.td" include "RISCVInstrInfoZclsd.td" +// Control Flow Integriy, this requires Zimop/Zcmop +include "RISCVInstrInfoZicfiss.td" + // Short Forward Branch include "RISCVInstrInfoSFB.td" diff --git a/llvm/lib/Target/RISCV/RISCVInstrInfoP.td b/llvm/lib/Target/RISCV/RISCVInstrInfoP.td index c342b41e41d0..6840dacaea54 100644 --- a/llvm/lib/Target/RISCV/RISCVInstrInfoP.td +++ b/llvm/lib/Target/RISCV/RISCVInstrInfoP.td @@ -25,7 +25,7 @@ def SImm8UnsignedAsmOperand : SImmAsmOperand<8, "Unsigned"> { } // A 8-bit signed immediate allowing range [-128, 255] -// but represented as [-128, 255]. +// but represented as [-128, 127]. def simm8_unsigned : RISCVOp { let ParserMatchClass = SImm8UnsignedAsmOperand; let EncoderMethod = "getImmOpValue"; @@ -98,6 +98,40 @@ class PLUI_i<bits<7> funct7, string opcodestr> let Inst{23-15} = imm10{9-1}; } +// Common base for widening Binary/Ternary ops +class RVPWideningBase<bits<2> w, bit arith_shift, dag outs, dag ins, + string opcodestr> + : RVInst<outs, ins, opcodestr, "$rd, $rs1, $rs2", [], InstFormatOther> { + bits<5> rs2; + bits<5> rs1; + bits<5> rd; + + let Inst{31} = 0b0; + let Inst{26-25} = w; + let Inst{24-20} = rs2; + let Inst{19-15} = rs1; + let Inst{14-12} = 0b010; + let Inst{11-8} = rd{4-1}; + let Inst{7} = arith_shift; + let Inst{6-0} = OPC_OP_IMM_32.Value; +} + +// Common base for narrowing ops +class RVPNarrowingBase<bits<3> f, bit r, bits<4> funct4, dag outs, dag ins, + string opcodestr, string argstr> + : RVInst<outs, ins, opcodestr, argstr, [], InstFormatOther> { + bits<5> rs1; + bits<5> rd; + + let Inst{31} = 0b0; + let Inst{30-28} = f; + let Inst{27} = r; + let Inst{19-16} = rs1{4-1}; + let Inst{15-12} = funct4; + let Inst{11-7} = rd; + let Inst{6-0} = OPC_OP_IMM_32.Value; +} + let hasSideEffects = 0, mayLoad = 0, mayStore = 0 in class RVPShift_ri<bits<3> f, bits<3> funct3, string opcodestr, Operand ImmType> : RVInstIBase<funct3, OPC_OP_IMM_32, (outs GPR:$rd), @@ -141,6 +175,100 @@ class RVPShiftB_ri<bits<3> f, bits<3> funct3, string opcodestr> } let hasSideEffects = 0, mayLoad = 0, mayStore = 0 in +class RVPWideningShift_ri<bits<3> f, string opcodestr, Operand ImmType> + : RVInst<(outs GPRPairRV32:$rd), (ins GPR:$rs1, ImmType:$shamt), opcodestr, + "$rd, $rs1, $shamt", [], InstFormatOther> { + bits<5> rs1; + bits<5> rd; + + let Inst{31} = 0b0; + let Inst{30-28} = f; + let Inst{27} = 0b0; + let Inst{19-15} = rs1; + let Inst{14-12} = 0b010; + let Inst{11-8} = rd{4-1}; + let Inst{7} = 0b0; + let Inst{6-0} = OPC_OP_IMM_32.Value; + + let hasSideEffects = 0; + let mayLoad = 0; + let mayStore = 0; +} + +class RVPWideningShiftW_ri<bits<3> f, string opcodestr> + : RVPWideningShift_ri<f, opcodestr, uimm6> { + bits<6> shamt; + + let Inst{26} = 0b1; + let Inst{25-20} = shamt; +} + +class RVPWideningShiftH_ri<bits<3> f, string opcodestr> + : RVPWideningShift_ri<f, opcodestr, uimm5> { + bits<5> shamt; + + let Inst{26-25} = 0b01; + let Inst{24-20} = shamt; +} + +class RVPWideningShiftB_ri<bits<3> f, string opcodestr> + : RVPWideningShift_ri<f, opcodestr, uimm4> { + bits<4> shamt; + + let Inst{26-24} = 0b001; + let Inst{23-20} = shamt; +} + +let hasSideEffects = 0, mayLoad = 0, mayStore = 0 in +class RVPNarrowingShift_ri<bits<3> f, string opcodestr, Operand ImmType> + : RVPNarrowingBase<f, 0b0, 0b1100, (outs GPR:$rd), + (ins GPRPairRV32:$rs1, ImmType:$shamt), opcodestr, + "$rd, $rs1, $shamt">; + +class RVPNarrowingShiftW_ri<bits<3> f, string opcodestr> + : RVPNarrowingShift_ri<f, opcodestr, uimm6> { + bits<6> shamt; + + let Inst{26} = 0b1; + let Inst{25-20} = shamt; +} + +class RVPNarrowingShiftH_ri<bits<3> f, string opcodestr> + : RVPNarrowingShift_ri<f, opcodestr, uimm5> { + bits<5> shamt; + + let Inst{26-25} = 0b01; + let Inst{24-20} = shamt; +} + +class RVPNarrowingShiftB_ri<bits<3> f, string opcodestr> + : RVPNarrowingShift_ri<f, opcodestr, uimm4> { + bits<4> shamt; + + let Inst{26-24} = 0b001; + let Inst{23-20} = shamt; +} + +let hasSideEffects = 0, mayLoad = 0, mayStore = 0 in +class RVPNarrowingShift_rr<bits<3> f, bits<2> w, string opcodestr> + : RVPNarrowingBase<f, 0b1, 0b1100, (outs GPR:$rd), + (ins GPRPairRV32:$rs1, GPR:$rs2), opcodestr, + "$rd, $rs1, $rs2"> { + bits<5> rs2; + + let Inst{26-25} = w; + let Inst{24-20} = rs2; +} + +let hasSideEffects = 0, mayLoad = 0, mayStore = 0 in +class RVPWideningShift_rr<bits<3> f, bits<2> w, string opcodestr> + : RVPWideningBase<w, 0b0, (outs GPRPairRV32:$rd), (ins GPR:$rs1, GPR:$rs2), + opcodestr> { + let Inst{30-28} = f; + let Inst{27} = 0b1; +} + +let hasSideEffects = 0, mayLoad = 0, mayStore = 0 in class RVPUnary_ri<bits<2> w, bits<5> uf, string opcodestr> : RVInstIBase<0b010, OPC_OP_IMM_32, (outs GPR:$rd), (ins GPR:$rs1), opcodestr, "$rd, $rs1"> { @@ -169,6 +297,24 @@ class RVPBinary_rr<bits<4> f, bits<2> w, bits<3> funct3, string opcodestr> } let hasSideEffects = 0, mayLoad = 0, mayStore = 0 in +class RVPWideningBinary_rr<bits<4> f, bits<2> w, string opcodestr> + : RVPWideningBase<w, 0b1, (outs GPRPairRV32:$rd), (ins GPR:$rs1, GPR:$rs2), + opcodestr> { + let Inst{30-27} = f; +} + +let hasSideEffects = 0, mayLoad = 0, mayStore = 0 in +class RVPNarrowingBinary_rr<bits<3> f, bits<2> w, string opcodestr> + : RVPNarrowingBase<f, 0b1, 0b0100, (outs GPR:$rd), + (ins GPRPairRV32:$rs1, GPR:$rs2), opcodestr, + "$rd, $rs1, $rs2"> { + bits<5> rs2; + + let Inst{26-25} = w; + let Inst{24-20} = rs2; +} + +let hasSideEffects = 0, mayLoad = 0, mayStore = 0 in class RVPTernary_rrr<bits<4> f, bits<2> w, bits<3> funct3, string opcodestr> : RVInstRBase<funct3, OPC_OP_32, (outs GPR:$rd_wb), (ins GPR:$rd, GPR:$rs1, GPR:$rs2), opcodestr, @@ -180,6 +326,15 @@ class RVPTernary_rrr<bits<4> f, bits<2> w, bits<3> funct3, string opcodestr> let Constraints = "$rd = $rd_wb"; } +let hasSideEffects = 0, mayLoad = 0, mayStore = 0 in +class RVPWideningTernary_rrr<bits<4> f, bits<2> w, string opcodestr> + : RVPWideningBase<w, 0b1, (outs GPRPairRV32:$rd_wb), + (ins GPR:$rd, GPR:$rs1, GPR:$rs2), opcodestr> { + let Inst{30-27} = f; + + let Constraints = "$rd = $rd_wb"; +} + // Common base for pli.db/h/w and plui.dh/w class RVPPairLoadImm_i<bits<7> funct7, dag ins, string opcodestr, string argstr> @@ -889,3 +1044,156 @@ let Predicates = [HasStdExtP, IsRV32] in { let Inst{23-15} = imm10{9-1}; } } + +let Predicates = [HasStdExtP, IsRV32] in { + def PWSLLI_B : RVPWideningShiftB_ri<0b000, "pwslli.b">; + def PWSLLI_H : RVPWideningShiftH_ri<0b000, "pwslli.h">; + def WSLLI : RVPWideningShiftW_ri<0b000, "wslli">; + + def PWSLAI_B : RVPWideningShiftB_ri<0b100, "pwslai.b">; + def PWSLAI_H : RVPWideningShiftH_ri<0b100, "pwslai.h">; + def WSLAI : RVPWideningShiftW_ri<0b100, "wslai">; + + def PWSLL_BS : RVPWideningShift_rr<0b000, 0b00, "pwsll.bs">; + def PWSLL_HS : RVPWideningShift_rr<0b000, 0b01, "pwsll.hs">; + def WSLL : RVPWideningShift_rr<0b000, 0b11, "wsll">; + + def PWSLA_BS : RVPWideningShift_rr<0b100, 0b00, "pwsla.bs">; + def PWSLA_HS : RVPWideningShift_rr<0b100, 0b01, "pwsla.hs">; + def WSLA : RVPWideningShift_rr<0b100, 0b11, "wsla">; + + def WZIP8P : RVPWideningShift_rr<0b111, 0b00, "wzip8p">; + def WZIP16P : RVPWideningShift_rr<0b111, 0b01, "wzip16p">; + + def PWADD_H : RVPWideningBinary_rr<0b0000, 0b00, "pwadd.h">; + def WADD : RVPWideningBinary_rr<0b0000, 0b01, "wadd">; + def PWADD_B : RVPWideningBinary_rr<0b0000, 0b10, "pwadd.b">; + def PM2WADD_H : RVPWideningBinary_rr<0b0000, 0b11, "pm2wadd.h">; + + def PWADDA_H : RVPWideningTernary_rrr<0b0001, 0b00, "pwadda.h">; + def WADDA : RVPWideningTernary_rrr<0b0001, 0b01, "wadda">; + def PWADDA_B : RVPWideningTernary_rrr<0b0001, 0b10, "pwadda.b">; + def PM2WADDA_H : RVPWideningTernary_rrr<0b0001, 0b11, "pm2wadda.h">; + + def PWADDU_H : RVPWideningBinary_rr<0b0010, 0b00, "pwaddu.h">; + def WADDU : RVPWideningBinary_rr<0b0010, 0b01, "waddu">; + def PWADDU_B : RVPWideningBinary_rr<0b0010, 0b10, "pwaddu.b">; + def PM2WADD_HX : RVPWideningBinary_rr<0b0010, 0b11, "pm2wadd.hx">; + + def PWADDAU_H : RVPWideningTernary_rrr<0b0011, 0b00, "pwaddau.h">; + def WADDAU : RVPWideningTernary_rrr<0b0011, 0b01, "waddau">; + def PWADDAU_B : RVPWideningTernary_rrr<0b0011, 0b10, "pwaddau.b">; + def PM2WADDA_HX : RVPWideningTernary_rrr<0b0011, 0b11, "pm2wadda.hx">; + + def PWMUL_H : RVPWideningBinary_rr<0b0100, 0b00, "pwmul.h">; + def WMUL : RVPWideningBinary_rr<0b0100, 0b01, "wmul">; + def PWMUL_B : RVPWideningBinary_rr<0b0100, 0b10, "pwmul.b">; + def PM2WADDU_H : RVPWideningBinary_rr<0b0100, 0b11, "pm2waddu.h">; + + def PWMACC_H : RVPWideningTernary_rrr<0b0101, 0b00, "pwmacc.h">; + def WMACC : RVPWideningTernary_rrr<0b0101, 0b01, "wmacc">; + def PM2WADDAU_H : RVPWideningTernary_rrr<0b0101, 0b11, "pm2waddau.h">; + + def PWMULU_H : RVPWideningBinary_rr<0b0110, 0b00, "pwmulu.h">; + def WMULU : RVPWideningBinary_rr<0b0110, 0b01, "wmulu">; + def PWMULU_B : RVPWideningBinary_rr<0b0110, 0b10, "pwmulu.b">; + + def PWMACCU_H : RVPWideningTernary_rrr<0b0111, 0b00, "pwmaccu.h">; + def WMACCU : RVPWideningTernary_rrr<0b0111, 0b01, "wmaccu">; + + def PWSUB_H : RVPWideningBinary_rr<0b1000, 0b00, "pwsub.h">; + def WSUB : RVPWideningBinary_rr<0b1000, 0b01, "wsub">; + def PWSUB_B : RVPWideningBinary_rr<0b1000, 0b10, "pwsub.b">; + def PM2WSUB_H : RVPWideningBinary_rr<0b1000, 0b11, "pm2wsub.h">; + + def PWSUBA_H : RVPWideningTernary_rrr<0b1001, 0b00, "pwsuba.h">; + def WSUBA : RVPWideningTernary_rrr<0b1001, 0b01, "wsuba">; + def PWSUBA_B : RVPWideningTernary_rrr<0b1001, 0b10, "pwsuba.b">; + def PM2WSUBA_H : RVPWideningTernary_rrr<0b1001, 0b11, "pm2wsuba.h">; + + def PWSUBU_H : RVPWideningBinary_rr<0b1010, 0b00, "pwsubu.h">; + def WSUBU : RVPWideningBinary_rr<0b1010, 0b01, "wsubu">; + def PWSUBU_B : RVPWideningBinary_rr<0b1010, 0b10, "pwsubu.b">; + def PM2WSUB_HX : RVPWideningBinary_rr<0b1010, 0b11, "pm2wsub.hx">; + + def PWSUBAU_H : RVPWideningTernary_rrr<0b1011, 0b00, "pwsubau.h">; + def WSUBAU : RVPWideningTernary_rrr<0b1011, 0b01, "wsubau">; + def PWSUBAU_B : RVPWideningTernary_rrr<0b1011, 0b10, "pwsubau.b">; + def PM2WSUBA_HX : RVPWideningTernary_rrr<0b1011, 0b11, "pm2wsuba.hx">; + + def PWMULSU_H : RVPWideningBinary_rr<0b1100, 0b00, "pwmulsu.h">; + def WMULSU : RVPWideningBinary_rr<0b1100, 0b01, "wmulsu">; + def PWMULSU_B : RVPWideningBinary_rr<0b1100, 0b10, "pwmulsu.b">; + def PM2WADDSU_H : RVPWideningBinary_rr<0b1100, 0b11, "pm2waddsu.h">; + + def PWMACCSU_H : RVPWideningTernary_rrr<0b1101, 0b00, "pwmaccsu.h">; + def WMACCSU : RVPWideningTernary_rrr<0b1101, 0b01, "wmaccsu">; + def PM2WADDASU_H : RVPWideningTernary_rrr<0b1101, 0b11, "pm2waddasu.h">; + + def PMQWACC_H : RVPWideningTernary_rrr<0b1111, 0b00, "pmqwacc.h">; + def PMQWACC : RVPWideningTernary_rrr<0b1111, 0b01, "pmqwacc">; + def PMQRWACC_H : RVPWideningTernary_rrr<0b1111, 0b10, "pmqrwacc.h">; + def PMQRWACC : RVPWideningTernary_rrr<0b1111, 0b11, "pmqrwacc">; + + def PREDSUM_DHS : RVPNarrowingBinary_rr<0b001, 0b00, "predsum.dhs">; + def PREDSUM_DBS : RVPNarrowingBinary_rr<0b001, 0b10, "predsum.dbs">; + + def PREDSUMU_DHS : RVPNarrowingBinary_rr<0b011, 0b00, "predsumu.dhs">; + def PREDSUMU_DBS : RVPNarrowingBinary_rr<0b011, 0b10, "predsumu.dbs">; + + def PNSRLI_B : RVPNarrowingShiftB_ri<0b000, "pnsrli.b">; + def PNSRLI_H : RVPNarrowingShiftH_ri<0b000, "pnsrli.h">; + def NSRLI : RVPNarrowingShiftW_ri<0b000, "nsrli">; + + def PNCLIPIU_B : RVPNarrowingShiftB_ri<0b010, "pnclipiu.b">; + def PNCLIPIU_H : RVPNarrowingShiftH_ri<0b010, "pnclipiu.h">; + def NCLIPIU : RVPNarrowingShiftW_ri<0b010, "nclipiu">; + + def PNCLIPRIU_B : RVPNarrowingShiftB_ri<0b011, "pnclipriu.b">; + def PNCLIPRIU_H : RVPNarrowingShiftH_ri<0b011, "pnclipriu.h">; + def NCLIPRIU : RVPNarrowingShiftW_ri<0b011, "nclipriu">; + + def PNSRAI_B : RVPNarrowingShiftB_ri<0b100, "pnsrai.b">; + def PNSRAI_H : RVPNarrowingShiftH_ri<0b100, "pnsrai.h">; + def NSRAI : RVPNarrowingShiftW_ri<0b100, "nsrai">; + + def PNSARI_B : RVPNarrowingShiftB_ri<0b101, "pnsari.b">; + def PNSARI_H : RVPNarrowingShiftH_ri<0b101, "pnsari.h">; + def NSARI : RVPNarrowingShiftW_ri<0b101, "nsari">; + + def PNCLIPI_B : RVPNarrowingShiftB_ri<0b110, "pnclipi.b">; + def PNCLIPI_H : RVPNarrowingShiftH_ri<0b110, "pnclipi.h">; + def NCLIPI : RVPNarrowingShiftW_ri<0b110, "nclipi">; + + def PNCLIPRI_B : RVPNarrowingShiftB_ri<0b111, "pnclipri.b">; + def PNCLIPRI_H : RVPNarrowingShiftH_ri<0b111, "pnclipri.h">; + def NCLIPRI : RVPNarrowingShiftW_ri<0b111, "nclipri">; + + def PNSRL_BS : RVPNarrowingShift_rr<0b000, 0b00, "pnsrl.bs">; + def PNSRL_HS : RVPNarrowingShift_rr<0b000, 0b01, "pnsrl.hs">; + def NSRL : RVPNarrowingShift_rr<0b000, 0b11, "nsrl">; + + def PNCLIPU_BS : RVPNarrowingShift_rr<0b010, 0b00, "pnclipu.bs">; + def PNCLIPU_HS : RVPNarrowingShift_rr<0b010, 0b01, "pnclipu.hs">; + def NCLIPU : RVPNarrowingShift_rr<0b010, 0b11, "nclipu">; + + def PNCLIPRU_BS : RVPNarrowingShift_rr<0b011, 0b00, "pnclipru.bs">; + def PNCLIPRU_HS : RVPNarrowingShift_rr<0b011, 0b01, "pnclipru.hs">; + def NCLIPRU : RVPNarrowingShift_rr<0b011, 0b11, "nclipru">; + + def PNSRA_BS : RVPNarrowingShift_rr<0b100, 0b00, "pnsra.bs">; + def PNSRA_HS : RVPNarrowingShift_rr<0b100, 0b01, "pnsra.hs">; + def NSRA : RVPNarrowingShift_rr<0b100, 0b11, "nsra">; + + def PNSRAR_BS : RVPNarrowingShift_rr<0b101, 0b00, "pnsrar.bs">; + def PNSRAR_HS : RVPNarrowingShift_rr<0b101, 0b01, "pnsrar.hs">; + def NSRAR : RVPNarrowingShift_rr<0b101, 0b11, "nsrar">; + + def PNCLIP_BS : RVPNarrowingShift_rr<0b110, 0b00, "pnclip.bs">; + def PNCLIP_HS : RVPNarrowingShift_rr<0b110, 0b01, "pnclip.hs">; + def NCLIP : RVPNarrowingShift_rr<0b110, 0b11, "nclip">; + + def PNCLIPR_BS : RVPNarrowingShift_rr<0b111, 0b00, "pnclipr.bs">; + def PNCLIPR_HS : RVPNarrowingShift_rr<0b111, 0b01, "pnclipr.hs">; + def NCLIPR : RVPNarrowingShift_rr<0b111, 0b11, "nclipr">; +} // Predicates = [HasStdExtP, IsRV32] diff --git a/llvm/lib/Target/RISCV/RISCVInstrInfoXTHead.td b/llvm/lib/Target/RISCV/RISCVInstrInfoXTHead.td index 4c31ce43c2e5..49c9bdd83d3f 100644 --- a/llvm/lib/Target/RISCV/RISCVInstrInfoXTHead.td +++ b/llvm/lib/Target/RISCV/RISCVInstrInfoXTHead.td @@ -594,8 +594,6 @@ def : Pat<(i64 (sra (bswap GPR:$rs1), (i64 32))), (TH_REVW GPR:$rs1)>; def : Pat<(binop_allwusers<srl> (bswap GPR:$rs1), (i64 32)), (TH_REVW GPR:$rs1)>; -def : Pat<(riscv_clzw GPR:$rs1), - (TH_FF0 (i64 (SLLI (i64 (XORI GPR:$rs1, -1)), 32)))>; } // Predicates = [HasVendorXTHeadBb, IsRV64] let Predicates = [HasVendorXTHeadBs] in { diff --git a/llvm/lib/Target/RISCV/RISCVInstrInfoZb.td b/llvm/lib/Target/RISCV/RISCVInstrInfoZb.td index 2abd3e613a03..a2b4302e19ed 100644 --- a/llvm/lib/Target/RISCV/RISCVInstrInfoZb.td +++ b/llvm/lib/Target/RISCV/RISCVInstrInfoZb.td @@ -459,15 +459,15 @@ let Predicates = [HasStdExtZba, IsRV64] in { def : InstAlias<"zext.w $rd, $rs", (ADD_UW GPR:$rd, GPR:$rs, X0)>; } // Predicates = [HasStdExtZba, IsRV64] -let Predicates = [HasStdExtZbb] in { +let Predicates = [HasStdExtZbbOrZbkb] in { def : InstAlias<"ror $rd, $rs1, $shamt", - (RORI GPR:$rd, GPR:$rs1, uimmlog2xlen:$shamt), 0>; -} // Predicates = [HasStdExtZbb] + (RORI GPR:$rd, GPR:$rs1, uimmlog2xlen:$shamt), 0>; +} // Predicates = [HasStdExtZbbOrZbkb] -let Predicates = [HasStdExtZbb, IsRV64] in { +let Predicates = [HasStdExtZbbOrZbkb, IsRV64] in { def : InstAlias<"rorw $rd, $rs1, $shamt", - (RORIW GPR:$rd, GPR:$rs1, uimm5:$shamt), 0>; -} // Predicates = [HasStdExtZbb, IsRV64] + (RORIW GPR:$rd, GPR:$rs1, uimm5:$shamt), 0>; +} // Predicates = [HasStdExtZbbOrZbkb, IsRV64] let Predicates = [HasStdExtZbs] in { def : InstAlias<"bset $rd, $rs1, $shamt", diff --git a/llvm/lib/Target/RISCV/RISCVInstrInfoZicfiss.td b/llvm/lib/Target/RISCV/RISCVInstrInfoZicfiss.td index 49a57f86cccd..50ebaa995197 100644 --- a/llvm/lib/Target/RISCV/RISCVInstrInfoZicfiss.td +++ b/llvm/lib/Target/RISCV/RISCVInstrInfoZicfiss.td @@ -62,6 +62,21 @@ defm SSAMOSWAP_W : AMO_rr_aq_rl<0b01001, 0b010, "ssamoswap.w">; let Predicates = [HasStdExtZicfiss, IsRV64] in defm SSAMOSWAP_D : AMO_rr_aq_rl<0b01001, 0b011, "ssamoswap.d">; +let Predicates = [HasStdExtZimop] in { +let hasSideEffects = 1, mayLoad = 0, mayStore = 1 in +def PseudoMOP_SSPUSH : Pseudo<(outs), (ins GPRX1X5:$rs2), []>, + PseudoInstExpansion<(MOP_RR_7 X0, X0, GPR:$rs2)>; +let hasSideEffects = 1, mayLoad = 1, mayStore = 0 in +def PseudoMOP_SSPOPCHK : Pseudo<(outs), (ins GPRX1X5:$rs1), []>, + PseudoInstExpansion<(MOP_R_28 X0, GPR:$rs1)>; +} // Predicates = [HasStdExtZimop] + +let Predicates = [HasStdExtZcmop] in { +let Uses = [X1], hasSideEffects = 1, mayLoad = 0, mayStore = 1 in +def PseudoMOP_C_SSPUSH : Pseudo<(outs), (ins), []>, + PseudoInstExpansion<(C_MOP_1)>; +} // Predicates = [HasStdExtZcmop] + //===----------------------------------------------------------------------===/ // Compress Instruction tablegen backend. //===----------------------------------------------------------------------===// diff --git a/llvm/lib/Target/RISCV/RISCVInstrPredicates.td b/llvm/lib/Target/RISCV/RISCVInstrPredicates.td index 4abe62f4e874..06309262f1b0 100644 --- a/llvm/lib/Target/RISCV/RISCVInstrPredicates.td +++ b/llvm/lib/Target/RISCV/RISCVInstrPredicates.td @@ -148,6 +148,14 @@ def isNonZeroLoadImmediate CheckNot<CheckImmOperand<2, 0>> ]>>>; +def isLPAD + : TIIPredicate<"isLPAD", + MCReturnStatement<CheckAll<[ + CheckOpcode<[AUIPC]>, + CheckIsRegOperand<0>, + CheckRegOperand<0, X0>, + ]>>>; + def ignoresVXRM : TIIPredicate<"ignoresVXRM", MCOpcodeSwitchStatement< diff --git a/llvm/lib/Target/RISCV/RISCVOptWInstrs.cpp b/llvm/lib/Target/RISCV/RISCVOptWInstrs.cpp index badc11111872..d08115b72977 100644 --- a/llvm/lib/Target/RISCV/RISCVOptWInstrs.cpp +++ b/llvm/lib/Target/RISCV/RISCVOptWInstrs.cpp @@ -356,6 +356,14 @@ static bool hasAllNBitUsers(const MachineInstr &OrigMI, return false; Worklist.emplace_back(UserMI, Bits); break; + case RISCV::TH_EXT: + case RISCV::TH_EXTU: + unsigned Msb = UserMI->getOperand(2).getImm(); + unsigned Lsb = UserMI->getOperand(3).getImm(); + // Behavior of Msb < Lsb is not well documented. + if (Msb >= Lsb && Bits > Msb) + break; + return false; } } } diff --git a/llvm/lib/Target/RISCV/RISCVRegisterInfo.td b/llvm/lib/Target/RISCV/RISCVRegisterInfo.td index ccb39e8f4291..82e768d7c1d1 100644 --- a/llvm/lib/Target/RISCV/RISCVRegisterInfo.td +++ b/llvm/lib/Target/RISCV/RISCVRegisterInfo.td @@ -358,8 +358,7 @@ let RegAltNameIndices = [ABIRegAltName] in { } } -let RegInfos = XLenPairRI, CopyCost = 2, - DecoderMethod = "DecodeGPRPairRegisterClass" in { +let RegInfos = XLenPairRI, CopyCost = 2 in { def GPRPair : RISCVRegisterClass<[XLenPairVT, XLenPairFVT], 64, (add X10_X11, X12_X13, X14_X15, X16_X17, X6_X7, @@ -370,12 +369,11 @@ def GPRPair : RISCVRegisterClass<[XLenPairVT, XLenPairFVT], 64, (add )>; def GPRPairNoX0 : RISCVRegisterClass<[XLenPairVT, XLenPairFVT], 64, (sub GPRPair, X0_Pair)>; -} // let RegInfos = XLenPairRI, DecoderMethod = "DecodeGPRPairRegisterClass" -let RegInfos = XLenPairRI, CopyCost = 2 in def GPRPairC : RISCVRegisterClass<[XLenPairVT, XLenPairFVT], 64, (add X10_X11, X12_X13, X14_X15, X8_X9 )>; +} // let RegInfos = XLenPairRI, CopyCost = 2 //===----------------------------------------------------------------------===// // Floating Point registers diff --git a/llvm/lib/Target/RISCV/RISCVSubtarget.h b/llvm/lib/Target/RISCV/RISCVSubtarget.h index fd57e02c25d0..0d9cd16a7793 100644 --- a/llvm/lib/Target/RISCV/RISCVSubtarget.h +++ b/llvm/lib/Target/RISCV/RISCVSubtarget.h @@ -186,6 +186,23 @@ public: return HasStdExtZfhmin || HasStdExtZfbfmin; } + bool hasCLZLike() const { + return HasStdExtZbb || HasVendorXTHeadBb || + (HasVendorXCVbitmanip && !IsRV64); + } + bool hasCTZLike() const { + return HasStdExtZbb || (HasVendorXCVbitmanip && !IsRV64); + } + bool hasCPOPLike() const { + return HasStdExtZbb || (HasVendorXCVbitmanip && !IsRV64); + } + + bool hasBEXTILike() const { return HasStdExtZbs || HasVendorXTHeadBs; } + + bool hasCZEROLike() const { + return HasStdExtZicond || HasVendorXVentanaCondOps; + } + bool hasConditionalMoveFusion() const { // Do we support fusing a branch+mv or branch+c.mv as a conditional move. return (hasConditionalCompressedMoveFusion() && hasStdExtZca()) || diff --git a/llvm/lib/Target/RISCV/RISCVTargetMachine.cpp b/llvm/lib/Target/RISCV/RISCVTargetMachine.cpp index 460bb33f2553..a1ec24f1fe71 100644 --- a/llvm/lib/Target/RISCV/RISCVTargetMachine.cpp +++ b/llvm/lib/Target/RISCV/RISCVTargetMachine.cpp @@ -141,39 +141,6 @@ extern "C" LLVM_ABI LLVM_EXTERNAL_VISIBILITY void LLVMInitializeRISCVTarget() { initializeRISCVAsmPrinterPass(*PR); } -static std::string computeDataLayout(const Triple &TT, - const TargetOptions &Opts) { - std::string Ret; - - if (TT.isLittleEndian()) - Ret += "e"; - else - Ret += "E"; - - Ret += "-m:e"; - - // Pointer and integer sizes. - if (TT.isArch64Bit()) { - Ret += "-p:64:64-i64:64-i128:128"; - Ret += "-n32:64"; - } else { - assert(TT.isArch32Bit() && "only RV32 and RV64 are currently supported"); - Ret += "-p:32:32-i64:64"; - Ret += "-n32"; - } - - // Stack alignment based on ABI. - StringRef ABI = Opts.MCOptions.getABIName(); - if (ABI == "ilp32e") - Ret += "-S32"; - else if (ABI == "lp64e") - Ret += "-S64"; - else - Ret += "-S128"; - - return Ret; -} - static Reloc::Model getEffectiveRelocModel(const Triple &TT, std::optional<Reloc::Model> RM) { return RM.value_or(Reloc::Static); @@ -185,9 +152,10 @@ RISCVTargetMachine::RISCVTargetMachine(const Target &T, const Triple &TT, std::optional<Reloc::Model> RM, std::optional<CodeModel::Model> CM, CodeGenOptLevel OL, bool JIT) - : CodeGenTargetMachineImpl(T, computeDataLayout(TT, Options), TT, CPU, FS, - Options, getEffectiveRelocModel(TT, RM), - getEffectiveCodeModel(CM, CodeModel::Small), OL), + : CodeGenTargetMachineImpl( + T, TT.computeDataLayout(Options.MCOptions.getABIName()), TT, CPU, FS, + Options, getEffectiveRelocModel(TT, RM), + getEffectiveCodeModel(CM, CodeModel::Small), OL), TLOF(std::make_unique<RISCVELFTargetObjectFile>()) { initAsmInfo(); diff --git a/llvm/lib/Target/SPIRV/SPIRVSubtarget.cpp b/llvm/lib/Target/SPIRV/SPIRVSubtarget.cpp index 5b746a1389af..ba09692fec51 100644 --- a/llvm/lib/Target/SPIRV/SPIRVSubtarget.cpp +++ b/llvm/lib/Target/SPIRV/SPIRVSubtarget.cpp @@ -90,6 +90,10 @@ SPIRVSubtarget::SPIRVSubtarget(const Triple &TT, const std::string &CPU, else Env = Unknown; + // Set the default extensions based on the target triple. + if (TargetTriple.getVendor() == Triple::Intel) + Extensions.insert(SPIRV::Extension::SPV_INTEL_function_pointers); + // The order of initialization is important. initAvailableExtensions(Extensions); initAvailableExtInstSets(); diff --git a/llvm/lib/Target/SPIRV/SPIRVTargetMachine.cpp b/llvm/lib/Target/SPIRV/SPIRVTargetMachine.cpp index e0bfb77f4b53..9f6f9c722535 100644 --- a/llvm/lib/Target/SPIRV/SPIRVTargetMachine.cpp +++ b/llvm/lib/Target/SPIRV/SPIRVTargetMachine.cpp @@ -60,27 +60,6 @@ extern "C" LLVM_ABI LLVM_EXTERNAL_VISIBILITY void LLVMInitializeSPIRVTarget() { initializeSPIRVStripConvergentIntrinsicsPass(PR); } -static std::string computeDataLayout(const Triple &TT) { - const auto Arch = TT.getArch(); - // TODO: this probably needs to be revisited: - // Logical SPIR-V has no pointer size, so any fixed pointer size would be - // wrong. The choice to default to 32 or 64 is just motivated by another - // memory model used for graphics: PhysicalStorageBuffer64. But it shouldn't - // mean anything. - if (Arch == Triple::spirv32) - return "e-p:32:32-i64:64-v16:16-v24:32-v32:32-v48:64-v96:128-v192:256-" - "v256:256-v512:512-v1024:1024-n8:16:32:64-G1"; - if (Arch == Triple::spirv) - return "e-i64:64-v16:16-v24:32-v32:32-v48:64-v96:128-v192:256-v256:256-" - "v512:512-v1024:1024-n8:16:32:64-G10"; - if (TT.getVendor() == Triple::VendorType::AMD && - TT.getOS() == Triple::OSType::AMDHSA) - return "e-i64:64-v16:16-v24:32-v32:32-v48:64-v96:128-v192:256-v256:256-" - "v512:512-v1024:1024-n32:64-S32-G1-P4-A0"; - return "e-i64:64-v16:16-v24:32-v32:32-v48:64-v96:128-v192:256-v256:256-" - "v512:512-v1024:1024-n8:16:32:64-G1"; -} - static Reloc::Model getEffectiveRelocModel(std::optional<Reloc::Model> RM) { if (!RM) return Reloc::PIC_; @@ -96,7 +75,7 @@ SPIRVTargetMachine::SPIRVTargetMachine(const Target &T, const Triple &TT, std::optional<Reloc::Model> RM, std::optional<CodeModel::Model> CM, CodeGenOptLevel OL, bool JIT) - : CodeGenTargetMachineImpl(T, computeDataLayout(TT), TT, CPU, FS, Options, + : CodeGenTargetMachineImpl(T, TT.computeDataLayout(), TT, CPU, FS, Options, getEffectiveRelocModel(RM), getEffectiveCodeModel(CM, CodeModel::Small), OL), TLOF(std::make_unique<SPIRVTargetObjectFile>()), @@ -189,6 +168,11 @@ TargetPassConfig *SPIRVTargetMachine::createPassConfig(PassManagerBase &PM) { void SPIRVPassConfig::addIRPasses() { TargetPassConfig::addIRPasses(); + addPass(createSPIRVRegularizerPass()); + addPass(createSPIRVPrepareFunctionsPass(TM)); +} + +void SPIRVPassConfig::addISelPrepare() { if (TM.getSubtargetImpl()->isShader()) { // Vulkan does not allow address space casts. This pass is run to remove // address space casts that can be removed. @@ -220,12 +204,7 @@ void SPIRVPassConfig::addIRPasses() { addPass(createPromoteMemoryToRegisterPass()); } - addPass(createSPIRVRegularizerPass()); - addPass(createSPIRVPrepareFunctionsPass(TM)); addPass(createSPIRVStripConvergenceIntrinsicsPass()); -} - -void SPIRVPassConfig::addISelPrepare() { addPass(createSPIRVLegalizeImplicitBindingPass()); addPass(createSPIRVEmitIntrinsicsPass(&getTM<SPIRVTargetMachine>())); if (TM.getSubtargetImpl()->isLogicalSPIRV()) diff --git a/llvm/lib/Target/Sparc/SparcTargetMachine.cpp b/llvm/lib/Target/Sparc/SparcTargetMachine.cpp index 754c8f63ca4e..27ab57c11cf7 100644 --- a/llvm/lib/Target/Sparc/SparcTargetMachine.cpp +++ b/llvm/lib/Target/Sparc/SparcTargetMachine.cpp @@ -38,39 +38,6 @@ static cl::opt<bool> BranchRelaxation("sparc-enable-branch-relax", cl::Hidden, cl::init(true), cl::desc("Relax out of range conditional branches")); -static std::string computeDataLayout(const Triple &T) { - const bool is64Bit = T.isSPARC64(); - - // Sparc is typically big endian, but some are little. - std::string Ret = T.getArch() == Triple::sparcel ? "e" : "E"; - Ret += "-m:e"; - - // Some ABIs have 32bit pointers. - if (!is64Bit) - Ret += "-p:32:32"; - - // Alignments for 64 bit integers. - Ret += "-i64:64"; - - // Alignments for 128 bit integers. - // This is not specified in the ABI document but is the de facto standard. - Ret += "-i128:128"; - - // On SparcV9 128 floats are aligned to 128 bits, on others only to 64. - // On SparcV9 registers can hold 64 or 32 bits, on others only 32. - if (is64Bit) - Ret += "-n32:64"; - else - Ret += "-f128:64-n32"; - - if (is64Bit) - Ret += "-S128"; - else - Ret += "-S64"; - - return Ret; -} - static Reloc::Model getEffectiveRelocModel(std::optional<Reloc::Model> RM) { return RM.value_or(Reloc::Static); } @@ -111,7 +78,7 @@ SparcTargetMachine::SparcTargetMachine(const Target &T, const Triple &TT, std::optional<CodeModel::Model> CM, CodeGenOptLevel OL, bool JIT) : CodeGenTargetMachineImpl( - T, computeDataLayout(TT), TT, CPU, FS, Options, + T, TT.computeDataLayout(), TT, CPU, FS, Options, getEffectiveRelocModel(RM), getEffectiveSparcCodeModel(CM, getEffectiveRelocModel(RM), TT.isSPARC64(), JIT), diff --git a/llvm/lib/Target/SystemZ/SystemZFeatures.td b/llvm/lib/Target/SystemZ/SystemZFeatures.td index 2c48da8320fb..4ccc3d3079fc 100644 --- a/llvm/lib/Target/SystemZ/SystemZFeatures.td +++ b/llvm/lib/Target/SystemZ/SystemZFeatures.td @@ -196,7 +196,7 @@ def FeatureVector : SystemZFeature< >; def FeatureNoVector : SystemZMissingFeature<"Vector">; -def NoVecHwMode : HwMode<"-vector", [FeatureNoVector]>; +def NoVecHwMode : HwMode<[FeatureNoVector]>; def Arch11NewFeatures : SystemZFeatureList<[ FeatureLoadAndZeroRightmostByte, @@ -426,4 +426,3 @@ def Arch9UnsupportedFeatures : SystemZFeatureAdd<Arch10UnsupportedFeatures.List, Arch10NewFeatures.List>; def Arch8UnsupportedFeatures : SystemZFeatureAdd<Arch9UnsupportedFeatures.List, Arch9NewFeatures.List>; - diff --git a/llvm/lib/Target/SystemZ/SystemZTargetMachine.cpp b/llvm/lib/Target/SystemZ/SystemZTargetMachine.cpp index ece8928accd0..3d0c04b57493 100644 --- a/llvm/lib/Target/SystemZ/SystemZTargetMachine.cpp +++ b/llvm/lib/Target/SystemZ/SystemZTargetMachine.cpp @@ -54,47 +54,6 @@ LLVMInitializeSystemZTarget() { initializeSystemZCopyPhysRegsPass(PR); } -static std::string computeDataLayout(const Triple &TT) { - std::string Ret; - - // Big endian. - Ret += "E"; - - // Data mangling. - Ret += DataLayout::getManglingComponent(TT); - - // Special features for z/OS. - if (TT.isOSzOS()) { - if (TT.isArch64Bit()) { - // Custom address space for ptr32. - Ret += "-p1:32:32"; - } - } - - // Make sure that global data has at least 16 bits of alignment by - // default, so that we can refer to it using LARL. We don't have any - // special requirements for stack variables though. - Ret += "-i1:8:16-i8:8:16"; - - // 64-bit integers are naturally aligned. - Ret += "-i64:64"; - - // 128-bit floats are aligned only to 64 bits. - Ret += "-f128:64"; - - // The DataLayout string always holds a vector alignment of 64 bits, see - // comment in clang/lib/Basic/Targets/SystemZ.h. - Ret += "-v128:64"; - - // We prefer 16 bits of aligned for all globals; see above. - Ret += "-a:8:16"; - - // Integer registers are 32 or 64 bits. - Ret += "-n32:64"; - - return Ret; -} - static std::unique_ptr<TargetLoweringObjectFile> createTLOF(const Triple &TT) { if (TT.isOSzOS()) return std::make_unique<TargetLoweringObjectFileGOFF>(); @@ -163,7 +122,7 @@ SystemZTargetMachine::SystemZTargetMachine(const Target &T, const Triple &TT, std::optional<CodeModel::Model> CM, CodeGenOptLevel OL, bool JIT) : CodeGenTargetMachineImpl( - T, computeDataLayout(TT), TT, CPU, FS, Options, + T, TT.computeDataLayout(), TT, CPU, FS, Options, getEffectiveRelocModel(RM), getEffectiveSystemZCodeModel(CM, getEffectiveRelocModel(RM), JIT), OL), diff --git a/llvm/lib/Target/TargetMachineC.cpp b/llvm/lib/Target/TargetMachineC.cpp index da6d35c8c8b4..aba6ea436e76 100644 --- a/llvm/lib/Target/TargetMachineC.cpp +++ b/llvm/lib/Target/TargetMachineC.cpp @@ -83,7 +83,8 @@ LLVMBool LLVMGetTargetFromTriple(const char* TripleStr, LLVMTargetRef *T, char **ErrorMessage) { std::string Error; - *T = wrap(TargetRegistry::lookupTarget(TripleStr, Error)); + Triple TT(TripleStr); + *T = wrap(TargetRegistry::lookupTarget(TT, Error)); if (!*T) { if (ErrorMessage) diff --git a/llvm/lib/Target/VE/VETargetMachine.cpp b/llvm/lib/Target/VE/VETargetMachine.cpp index 14b8e330d87a..dc9ca48cc221 100644 --- a/llvm/lib/Target/VE/VETargetMachine.cpp +++ b/llvm/lib/Target/VE/VETargetMachine.cpp @@ -35,38 +35,6 @@ extern "C" LLVM_ABI LLVM_EXTERNAL_VISIBILITY void LLVMInitializeVETarget() { initializeVEDAGToDAGISelLegacyPass(PR); } -static std::string computeDataLayout(const Triple &T) { - // Aurora VE is little endian - std::string Ret = "e"; - - // Use ELF mangling - Ret += "-m:e"; - - // Alignments for 64 bit integers. - Ret += "-i64:64"; - - // VE supports 32 bit and 64 bits integer on registers - Ret += "-n32:64"; - - // Stack alignment is 128 bits - Ret += "-S128"; - - // Vector alignments are 64 bits - // Need to define all of them. Otherwise, each alignment becomes - // the size of each data by default. - Ret += "-v64:64:64"; // for v2f32 - Ret += "-v128:64:64"; - Ret += "-v256:64:64"; - Ret += "-v512:64:64"; - Ret += "-v1024:64:64"; - Ret += "-v2048:64:64"; - Ret += "-v4096:64:64"; - Ret += "-v8192:64:64"; - Ret += "-v16384:64:64"; // for v256f64 - - return Ret; -} - static Reloc::Model getEffectiveRelocModel(std::optional<Reloc::Model> RM) { return RM.value_or(Reloc::Static); } @@ -91,7 +59,7 @@ VETargetMachine::VETargetMachine(const Target &T, const Triple &TT, std::optional<Reloc::Model> RM, std::optional<CodeModel::Model> CM, CodeGenOptLevel OL, bool JIT) - : CodeGenTargetMachineImpl(T, computeDataLayout(TT), TT, CPU, FS, Options, + : CodeGenTargetMachineImpl(T, TT.computeDataLayout(), TT, CPU, FS, Options, getEffectiveRelocModel(RM), getEffectiveCodeModel(CM, CodeModel::Small), OL), TLOF(createTLOF()), diff --git a/llvm/lib/Target/WebAssembly/WebAssemblyISD.def b/llvm/lib/Target/WebAssembly/WebAssemblyISD.def index 378ef2c8f250..1eae3586d16b 100644 --- a/llvm/lib/Target/WebAssembly/WebAssemblyISD.def +++ b/llvm/lib/Target/WebAssembly/WebAssemblyISD.def @@ -27,6 +27,7 @@ HANDLE_NODETYPE(WrapperREL) HANDLE_NODETYPE(BR_IF) HANDLE_NODETYPE(BR_TABLE) HANDLE_NODETYPE(DOT) +HANDLE_NODETYPE(EXT_ADD_PAIRWISE_U) HANDLE_NODETYPE(SHUFFLE) HANDLE_NODETYPE(SWIZZLE) HANDLE_NODETYPE(VEC_SHL) diff --git a/llvm/lib/Target/WebAssembly/WebAssemblyISelLowering.cpp b/llvm/lib/Target/WebAssembly/WebAssemblyISelLowering.cpp index 5a4513469286..fe100dab427e 100644 --- a/llvm/lib/Target/WebAssembly/WebAssemblyISelLowering.cpp +++ b/llvm/lib/Target/WebAssembly/WebAssemblyISelLowering.cpp @@ -2183,13 +2183,10 @@ SDValue performLowerPartialReduction(SDNode *N, SelectionDAG &DAG) { SDValue MulLow = DAG.getNode(ISD::MUL, DL, MVT::v8i16, LowLHS, LowRHS); SDValue MulHigh = DAG.getNode(ISD::MUL, DL, MVT::v8i16, HighLHS, HighRHS); - SDValue LowLow = DAG.getNode(LowOpc, DL, MVT::v4i32, MulLow); - SDValue LowHigh = DAG.getNode(LowOpc, DL, MVT::v4i32, MulHigh); - SDValue HighLow = DAG.getNode(HighOpc, DL, MVT::v4i32, MulLow); - SDValue HighHigh = DAG.getNode(HighOpc, DL, MVT::v4i32, MulHigh); - - SDValue AddLow = DAG.getNode(ISD::ADD, DL, MVT::v4i32, LowLow, HighLow); - SDValue AddHigh = DAG.getNode(ISD::ADD, DL, MVT::v4i32, LowHigh, HighHigh); + SDValue AddLow = + DAG.getNode(WebAssemblyISD::EXT_ADD_PAIRWISE_U, DL, MVT::v4i32, MulLow); + SDValue AddHigh = DAG.getNode(WebAssemblyISD::EXT_ADD_PAIRWISE_U, DL, + MVT::v4i32, MulHigh); SDValue Add = DAG.getNode(ISD::ADD, DL, MVT::v4i32, AddLow, AddHigh); return DAG.getNode(ISD::ADD, DL, MVT::v4i32, N->getOperand(1), Add); } diff --git a/llvm/lib/Target/WebAssembly/WebAssemblyInstrSIMD.td b/llvm/lib/Target/WebAssembly/WebAssemblyInstrSIMD.td index f06f8d5174e3..3c26b453c448 100644 --- a/llvm/lib/Target/WebAssembly/WebAssemblyInstrSIMD.td +++ b/llvm/lib/Target/WebAssembly/WebAssemblyInstrSIMD.td @@ -1453,15 +1453,22 @@ if !ne(t1, t2) then def : Pat<(t1.vt (bitconvert (t2.vt V128:$v))), (t1.vt V128:$v)>; // Extended pairwise addition +def extadd_pairwise_u : SDNode<"WebAssemblyISD::EXT_ADD_PAIRWISE_U", extend_t>; + defm "" : SIMDConvert<I16x8, I8x16, int_wasm_extadd_pairwise_signed, "extadd_pairwise_i8x16_s", 0x7c>; -defm "" : SIMDConvert<I16x8, I8x16, int_wasm_extadd_pairwise_unsigned, +defm "" : SIMDConvert<I16x8, I8x16, extadd_pairwise_u, "extadd_pairwise_i8x16_u", 0x7d>; defm "" : SIMDConvert<I32x4, I16x8, int_wasm_extadd_pairwise_signed, "extadd_pairwise_i16x8_s", 0x7e>; -defm "" : SIMDConvert<I32x4, I16x8, int_wasm_extadd_pairwise_unsigned, +defm "" : SIMDConvert<I32x4, I16x8, extadd_pairwise_u, "extadd_pairwise_i16x8_u", 0x7f>; +def : Pat<(v4i32 (int_wasm_extadd_pairwise_unsigned (v8i16 V128:$in))), + (extadd_pairwise_u_I32x4 V128:$in)>; +def : Pat<(v8i16 (int_wasm_extadd_pairwise_unsigned (v16i8 V128:$in))), + (extadd_pairwise_u_I16x8 V128:$in)>; + // f64x2 <-> f32x4 conversions def demote_t : SDTypeProfile<1, 1, [SDTCisVec<0>, SDTCisVec<1>]>; def demote_zero : SDNode<"WebAssemblyISD::DEMOTE_ZERO", demote_t>; diff --git a/llvm/lib/Target/WebAssembly/WebAssemblyTargetMachine.cpp b/llvm/lib/Target/WebAssembly/WebAssemblyTargetMachine.cpp index 6827ee652794..a9c638cde125 100644 --- a/llvm/lib/Target/WebAssembly/WebAssemblyTargetMachine.cpp +++ b/llvm/lib/Target/WebAssembly/WebAssemblyTargetMachine.cpp @@ -196,19 +196,9 @@ WebAssemblyTargetMachine::WebAssemblyTargetMachine( const Target &T, const Triple &TT, StringRef CPU, StringRef FS, const TargetOptions &Options, std::optional<Reloc::Model> RM, std::optional<CodeModel::Model> CM, CodeGenOptLevel OL, bool JIT) - : CodeGenTargetMachineImpl( - T, - TT.isArch64Bit() - ? (TT.isOSEmscripten() ? "e-m:e-p:64:64-p10:8:8-p20:8:8-i64:64-" - "i128:128-f128:64-n32:64-S128-ni:1:10:20" - : "e-m:e-p:64:64-p10:8:8-p20:8:8-i64:64-" - "i128:128-n32:64-S128-ni:1:10:20") - : (TT.isOSEmscripten() ? "e-m:e-p:32:32-p10:8:8-p20:8:8-i64:64-" - "i128:128-f128:64-n32:64-S128-ni:1:10:20" - : "e-m:e-p:32:32-p10:8:8-p20:8:8-i64:64-" - "i128:128-n32:64-S128-ni:1:10:20"), - TT, CPU, FS, Options, getEffectiveRelocModel(RM, TT), - getEffectiveCodeModel(CM, CodeModel::Large), OL), + : CodeGenTargetMachineImpl(T, TT.computeDataLayout(), TT, CPU, FS, Options, + getEffectiveRelocModel(RM, TT), + getEffectiveCodeModel(CM, CodeModel::Large), OL), TLOF(new WebAssemblyTargetObjectFile()), UsesMultivalueABI(Options.MCOptions.getABIName() == "experimental-mv") { // WebAssembly type-checks instructions, but a noreturn function with a return diff --git a/llvm/lib/Target/X86/X86AsmPrinter.cpp b/llvm/lib/Target/X86/X86AsmPrinter.cpp index d406277e440b..a7734e9200a1 100644 --- a/llvm/lib/Target/X86/X86AsmPrinter.cpp +++ b/llvm/lib/Target/X86/X86AsmPrinter.cpp @@ -476,10 +476,11 @@ static bool isIndirectBranchOrTailCall(const MachineInstr &MI) { return MI.getDesc().isIndirectBranch() /*Make below code in a good shape*/ || Opc == X86::TAILJMPr || Opc == X86::TAILJMPm || Opc == X86::TAILJMPr64 || Opc == X86::TAILJMPm64 || - Opc == X86::TCRETURNri || Opc == X86::TCRETURNmi || - Opc == X86::TCRETURNri64 || Opc == X86::TCRETURNmi64 || - Opc == X86::TCRETURNri64_ImpCall || Opc == X86::TAILJMPr64_REX || - Opc == X86::TAILJMPm64_REX; + Opc == X86::TCRETURNri || Opc == X86::TCRETURN_WIN64ri || + Opc == X86::TCRETURN_HIPE32ri || Opc == X86::TCRETURNmi || + Opc == X86::TCRETURN_WINmi64 || Opc == X86::TCRETURNri64 || + Opc == X86::TCRETURNmi64 || Opc == X86::TCRETURNri64_ImpCall || + Opc == X86::TAILJMPr64_REX || Opc == X86::TAILJMPm64_REX; } void X86AsmPrinter::emitBasicBlockEnd(const MachineBasicBlock &MBB) { diff --git a/llvm/lib/Target/X86/X86ExpandPseudo.cpp b/llvm/lib/Target/X86/X86ExpandPseudo.cpp index 0e6b4dffec3a..4a9b824b0db1 100644 --- a/llvm/lib/Target/X86/X86ExpandPseudo.cpp +++ b/llvm/lib/Target/X86/X86ExpandPseudo.cpp @@ -269,13 +269,17 @@ bool X86ExpandPseudo::expandMI(MachineBasicBlock &MBB, case X86::TCRETURNdi: case X86::TCRETURNdicc: case X86::TCRETURNri: + case X86::TCRETURN_WIN64ri: + case X86::TCRETURN_HIPE32ri: case X86::TCRETURNmi: case X86::TCRETURNdi64: case X86::TCRETURNdi64cc: case X86::TCRETURNri64: case X86::TCRETURNri64_ImpCall: - case X86::TCRETURNmi64: { - bool isMem = Opcode == X86::TCRETURNmi || Opcode == X86::TCRETURNmi64; + case X86::TCRETURNmi64: + case X86::TCRETURN_WINmi64: { + bool isMem = Opcode == X86::TCRETURNmi || Opcode == X86::TCRETURNmi64 || + Opcode == X86::TCRETURN_WINmi64; MachineOperand &JumpTarget = MBBI->getOperand(0); MachineOperand &StackAdjust = MBBI->getOperand(isMem ? X86::AddrNumOperands : 1); @@ -339,15 +343,17 @@ bool X86ExpandPseudo::expandMI(MachineBasicBlock &MBB, MIB.addImm(MBBI->getOperand(2).getImm()); } - } else if (Opcode == X86::TCRETURNmi || Opcode == X86::TCRETURNmi64) { + } else if (Opcode == X86::TCRETURNmi || Opcode == X86::TCRETURNmi64 || + Opcode == X86::TCRETURN_WINmi64) { unsigned Op = (Opcode == X86::TCRETURNmi) ? X86::TAILJMPm : (IsX64 ? X86::TAILJMPm64_REX : X86::TAILJMPm64); MachineInstrBuilder MIB = BuildMI(MBB, MBBI, DL, TII->get(Op)); for (unsigned i = 0; i != X86::AddrNumOperands; ++i) MIB.add(MBBI->getOperand(i)); - } else if ((Opcode == X86::TCRETURNri64) || - (Opcode == X86::TCRETURNri64_ImpCall)) { + } else if (Opcode == X86::TCRETURNri64 || + Opcode == X86::TCRETURNri64_ImpCall || + Opcode == X86::TCRETURN_WIN64ri) { JumpTarget.setIsKill(); BuildMI(MBB, MBBI, DL, TII->get(IsX64 ? X86::TAILJMPr64_REX : X86::TAILJMPr64)) diff --git a/llvm/lib/Target/X86/X86FrameLowering.cpp b/llvm/lib/Target/X86/X86FrameLowering.cpp index cba7843d53e3..08c9d738bace 100644 --- a/llvm/lib/Target/X86/X86FrameLowering.cpp +++ b/llvm/lib/Target/X86/X86FrameLowering.cpp @@ -2398,10 +2398,11 @@ X86FrameLowering::getWinEHFuncletFrameSize(const MachineFunction &MF) const { } static bool isTailCallOpcode(unsigned Opc) { - return Opc == X86::TCRETURNri || Opc == X86::TCRETURNdi || + return Opc == X86::TCRETURNri || Opc == X86::TCRETURN_WIN64ri || + Opc == X86::TCRETURN_HIPE32ri || Opc == X86::TCRETURNdi || Opc == X86::TCRETURNmi || Opc == X86::TCRETURNri64 || Opc == X86::TCRETURNri64_ImpCall || Opc == X86::TCRETURNdi64 || - Opc == X86::TCRETURNmi64; + Opc == X86::TCRETURNmi64 || Opc == X86::TCRETURN_WINmi64; } void X86FrameLowering::emitEpilogue(MachineFunction &MF, diff --git a/llvm/lib/Target/X86/X86ISelLowering.cpp b/llvm/lib/Target/X86/X86ISelLowering.cpp index 4a6e193e09e6..3631016b0f5c 100644 --- a/llvm/lib/Target/X86/X86ISelLowering.cpp +++ b/llvm/lib/Target/X86/X86ISelLowering.cpp @@ -57970,6 +57970,51 @@ static SDValue pushAddIntoCmovOfConsts(SDNode *N, const SDLoc &DL, Cmov.getOperand(3)); } +// Attempt to turn ADD(MUL(x, y), acc)) -> VPMADD52L +// When upper 12 bits of x, y and MUL(x, y) are known to be 0 +static SDValue matchVPMADD52(SDNode *N, SelectionDAG &DAG, const SDLoc &DL, + EVT VT, const X86Subtarget &Subtarget) { + using namespace SDPatternMatch; + if (!VT.isVector() || VT.getScalarSizeInBits() != 64 || + (!Subtarget.hasAVXIFMA() && !Subtarget.hasIFMA())) + return SDValue(); + + // Need AVX-512VL vector length extensions if operating on XMM/YMM registers + if (!Subtarget.hasAVXIFMA() && !Subtarget.hasVLX() && + VT.getSizeInBits() < 512) + return SDValue(); + + const auto TotalSize = VT.getSizeInBits(); + if (TotalSize < 128 || !isPowerOf2_64(TotalSize)) + return SDValue(); + + SDValue X, Y, Acc; + if (!sd_match(N, m_Add(m_Mul(m_Value(X), m_Value(Y)), m_Value(Acc)))) + return SDValue(); + + KnownBits KnownX = DAG.computeKnownBits(X); + if (KnownX.countMinLeadingZeros() < 12) + return SDValue(); + KnownBits KnownY = DAG.computeKnownBits(Y); + if (KnownY.countMinLeadingZeros() < 12) + return SDValue(); + KnownBits KnownMul = KnownBits::mul(KnownX, KnownY); + if (KnownMul.countMinLeadingZeros() < 12) + return SDValue(); + + auto VPMADD52Builder = [](SelectionDAG &G, SDLoc DL, + ArrayRef<SDValue> SubOps) { + EVT SubVT = SubOps[0].getValueType(); + assert(SubVT.getScalarSizeInBits() == 64 && + "Unexpected element size, only supports 64bit size"); + return G.getNode(X86ISD::VPMADD52L, DL, SubVT, SubOps[1] /*X*/, + SubOps[2] /*Y*/, SubOps[0] /*Acc*/); + }; + + return SplitOpsAndApply(DAG, Subtarget, DL, VT, {Acc, X, Y}, VPMADD52Builder, + /*CheckBWI*/ false); +} + static SDValue combineAdd(SDNode *N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI, const X86Subtarget &Subtarget) { @@ -58073,6 +58118,9 @@ static SDValue combineAdd(SDNode *N, SelectionDAG &DAG, Op0.getOperand(0), Op0.getOperand(2)); } + if (SDValue IFMA52 = matchVPMADD52(N, DAG, DL, VT, Subtarget)) + return IFMA52; + return combineAddOrSubToADCOrSBB(N, DL, DAG); } diff --git a/llvm/lib/Target/X86/X86InstrAVX512.td b/llvm/lib/Target/X86/X86InstrAVX512.td index 3401f6f04800..b8f299965faa 100644 --- a/llvm/lib/Target/X86/X86InstrAVX512.td +++ b/llvm/lib/Target/X86/X86InstrAVX512.td @@ -12404,14 +12404,14 @@ multiclass VNNI_rmb<bits<8> Op, string OpStr, SDNode OpNode, X86FoldableSchedWrite sched, X86VectorVTInfo VTI, bit IsCommutable> { let ExeDomain = VTI.ExeDomain in { - defm r : AVX512_maskable_3src<Op, MRMSrcReg, VTI, (outs VTI.RC:$dst), + defm rr : AVX512_maskable_3src<Op, MRMSrcReg, VTI, (outs VTI.RC:$dst), (ins VTI.RC:$src2, VTI.RC:$src3), OpStr, "$src3, $src2", "$src2, $src3", (VTI.VT (OpNode VTI.RC:$src1, VTI.RC:$src2, VTI.RC:$src3)), IsCommutable, IsCommutable>, EVEX, VVVV, T8, Sched<[sched]>; - defm m : AVX512_maskable_3src<Op, MRMSrcMem, VTI, (outs VTI.RC:$dst), + defm rm : AVX512_maskable_3src<Op, MRMSrcMem, VTI, (outs VTI.RC:$dst), (ins VTI.RC:$src2, VTI.MemOp:$src3), OpStr, "$src3, $src2", "$src2, $src3", (VTI.VT (OpNode VTI.RC:$src1, VTI.RC:$src2, @@ -12419,7 +12419,7 @@ multiclass VNNI_rmb<bits<8> Op, string OpStr, SDNode OpNode, EVEX, VVVV, EVEX_CD8<32, CD8VF>, T8, Sched<[sched.Folded, sched.ReadAfterFold, sched.ReadAfterFold]>; - defm mb : AVX512_maskable_3src<Op, MRMSrcMem, VTI, (outs VTI.RC:$dst), + defm rmb : AVX512_maskable_3src<Op, MRMSrcMem, VTI, (outs VTI.RC:$dst), (ins VTI.RC:$src2, VTI.ScalarMemOp:$src3), OpStr, "${src3}"#VTI.BroadcastStr#", $src2", "$src2, ${src3}"#VTI.BroadcastStr, @@ -12459,24 +12459,24 @@ defm VPDPWSSDS : VNNI_common<0x53, "vpdpwssds", X86Vpdpwssds, SchedWriteVecIMul let Predicates = [HasVNNI] in { def : Pat<(v16i32 (add VR512:$src1, (X86vpmaddwd_su VR512:$src2, VR512:$src3))), - (VPDPWSSDZr VR512:$src1, VR512:$src2, VR512:$src3)>; + (VPDPWSSDZrr VR512:$src1, VR512:$src2, VR512:$src3)>; def : Pat<(v16i32 (add VR512:$src1, (X86vpmaddwd_su VR512:$src2, (load addr:$src3)))), - (VPDPWSSDZm VR512:$src1, VR512:$src2, addr:$src3)>; + (VPDPWSSDZrm VR512:$src1, VR512:$src2, addr:$src3)>; } let Predicates = [HasVNNI,HasVLX] in { def : Pat<(v8i32 (add VR256X:$src1, (X86vpmaddwd_su VR256X:$src2, VR256X:$src3))), - (VPDPWSSDZ256r VR256X:$src1, VR256X:$src2, VR256X:$src3)>; + (VPDPWSSDZ256rr VR256X:$src1, VR256X:$src2, VR256X:$src3)>; def : Pat<(v8i32 (add VR256X:$src1, (X86vpmaddwd_su VR256X:$src2, (load addr:$src3)))), - (VPDPWSSDZ256m VR256X:$src1, VR256X:$src2, addr:$src3)>; + (VPDPWSSDZ256rm VR256X:$src1, VR256X:$src2, addr:$src3)>; def : Pat<(v4i32 (add VR128X:$src1, (X86vpmaddwd_su VR128X:$src2, VR128X:$src3))), - (VPDPWSSDZ128r VR128X:$src1, VR128X:$src2, VR128X:$src3)>; + (VPDPWSSDZ128rr VR128X:$src1, VR128X:$src2, VR128X:$src3)>; def : Pat<(v4i32 (add VR128X:$src1, (X86vpmaddwd_su VR128X:$src2, (load addr:$src3)))), - (VPDPWSSDZ128m VR128X:$src1, VR128X:$src2, addr:$src3)>; + (VPDPWSSDZ128rm VR128X:$src1, VR128X:$src2, addr:$src3)>; } //===----------------------------------------------------------------------===// diff --git a/llvm/lib/Target/X86/X86InstrCompiler.td b/llvm/lib/Target/X86/X86InstrCompiler.td index 927b2c8b22f0..af7a33abaf75 100644 --- a/llvm/lib/Target/X86/X86InstrCompiler.td +++ b/llvm/lib/Target/X86/X86InstrCompiler.td @@ -1326,7 +1326,11 @@ def : Pat<(X86imp_call (i64 tglobaladdr:$dst)), // Match an X86tcret that uses less than 7 volatile registers. def : Pat<(X86tcret ptr_rc_tailcall:$dst, timm:$off), (TCRETURNri ptr_rc_tailcall:$dst, timm:$off)>, - Requires<[Not64BitMode, NotUseIndirectThunkCalls]>; + Requires<[Not64BitMode, IsNotHiPECCFunc, NotUseIndirectThunkCalls]>; + +def : Pat<(X86tcret GR32:$dst, timm:$off), + (TCRETURN_HIPE32ri GR32:$dst, timm:$off)>, + Requires<[Not64BitMode, IsHiPECCFunc, NotUseIndirectThunkCalls]>; // FIXME: This is disabled for 32-bit PIC mode because the global base // register which is part of the address mode may be assigned a @@ -1346,7 +1350,11 @@ def : Pat<(X86tcret (i32 texternalsym:$dst), timm:$off), def : Pat<(X86tcret ptr_rc_tailcall:$dst, timm:$off), (TCRETURNri64 ptr_rc_tailcall:$dst, timm:$off)>, - Requires<[In64BitMode, NotUseIndirectThunkCalls, ImportCallOptimizationDisabled]>; + Requires<[In64BitMode, IsNotWin64CCFunc, NotUseIndirectThunkCalls, ImportCallOptimizationDisabled]>; + +def : Pat<(X86tcret GR64_TCW64:$dst, timm:$off), + (TCRETURN_WIN64ri GR64_TCW64:$dst, timm:$off)>, + Requires<[IsWin64CCFunc, NotUseIndirectThunkCalls, ImportCallOptimizationDisabled]>; def : Pat<(X86tcret ptr_rc_tailcall:$dst, timm:$off), (TCRETURNri64_ImpCall ptr_rc_tailcall:$dst, timm:$off)>, @@ -1356,15 +1364,19 @@ def : Pat<(X86tcret ptr_rc_tailcall:$dst, timm:$off), // There wouldn't be enough scratch registers for base+index. def : Pat<(X86tcret_6regs (load addr:$dst), timm:$off), (TCRETURNmi64 addr:$dst, timm:$off)>, - Requires<[In64BitMode, NotUseIndirectThunkCalls]>; + Requires<[In64BitMode, IsNotWin64CCFunc, NotUseIndirectThunkCalls]>; + +def : Pat<(X86tcret_6regs (load addr:$dst), timm:$off), + (TCRETURN_WINmi64 addr:$dst, timm:$off)>, + Requires<[IsWin64CCFunc, NotUseIndirectThunkCalls]>; def : Pat<(X86tcret ptr_rc_tailcall:$dst, timm:$off), (INDIRECT_THUNK_TCRETURN64 ptr_rc_tailcall:$dst, timm:$off)>, - Requires<[In64BitMode, UseIndirectThunkCalls]>; + Requires<[In64BitMode, IsNotWin64CCFunc, UseIndirectThunkCalls]>; def : Pat<(X86tcret ptr_rc_tailcall:$dst, timm:$off), (INDIRECT_THUNK_TCRETURN32 ptr_rc_tailcall:$dst, timm:$off)>, - Requires<[Not64BitMode, UseIndirectThunkCalls]>; + Requires<[Not64BitMode, IsNotWin64CCFunc, UseIndirectThunkCalls]>; def : Pat<(X86tcret (i64 tglobaladdr:$dst), timm:$off), (TCRETURNdi64 tglobaladdr:$dst, timm:$off)>, @@ -2207,7 +2219,7 @@ let Predicates = [HasZU] in { def : Pat<(i64 (zext (i16 (mul (loadi16 addr:$src1), imm:$src2)))), (SUBREG_TO_REG (i64 0), (IMULZU16rmi addr:$src1, imm:$src2), sub_16bit)>; } - + // mul reg, imm def : Pat<(mul GR16:$src1, imm:$src2), (IMUL16rri GR16:$src1, imm:$src2)>; diff --git a/llvm/lib/Target/X86/X86InstrControl.td b/llvm/lib/Target/X86/X86InstrControl.td index 22253bf0413a..d962bfff1444 100644 --- a/llvm/lib/Target/X86/X86InstrControl.td +++ b/llvm/lib/Target/X86/X86InstrControl.td @@ -282,6 +282,10 @@ let isCall = 1, isTerminator = 1, isReturn = 1, isBarrier = 1, []>, Sched<[WriteJump]>; def TCRETURNri : PseudoI<(outs), (ins ptr_rc_tailcall:$dst, i32imm:$offset), []>, Sched<[WriteJump]>; + + def TCRETURN_HIPE32ri : PseudoI<(outs), (ins GR32:$dst, i32imm:$offset), + []>, Sched<[WriteJump]>; + let mayLoad = 1 in def TCRETURNmi : PseudoI<(outs), (ins i32mem_TC:$dst, i32imm:$offset), []>, Sched<[WriteJumpLd]>; @@ -357,6 +361,9 @@ let isCall = 1, isTerminator = 1, isReturn = 1, isBarrier = 1, def TCRETURNri64 : PseudoI<(outs), (ins ptr_rc_tailcall:$dst, i32imm:$offset), []>, Sched<[WriteJump]>; + def TCRETURN_WIN64ri : PseudoI<(outs), (ins GR64_TCW64:$dst, i32imm:$offset), + []>, Sched<[WriteJump]>; + def TCRETURNri64_ImpCall : PseudoI<(outs), (ins GR64_A:$dst, i32imm:$offset), []>, Sched<[WriteJump]>; @@ -365,6 +372,9 @@ let isCall = 1, isTerminator = 1, isReturn = 1, isBarrier = 1, def TCRETURNmi64 : PseudoI<(outs), (ins i64mem_TC:$dst, i32imm:$offset), []>, Sched<[WriteJumpLd]>; + def TCRETURN_WINmi64 : PseudoI<(outs), + (ins i64mem_w64TC:$dst, i32imm:$offset), + []>, Sched<[WriteJumpLd]>; def TAILJMPd64 : PseudoI<(outs), (ins i64i32imm_brtarget:$dst), []>, Sched<[WriteJump]>; diff --git a/llvm/lib/Target/X86/X86InstrInfo.cpp b/llvm/lib/Target/X86/X86InstrInfo.cpp index a68edf4d2b7e..f109e29c0bff 100644 --- a/llvm/lib/Target/X86/X86InstrInfo.cpp +++ b/llvm/lib/Target/X86/X86InstrInfo.cpp @@ -2939,78 +2939,78 @@ bool X86InstrInfo::findCommutedOpIndices(const MachineInstr &MI, case X86::VPDPBUUDSYrr: case X86::VPDPBUUDrr: case X86::VPDPBUUDYrr: - case X86::VPDPBSSDSZ128r: - case X86::VPDPBSSDSZ128rk: - case X86::VPDPBSSDSZ128rkz: - case X86::VPDPBSSDSZ256r: - case X86::VPDPBSSDSZ256rk: - case X86::VPDPBSSDSZ256rkz: - case X86::VPDPBSSDSZr: - case X86::VPDPBSSDSZrk: - case X86::VPDPBSSDSZrkz: - case X86::VPDPBSSDZ128r: - case X86::VPDPBSSDZ128rk: - case X86::VPDPBSSDZ128rkz: - case X86::VPDPBSSDZ256r: - case X86::VPDPBSSDZ256rk: - case X86::VPDPBSSDZ256rkz: - case X86::VPDPBSSDZr: - case X86::VPDPBSSDZrk: - case X86::VPDPBSSDZrkz: - case X86::VPDPBUUDSZ128r: - case X86::VPDPBUUDSZ128rk: - case X86::VPDPBUUDSZ128rkz: - case X86::VPDPBUUDSZ256r: - case X86::VPDPBUUDSZ256rk: - case X86::VPDPBUUDSZ256rkz: - case X86::VPDPBUUDSZr: - case X86::VPDPBUUDSZrk: - case X86::VPDPBUUDSZrkz: - case X86::VPDPBUUDZ128r: - case X86::VPDPBUUDZ128rk: - case X86::VPDPBUUDZ128rkz: - case X86::VPDPBUUDZ256r: - case X86::VPDPBUUDZ256rk: - case X86::VPDPBUUDZ256rkz: - case X86::VPDPBUUDZr: - case X86::VPDPBUUDZrk: - case X86::VPDPBUUDZrkz: - case X86::VPDPWSSDZ128r: - case X86::VPDPWSSDZ128rk: - case X86::VPDPWSSDZ128rkz: - case X86::VPDPWSSDZ256r: - case X86::VPDPWSSDZ256rk: - case X86::VPDPWSSDZ256rkz: - case X86::VPDPWSSDZr: - case X86::VPDPWSSDZrk: - case X86::VPDPWSSDZrkz: - case X86::VPDPWSSDSZ128r: - case X86::VPDPWSSDSZ128rk: - case X86::VPDPWSSDSZ128rkz: - case X86::VPDPWSSDSZ256r: - case X86::VPDPWSSDSZ256rk: - case X86::VPDPWSSDSZ256rkz: - case X86::VPDPWSSDSZr: - case X86::VPDPWSSDSZrk: - case X86::VPDPWSSDSZrkz: - case X86::VPDPWUUDZ128r: - case X86::VPDPWUUDZ128rk: - case X86::VPDPWUUDZ128rkz: - case X86::VPDPWUUDZ256r: - case X86::VPDPWUUDZ256rk: - case X86::VPDPWUUDZ256rkz: - case X86::VPDPWUUDZr: - case X86::VPDPWUUDZrk: - case X86::VPDPWUUDZrkz: - case X86::VPDPWUUDSZ128r: - case X86::VPDPWUUDSZ128rk: - case X86::VPDPWUUDSZ128rkz: - case X86::VPDPWUUDSZ256r: - case X86::VPDPWUUDSZ256rk: - case X86::VPDPWUUDSZ256rkz: - case X86::VPDPWUUDSZr: - case X86::VPDPWUUDSZrk: - case X86::VPDPWUUDSZrkz: + case X86::VPDPBSSDSZ128rr: + case X86::VPDPBSSDSZ128rrk: + case X86::VPDPBSSDSZ128rrkz: + case X86::VPDPBSSDSZ256rr: + case X86::VPDPBSSDSZ256rrk: + case X86::VPDPBSSDSZ256rrkz: + case X86::VPDPBSSDSZrr: + case X86::VPDPBSSDSZrrk: + case X86::VPDPBSSDSZrrkz: + case X86::VPDPBSSDZ128rr: + case X86::VPDPBSSDZ128rrk: + case X86::VPDPBSSDZ128rrkz: + case X86::VPDPBSSDZ256rr: + case X86::VPDPBSSDZ256rrk: + case X86::VPDPBSSDZ256rrkz: + case X86::VPDPBSSDZrr: + case X86::VPDPBSSDZrrk: + case X86::VPDPBSSDZrrkz: + case X86::VPDPBUUDSZ128rr: + case X86::VPDPBUUDSZ128rrk: + case X86::VPDPBUUDSZ128rrkz: + case X86::VPDPBUUDSZ256rr: + case X86::VPDPBUUDSZ256rrk: + case X86::VPDPBUUDSZ256rrkz: + case X86::VPDPBUUDSZrr: + case X86::VPDPBUUDSZrrk: + case X86::VPDPBUUDSZrrkz: + case X86::VPDPBUUDZ128rr: + case X86::VPDPBUUDZ128rrk: + case X86::VPDPBUUDZ128rrkz: + case X86::VPDPBUUDZ256rr: + case X86::VPDPBUUDZ256rrk: + case X86::VPDPBUUDZ256rrkz: + case X86::VPDPBUUDZrr: + case X86::VPDPBUUDZrrk: + case X86::VPDPBUUDZrrkz: + case X86::VPDPWSSDZ128rr: + case X86::VPDPWSSDZ128rrk: + case X86::VPDPWSSDZ128rrkz: + case X86::VPDPWSSDZ256rr: + case X86::VPDPWSSDZ256rrk: + case X86::VPDPWSSDZ256rrkz: + case X86::VPDPWSSDZrr: + case X86::VPDPWSSDZrrk: + case X86::VPDPWSSDZrrkz: + case X86::VPDPWSSDSZ128rr: + case X86::VPDPWSSDSZ128rrk: + case X86::VPDPWSSDSZ128rrkz: + case X86::VPDPWSSDSZ256rr: + case X86::VPDPWSSDSZ256rrk: + case X86::VPDPWSSDSZ256rrkz: + case X86::VPDPWSSDSZrr: + case X86::VPDPWSSDSZrrk: + case X86::VPDPWSSDSZrrkz: + case X86::VPDPWUUDZ128rr: + case X86::VPDPWUUDZ128rrk: + case X86::VPDPWUUDZ128rrkz: + case X86::VPDPWUUDZ256rr: + case X86::VPDPWUUDZ256rrk: + case X86::VPDPWUUDZ256rrkz: + case X86::VPDPWUUDZrr: + case X86::VPDPWUUDZrrk: + case X86::VPDPWUUDZrrkz: + case X86::VPDPWUUDSZ128rr: + case X86::VPDPWUUDSZ128rrk: + case X86::VPDPWUUDSZ128rrkz: + case X86::VPDPWUUDSZ256rr: + case X86::VPDPWUUDSZ256rrk: + case X86::VPDPWUUDSZ256rrkz: + case X86::VPDPWUUDSZrr: + case X86::VPDPWUUDSZrrk: + case X86::VPDPWUUDSZrrkz: case X86::VPMADD52HUQrr: case X86::VPMADD52HUQYrr: case X86::VPMADD52HUQZ128r: @@ -10822,15 +10822,15 @@ bool X86InstrInfo::getMachineCombinerPatterns( } break; } - case X86::VPDPWSSDZ128r: - case X86::VPDPWSSDZ128m: - case X86::VPDPWSSDZ256r: - case X86::VPDPWSSDZ256m: - case X86::VPDPWSSDZr: - case X86::VPDPWSSDZm: { - if (Subtarget.hasBWI() && !Subtarget.hasFastDPWSSD()) { - Patterns.push_back(X86MachineCombinerPattern::DPWSSD); - return true; + case X86::VPDPWSSDZ128rr: + case X86::VPDPWSSDZ128rm: + case X86::VPDPWSSDZ256rr: + case X86::VPDPWSSDZ256rm: + case X86::VPDPWSSDZrr: + case X86::VPDPWSSDZrm: { + if (Subtarget.hasBWI() && !Subtarget.hasFastDPWSSD()) { + Patterns.push_back(X86MachineCombinerPattern::DPWSSD); + return true; } break; } @@ -10866,11 +10866,11 @@ genAlternativeDpCodeSequence(MachineInstr &Root, const TargetInstrInfo &TII, MaddOpc = X86::VPMADDWDrm; AddOpc = X86::VPADDDrr; break; - case X86::VPDPWSSDZ128r: + case X86::VPDPWSSDZ128rr: MaddOpc = X86::VPMADDWDZ128rr; AddOpc = X86::VPADDDZ128rr; break; - case X86::VPDPWSSDZ128m: + case X86::VPDPWSSDZ128rm: MaddOpc = X86::VPMADDWDZ128rm; AddOpc = X86::VPADDDZ128rr; break; @@ -10886,11 +10886,11 @@ genAlternativeDpCodeSequence(MachineInstr &Root, const TargetInstrInfo &TII, MaddOpc = X86::VPMADDWDYrm; AddOpc = X86::VPADDDYrr; break; - case X86::VPDPWSSDZ256r: + case X86::VPDPWSSDZ256rr: MaddOpc = X86::VPMADDWDZ256rr; AddOpc = X86::VPADDDZ256rr; break; - case X86::VPDPWSSDZ256m: + case X86::VPDPWSSDZ256rm: MaddOpc = X86::VPMADDWDZ256rm; AddOpc = X86::VPADDDZ256rr; break; @@ -10898,11 +10898,11 @@ genAlternativeDpCodeSequence(MachineInstr &Root, const TargetInstrInfo &TII, // --> // vpmaddwd zmm3,zmm3,zmm1 // vpaddd zmm2,zmm2,zmm3 - case X86::VPDPWSSDZr: + case X86::VPDPWSSDZrr: MaddOpc = X86::VPMADDWDZrr; AddOpc = X86::VPADDDZrr; break; - case X86::VPDPWSSDZm: + case X86::VPDPWSSDZrm: MaddOpc = X86::VPMADDWDZrm; AddOpc = X86::VPADDDZrr; break; diff --git a/llvm/lib/Target/X86/X86InstrOperands.td b/llvm/lib/Target/X86/X86InstrOperands.td index 53a6b7c4c4c9..80843f6bb80e 100644 --- a/llvm/lib/Target/X86/X86InstrOperands.td +++ b/llvm/lib/Target/X86/X86InstrOperands.td @@ -141,6 +141,11 @@ def i64mem_TC : X86MemOperand<"printqwordmem", X86Mem64AsmOperand, 64> { ptr_rc_tailcall, i32imm, SEGMENT_REG); } +def i64mem_w64TC : X86MemOperand<"printqwordmem", X86Mem64AsmOperand, 64> { + let MIOperandInfo = (ops GR64_TCW64, i8imm, + GR64_TCW64, i32imm, SEGMENT_REG); +} + // Special parser to detect 16-bit mode to select 16-bit displacement. def X86AbsMemMode16AsmOperand : AsmOperandClass { let Name = "AbsMemMode16"; diff --git a/llvm/lib/Target/X86/X86InstrPredicates.td b/llvm/lib/Target/X86/X86InstrPredicates.td index 8a2846bcfdf3..8339c2081842 100644 --- a/llvm/lib/Target/X86/X86InstrPredicates.td +++ b/llvm/lib/Target/X86/X86InstrPredicates.td @@ -230,6 +230,13 @@ let RecomputePerFunction = 1 in { "!Subtarget->hasSSE41()">; def ImportCallOptimizationEnabled : Predicate<"MF->getFunction().getParent()->getModuleFlag(\"import-call-optimization\")">; def ImportCallOptimizationDisabled : Predicate<"!MF->getFunction().getParent()->getModuleFlag(\"import-call-optimization\")">; + + def IsWin64CCFunc : Predicate<"Subtarget->isCallingConvWin64(MF->getFunction().getCallingConv())">; + def IsNotWin64CCFunc : Predicate<"!Subtarget->isCallingConvWin64(MF->getFunction().getCallingConv())">; + def IsHiPECCFunc : Predicate<"MF->getFunction().getCallingConv() == CallingConv::HiPE">; + + def IsNotHiPECCFunc : Predicate< + "MF->getFunction().getCallingConv() != CallingConv::HiPE">; } def CallImmAddr : Predicate<"Subtarget->isLegalToCallImmediateAddr()">; diff --git a/llvm/lib/Target/X86/X86InstrSSE.td b/llvm/lib/Target/X86/X86InstrSSE.td index b7926497c92b..806b02b9f935 100644 --- a/llvm/lib/Target/X86/X86InstrSSE.td +++ b/llvm/lib/Target/X86/X86InstrSSE.td @@ -6898,26 +6898,26 @@ def AESIMCrm : AES8I<0xDB, MRMSrcMem, (outs VR128:$dst), // AES Round Key Generation Assist let Predicates = [HasAVX, HasAES] in { - def VAESKEYGENASSIST128rr : AESAI<0xDF, MRMSrcReg, (outs VR128:$dst), + def VAESKEYGENASSISTrri : AESAI<0xDF, MRMSrcReg, (outs VR128:$dst), (ins VR128:$src1, u8imm:$src2), "vaeskeygenassist\t{$src2, $src1, $dst|$dst, $src1, $src2}", [(set VR128:$dst, (int_x86_aesni_aeskeygenassist VR128:$src1, timm:$src2))]>, Sched<[WriteAESKeyGen]>, VEX, WIG; - def VAESKEYGENASSIST128rm : AESAI<0xDF, MRMSrcMem, (outs VR128:$dst), + def VAESKEYGENASSISTrmi : AESAI<0xDF, MRMSrcMem, (outs VR128:$dst), (ins i128mem:$src1, u8imm:$src2), "vaeskeygenassist\t{$src2, $src1, $dst|$dst, $src1, $src2}", [(set VR128:$dst, (int_x86_aesni_aeskeygenassist (load addr:$src1), timm:$src2))]>, Sched<[WriteAESKeyGen.Folded]>, VEX, WIG; } -def AESKEYGENASSIST128rr : AESAI<0xDF, MRMSrcReg, (outs VR128:$dst), +def AESKEYGENASSISTrri : AESAI<0xDF, MRMSrcReg, (outs VR128:$dst), (ins VR128:$src1, u8imm:$src2), "aeskeygenassist\t{$src2, $src1, $dst|$dst, $src1, $src2}", [(set VR128:$dst, (int_x86_aesni_aeskeygenassist VR128:$src1, timm:$src2))]>, Sched<[WriteAESKeyGen]>; -def AESKEYGENASSIST128rm : AESAI<0xDF, MRMSrcMem, (outs VR128:$dst), +def AESKEYGENASSISTrmi : AESAI<0xDF, MRMSrcMem, (outs VR128:$dst), (ins i128mem:$src1, u8imm:$src2), "aeskeygenassist\t{$src2, $src1, $dst|$dst, $src1, $src2}", [(set VR128:$dst, diff --git a/llvm/lib/Target/X86/X86RegisterInfo.cpp b/llvm/lib/Target/X86/X86RegisterInfo.cpp index b79e508df7c9..7963dc1b755c 100644 --- a/llvm/lib/Target/X86/X86RegisterInfo.cpp +++ b/llvm/lib/Target/X86/X86RegisterInfo.cpp @@ -220,25 +220,11 @@ X86RegisterInfo::getPointerRegClass(const MachineFunction &MF, // NOSP does not contain RIP, so no special case here. return &X86::GR32_NOREX_NOSPRegClass; case 4: // Available for tailcall (not callee-saved GPRs). - return getGPRsForTailCall(MF); + return Is64Bit ? &X86::GR64_TCRegClass : &X86::GR32_TCRegClass; } } const TargetRegisterClass * -X86RegisterInfo::getGPRsForTailCall(const MachineFunction &MF) const { - const Function &F = MF.getFunction(); - if (IsWin64 || IsUEFI64 || (F.getCallingConv() == CallingConv::Win64)) - return &X86::GR64_TCW64RegClass; - else if (Is64Bit) - return &X86::GR64_TCRegClass; - - bool hasHipeCC = (F.getCallingConv() == CallingConv::HiPE); - if (hasHipeCC) - return &X86::GR32RegClass; - return &X86::GR32_TCRegClass; -} - -const TargetRegisterClass * X86RegisterInfo::getCrossCopyRegClass(const TargetRegisterClass *RC) const { if (RC == &X86::CCRRegClass) { if (Is64Bit) @@ -999,11 +985,10 @@ X86RegisterInfo::eliminateFrameIndex(MachineBasicBlock::iterator II, unsigned X86RegisterInfo::findDeadCallerSavedReg( MachineBasicBlock &MBB, MachineBasicBlock::iterator &MBBI) const { const MachineFunction *MF = MBB.getParent(); + const MachineRegisterInfo &MRI = MF->getRegInfo(); if (MF->callsEHReturn()) return 0; - const TargetRegisterClass &AvailableRegs = *getGPRsForTailCall(*MF); - if (MBBI == MBB.end()) return 0; @@ -1018,27 +1003,26 @@ unsigned X86RegisterInfo::findDeadCallerSavedReg( case X86::RETI64: case X86::TCRETURNdi: case X86::TCRETURNri: + case X86::TCRETURN_WIN64ri: + case X86::TCRETURN_HIPE32ri: case X86::TCRETURNmi: case X86::TCRETURNdi64: case X86::TCRETURNri64: case X86::TCRETURNri64_ImpCall: case X86::TCRETURNmi64: + case X86::TCRETURN_WINmi64: case X86::EH_RETURN: case X86::EH_RETURN64: { - SmallSet<uint16_t, 8> Uses; - for (MachineOperand &MO : MBBI->operands()) { - if (!MO.isReg() || MO.isDef()) - continue; - Register Reg = MO.getReg(); - if (!Reg) - continue; - for (MCRegAliasIterator AI(Reg, this, true); AI.isValid(); ++AI) - Uses.insert(*AI); + LiveRegUnits LRU(*this); + LRU.addLiveOuts(MBB); + LRU.stepBackward(*MBBI); + + const TargetRegisterClass &RC = + Is64Bit ? X86::GR64_NOSPRegClass : X86::GR32_NOSPRegClass; + for (MCRegister Reg : RC) { + if (LRU.available(Reg) && !MRI.isReserved(Reg)) + return Reg; } - - for (auto CS : AvailableRegs) - if (!Uses.count(CS) && CS != X86::RIP && CS != X86::RSP && CS != X86::ESP) - return CS; } } diff --git a/llvm/lib/Target/X86/X86RegisterInfo.h b/llvm/lib/Target/X86/X86RegisterInfo.h index 2f4c55cfad6d..d022e5ab8794 100644 --- a/llvm/lib/Target/X86/X86RegisterInfo.h +++ b/llvm/lib/Target/X86/X86RegisterInfo.h @@ -87,11 +87,6 @@ public: const TargetRegisterClass * getCrossCopyRegClass(const TargetRegisterClass *RC) const override; - /// getGPRsForTailCall - Returns a register class with registers that can be - /// used in forming tail calls. - const TargetRegisterClass * - getGPRsForTailCall(const MachineFunction &MF) const; - unsigned getRegPressureLimit(const TargetRegisterClass *RC, MachineFunction &MF) const override; diff --git a/llvm/lib/Target/X86/X86ScheduleZnver3.td b/llvm/lib/Target/X86/X86ScheduleZnver3.td index 9e271c1ee370..044b77f7aacf 100644 --- a/llvm/lib/Target/X86/X86ScheduleZnver3.td +++ b/llvm/lib/Target/X86/X86ScheduleZnver3.td @@ -992,14 +992,14 @@ def Zn3WriteVEXTRACTF128rr_VEXTRACTI128rr : SchedWriteRes<[Zn3FPFMisc0]> { def : InstRW<[Zn3WriteVEXTRACTF128rr_VEXTRACTI128rr], (instrs VEXTRACTF128rri, VEXTRACTI128rri)>; def Zn3WriteVEXTRACTI128mr : SchedWriteRes<[Zn3FPFMisc0, Zn3FPSt, Zn3Store]> { - let Latency = !add(Znver3Model.LoadLatency, Zn3WriteVEXTRACTF128rr_VEXTRACTI128rr.Latency); + let Latency = !add(Znver3Model.VecLoadLatency, Zn3WriteVEXTRACTF128rr_VEXTRACTI128rr.Latency); let ReleaseAtCycles = [1, 1, 1]; let NumMicroOps = !add(Zn3WriteVEXTRACTF128rr_VEXTRACTI128rr.NumMicroOps, 1); } def : InstRW<[Zn3WriteVEXTRACTI128mr], (instrs VEXTRACTI128mri, VEXTRACTF128mri)>; def Zn3WriteVINSERTF128rmr : SchedWriteRes<[Zn3AGU012, Zn3Load, Zn3FPFMisc0]> { - let Latency = !add(Znver3Model.LoadLatency, Zn3WriteVEXTRACTF128rr_VEXTRACTI128rr.Latency); + let Latency = !add(Znver3Model.VecLoadLatency, Zn3WriteVEXTRACTF128rr_VEXTRACTI128rr.Latency); let ReleaseAtCycles = [1, 1, 1]; let NumMicroOps = !add(Zn3WriteVEXTRACTF128rr_VEXTRACTI128rr.NumMicroOps, 0); } @@ -1221,7 +1221,7 @@ def Zn3WriteSHA1MSG1rr : SchedWriteRes<[Zn3FPU0123]> { def : InstRW<[Zn3WriteSHA1MSG1rr], (instrs SHA1MSG1rr)>; def Zn3WriteSHA1MSG1rm : SchedWriteRes<[Zn3AGU012, Zn3Load, Zn3FPU0123]> { - let Latency = !add(Znver3Model.LoadLatency, Zn3WriteSHA1MSG1rr.Latency); + let Latency = !add(Znver3Model.VecLoadLatency, Zn3WriteSHA1MSG1rr.Latency); let ReleaseAtCycles = [1, 1, 2]; let NumMicroOps = !add(Zn3WriteSHA1MSG1rr.NumMicroOps, 0); } @@ -1235,7 +1235,7 @@ def Zn3WriteSHA1MSG2rr_SHA1NEXTErr : SchedWriteRes<[Zn3FPU0123]> { def : InstRW<[Zn3WriteSHA1MSG2rr_SHA1NEXTErr], (instrs SHA1MSG2rr, SHA1NEXTErr)>; def Zn3Writerm_SHA1MSG2rm_SHA1NEXTErm : SchedWriteRes<[Zn3AGU012, Zn3Load, Zn3FPU0123]> { - let Latency = !add(Znver3Model.LoadLatency, Zn3WriteSHA1MSG2rr_SHA1NEXTErr.Latency); + let Latency = !add(Znver3Model.VecLoadLatency, Zn3WriteSHA1MSG2rr_SHA1NEXTErr.Latency); let ReleaseAtCycles = [1, 1, 2]; let NumMicroOps = !add(Zn3WriteSHA1MSG2rr_SHA1NEXTErr.NumMicroOps, 0); } @@ -1249,7 +1249,7 @@ def Zn3WriteSHA256MSG1rr : SchedWriteRes<[Zn3FPU0123]> { def : InstRW<[Zn3WriteSHA256MSG1rr], (instrs SHA256MSG1rr)>; def Zn3Writerm_SHA256MSG1rm : SchedWriteRes<[Zn3AGU012, Zn3Load, Zn3FPU0123]> { - let Latency = !add(Znver3Model.LoadLatency, Zn3WriteSHA256MSG1rr.Latency); + let Latency = !add(Znver3Model.VecLoadLatency, Zn3WriteSHA256MSG1rr.Latency); let ReleaseAtCycles = [1, 1, 3]; let NumMicroOps = !add(Zn3WriteSHA256MSG1rr.NumMicroOps, 0); } @@ -1263,7 +1263,7 @@ def Zn3WriteSHA256MSG2rr : SchedWriteRes<[Zn3FPU0123]> { def : InstRW<[Zn3WriteSHA256MSG2rr], (instrs SHA256MSG2rr)>; def Zn3WriteSHA256MSG2rm : SchedWriteRes<[Zn3AGU012, Zn3Load, Zn3FPU0123]> { - let Latency = !add(Znver3Model.LoadLatency, Zn3WriteSHA256MSG2rr.Latency); + let Latency = !add(Znver3Model.VecLoadLatency, Zn3WriteSHA256MSG2rr.Latency); let ReleaseAtCycles = [1, 1, 8]; let NumMicroOps = !add(Zn3WriteSHA256MSG2rr.NumMicroOps, 1); } @@ -1338,14 +1338,14 @@ def Zn3WriteVPERM2I128rr_VPERM2F128rr : SchedWriteRes<[Zn3FPVShuf]> { def : InstRW<[Zn3WriteVPERM2I128rr_VPERM2F128rr], (instrs VPERM2I128rri, VPERM2F128rri)>; def Zn3WriteVPERM2F128rm : SchedWriteRes<[Zn3AGU012, Zn3Load, Zn3FPVShuf]> { - let Latency = !add(Znver3Model.LoadLatency, Zn3WriteVPERM2I128rr_VPERM2F128rr.Latency); + let Latency = !add(Znver3Model.VecLoadLatency, Zn3WriteVPERM2I128rr_VPERM2F128rr.Latency); let ReleaseAtCycles = [1, 1, 1]; let NumMicroOps = !add(Zn3WriteVPERM2I128rr_VPERM2F128rr.NumMicroOps, 0); } def : InstRW<[Zn3WriteVPERM2F128rm], (instrs VPERM2F128rmi)>; def Zn3WriteVPERMPSYrm : SchedWriteRes<[Zn3AGU012, Zn3Load, Zn3FPVShuf]> { - let Latency = !add(Znver3Model.LoadLatency, 7); + let Latency = !add(Znver3Model.VecLoadLatency, 7); let ReleaseAtCycles = [1, 1, 2]; let NumMicroOps = 3; } @@ -1359,14 +1359,14 @@ def Zn3WriteVPERMYri : SchedWriteRes<[Zn3FPVShuf]> { def : InstRW<[Zn3WriteVPERMYri], (instrs VPERMPDYri, VPERMQYri)>; def Zn3WriteVPERMPDYmi : SchedWriteRes<[Zn3AGU012, Zn3Load, Zn3FPVShuf]> { - let Latency = !add(Znver3Model.LoadLatency, Zn3WriteVPERMYri.Latency); + let Latency = !add(Znver3Model.VecLoadLatency, Zn3WriteVPERMYri.Latency); let ReleaseAtCycles = [1, 1, 2]; let NumMicroOps = !add(Zn3WriteVPERMYri.NumMicroOps, 1); } def : InstRW<[Zn3WriteVPERMPDYmi], (instrs VPERMPDYmi)>; def Zn3WriteVPERMDYm : SchedWriteRes<[Zn3AGU012, Zn3Load, Zn3FPVShuf]> { - let Latency = !add(Znver3Model.LoadLatency, 5); + let Latency = !add(Znver3Model.VecLoadLatency, 5); let ReleaseAtCycles = [1, 1, 2]; let NumMicroOps = 2; } diff --git a/llvm/lib/Target/X86/X86ScheduleZnver4.td b/llvm/lib/Target/X86/X86ScheduleZnver4.td index 74d916d41f83..cc300548a50e 100644 --- a/llvm/lib/Target/X86/X86ScheduleZnver4.td +++ b/llvm/lib/Target/X86/X86ScheduleZnver4.td @@ -1005,14 +1005,14 @@ def Zn4WriteVEXTRACTF128rr_VEXTRACTI128rr : SchedWriteRes<[Zn4FPFMisc0]> { def : InstRW<[Zn4WriteVEXTRACTF128rr_VEXTRACTI128rr], (instrs VEXTRACTF128rri, VEXTRACTI128rri)>; def Zn4WriteVEXTRACTI128mr : SchedWriteRes<[Zn4FPFMisc0, Zn4FPSt, Zn4Store]> { - let Latency = !add(Znver4Model.LoadLatency, Zn4WriteVEXTRACTF128rr_VEXTRACTI128rr.Latency); + let Latency = !add(Znver4Model.VecLoadLatency, Zn4WriteVEXTRACTF128rr_VEXTRACTI128rr.Latency); let ReleaseAtCycles = [1, 1, 1]; let NumMicroOps = !add(Zn4WriteVEXTRACTF128rr_VEXTRACTI128rr.NumMicroOps, 1); } def : InstRW<[Zn4WriteVEXTRACTI128mr], (instrs VEXTRACTI128mri, VEXTRACTF128mri)>; def Zn4WriteVINSERTF128rmr : SchedWriteRes<[Zn4AGU012, Zn4Load, Zn4FPFMisc0]> { - let Latency = !add(Znver4Model.LoadLatency, Zn4WriteVEXTRACTF128rr_VEXTRACTI128rr.Latency); + let Latency = !add(Znver4Model.VecLoadLatency, Zn4WriteVEXTRACTF128rr_VEXTRACTI128rr.Latency); let ReleaseAtCycles = [1, 1, 1]; let NumMicroOps = !add(Zn4WriteVEXTRACTF128rr_VEXTRACTI128rr.NumMicroOps, 0); } @@ -1262,7 +1262,7 @@ def Zn4WriteSHA1MSG1rr : SchedWriteRes<[Zn4FPU0123]> { def : InstRW<[Zn4WriteSHA1MSG1rr], (instrs SHA1MSG1rr)>; def Zn4WriteSHA1MSG1rm : SchedWriteRes<[Zn4AGU012, Zn4Load, Zn4FPU0123]> { - let Latency = !add(Znver4Model.LoadLatency, Zn4WriteSHA1MSG1rr.Latency); + let Latency = !add(Znver4Model.VecLoadLatency, Zn4WriteSHA1MSG1rr.Latency); let ReleaseAtCycles = [1, 1, 2]; let NumMicroOps = !add(Zn4WriteSHA1MSG1rr.NumMicroOps, 0); } @@ -1276,7 +1276,7 @@ def Zn4WriteSHA1MSG2rr_SHA1NEXTErr : SchedWriteRes<[Zn4FPU0123]> { def : InstRW<[Zn4WriteSHA1MSG2rr_SHA1NEXTErr], (instrs SHA1MSG2rr, SHA1NEXTErr)>; def Zn4Writerm_SHA1MSG2rm_SHA1NEXTErm : SchedWriteRes<[Zn4AGU012, Zn4Load, Zn4FPU0123]> { - let Latency = !add(Znver4Model.LoadLatency, Zn4WriteSHA1MSG2rr_SHA1NEXTErr.Latency); + let Latency = !add(Znver4Model.VecLoadLatency, Zn4WriteSHA1MSG2rr_SHA1NEXTErr.Latency); let ReleaseAtCycles = [1, 1, 2]; let NumMicroOps = !add(Zn4WriteSHA1MSG2rr_SHA1NEXTErr.NumMicroOps, 0); } @@ -1290,7 +1290,7 @@ def Zn4WriteSHA256MSG1rr : SchedWriteRes<[Zn4FPU0123]> { def : InstRW<[Zn4WriteSHA256MSG1rr], (instrs SHA256MSG1rr)>; def Zn4Writerm_SHA256MSG1rm : SchedWriteRes<[Zn4AGU012, Zn4Load, Zn4FPU0123]> { - let Latency = !add(Znver4Model.LoadLatency, Zn4WriteSHA256MSG1rr.Latency); + let Latency = !add(Znver4Model.VecLoadLatency, Zn4WriteSHA256MSG1rr.Latency); let ReleaseAtCycles = [1, 1, 3]; let NumMicroOps = !add(Zn4WriteSHA256MSG1rr.NumMicroOps, 0); } @@ -1304,7 +1304,7 @@ def Zn4WriteSHA256MSG2rr : SchedWriteRes<[Zn4FPU0123]> { def : InstRW<[Zn4WriteSHA256MSG2rr], (instrs SHA256MSG2rr)>; def Zn4WriteSHA256MSG2rm : SchedWriteRes<[Zn4AGU012, Zn4Load, Zn4FPU0123]> { - let Latency = !add(Znver4Model.LoadLatency, Zn4WriteSHA256MSG2rr.Latency); + let Latency = !add(Znver4Model.VecLoadLatency, Zn4WriteSHA256MSG2rr.Latency); let ReleaseAtCycles = [1, 1, 8]; let NumMicroOps = !add(Zn4WriteSHA256MSG2rr.NumMicroOps, 1); } @@ -1379,7 +1379,7 @@ def Zn4WriteVPERM2I128rr_VPERM2F128rr : SchedWriteRes<[Zn4FPVShuf]> { def : InstRW<[Zn4WriteVPERM2I128rr_VPERM2F128rr], (instrs VPERM2I128rri, VPERM2F128rri)>; def Zn4WriteVPERM2F128rm : SchedWriteRes<[Zn4AGU012, Zn4Load, Zn4FPVShuf]> { - let Latency = !add(Znver4Model.LoadLatency, Zn4WriteVPERM2I128rr_VPERM2F128rr.Latency); + let Latency = !add(Znver4Model.VecLoadLatency, Zn4WriteVPERM2I128rr_VPERM2F128rr.Latency); let ReleaseAtCycles = [1, 1, 1]; let NumMicroOps = !add(Zn4WriteVPERM2I128rr_VPERM2F128rr.NumMicroOps, 0); } @@ -1393,7 +1393,7 @@ def Zn4WriteVPERMPSYrr : SchedWriteRes<[Zn4FPVShuf]> { def : InstRW<[Zn4WriteVPERMPSYrr], (instrs VPERMPSYrr)>; def Zn4WriteVPERMPSYrm : SchedWriteRes<[Zn4AGU012, Zn4Load, Zn4FPVShuf]> { - let Latency = !add(Znver4Model.LoadLatency, Zn4WriteVPERMPSYrr.Latency); + let Latency = !add(Znver4Model.VecLoadLatency, Zn4WriteVPERMPSYrr.Latency); let ReleaseAtCycles = [1, 1, 2]; let NumMicroOps = !add(Zn4WriteVPERMPSYrr.NumMicroOps, 1); } @@ -1407,7 +1407,7 @@ def Zn4WriteVPERMYri : SchedWriteRes<[Zn4FPVShuf]> { def : InstRW<[Zn4WriteVPERMYri], (instrs VPERMPDYri, VPERMQYri)>; def Zn4WriteVPERMPDYmi : SchedWriteRes<[Zn4AGU012, Zn4Load, Zn4FPVShuf]> { - let Latency = !add(Znver4Model.LoadLatency, Zn4WriteVPERMYri.Latency); + let Latency = !add(Znver4Model.VecLoadLatency, Zn4WriteVPERMYri.Latency); let ReleaseAtCycles = [1, 1, 2]; let NumMicroOps = !add(Zn4WriteVPERMYri.NumMicroOps, 1); } @@ -1421,7 +1421,7 @@ def Zn4WriteVPERMDYrr : SchedWriteRes<[Zn4FPVShuf]> { def : InstRW<[Zn4WriteVPERMDYrr], (instrs VPERMDYrr)>; def Zn4WriteVPERMYm : SchedWriteRes<[Zn4AGU012, Zn4Load, Zn4FPVShuf]> { - let Latency = !add(Znver4Model.LoadLatency, Zn4WriteVPERMDYrr.Latency); + let Latency = !add(Znver4Model.VecLoadLatency, Zn4WriteVPERMDYrr.Latency); let ReleaseAtCycles = [1, 1, 2]; let NumMicroOps = !add(Zn4WriteVPERMDYrr.NumMicroOps, 0); } @@ -1534,9 +1534,9 @@ def Zn4WriteVFIXUPIMMPDZrr_VRANGESDrr : SchedWriteRes<[Zn4FPFMisc01]> { let NumMicroOps = 1; } def : InstRW<[Zn4WriteVFIXUPIMMPDZrr_VRANGESDrr], (instregex - "VFIXUPIMM(S|P)(S|D)(Z|Z128|Z256?)rrik", "VFIXUPIMM(S|P)(S|D)(Z?|Z128?|Z256?)rrikz", + "VFIXUPIMM(S|P)(S|D)(Z|Z128|Z256?)rrik", "VFIXUPIMM(S|P)(S|D)(Z?|Z128?|Z256?)rrikz", "VFIXUPIMM(S|P)(S|D)(Z128|Z256?)rri", "VRANGE(S|P)(S|D)(Z?|Z128?|Z256?)rri(b?)", - "VRANGE(S|P)(S|D)(Z|Z128|Z256?)rri(b?)k","VRANGE(S|P)(S|D)(Z?|Z128?|Z256?)rri(b?)kz" + "VRANGE(S|P)(S|D)(Z|Z128|Z256?)rri(b?)k","VRANGE(S|P)(S|D)(Z?|Z128?|Z256?)rri(b?)kz" )>; // SCALE & REDUCE instructions @@ -1567,7 +1567,7 @@ def Zn4WriteBUSDr_VPMADDr: SchedWriteRes<[Zn4FPFMisc01]> { let NumMicroOps = 1; } def : InstRW<[Zn4WriteBUSDr_VPMADDr], (instregex - "VPDP(BU|WS)(S|P)(S|D|DS)(Z|Z128|Z256)(r|rk|rkz)", + "VPDP(BU|WS)(S|P)(S|D|DS)(Z?|Z128?|Z256?|Y?)r(r|rk|rkz)", "VPMADD52(H|L)UQ(Z|Z128|Z256)(r|rk|rkz)" )>; @@ -1586,7 +1586,7 @@ def : InstRW<[Zn4WriteSHIFTrr], (instregex "(V?)P(ROL|ROR)(D|Q|VD|VQ)(Z?|Z128?|Z256?)(rr|rrk|rrkz)", "(V?)P(ROL|ROR)(D|Q|VD|VQ)(Z256?)(ri|rik|rikz)", "(V?)P(ROL|ROR)(D|Q)(Z?|Z128?)(ri|rik|rikz)", - "VPSHUFBITQMBZ128rr", "VFMSUB231SSZrkz_Int" + "VPSHUFBITQMBZ128rr", "VFMSUB231SSZrkz_Int" )>; def Zn4WriteSHIFTri: SchedWriteRes<[Zn4FPFMisc01]> { @@ -1598,24 +1598,40 @@ def : InstRW<[Zn4WriteSHIFTri], (instregex "VP(SLL|SRL|SRA)(D|Q|W)(Z|Z128|Z256?)(ri|rik|rikz)" )>; -// ALIGN Instructions -def Zn4WriteALIGN: SchedWriteRes<[Zn4FPFMisc12]> { +// ALIGNR Instructions +def Zn4WriteALIGNR: SchedWriteRes<[Zn4FPFMisc12]> { + let Latency = 2; + let ReleaseAtCycles = [1]; + let NumMicroOps = 1; +} +def : InstRW<[Zn4WriteALIGNR], (instregex + "(V?)PALIGNR(Y?|Z128?|Z256?)(rri|rrik|rrikz)" + )>; +def Zn4WriteALIGNRZ: SchedWriteRes<[Zn4FPFMisc12]> { let Latency = 2; let ReleaseAtCycles = [2]; let NumMicroOps = 1; } -def : InstRW<[Zn4WriteALIGN], (instregex - "(V?)PALIGNR(Z?|Z128?|Z256?)(rri|rrik|rrikz)" +def : InstRW<[Zn4WriteALIGNRZ], (instregex + "(V?)PALIGNRZ(rri|rrik|rrikz)" )>; -//PACK Instructions +// PACK Instructions def Zn4WritePACK: SchedWriteRes<[Zn4FPFMisc12]> { let Latency = 2; - let ReleaseAtCycles = [2]; + let ReleaseAtCycles = [1]; let NumMicroOps = 1; } def : InstRW<[Zn4WritePACK], (instregex - "(V?)PACK(SS|US)(DW|WB)(Z?|Z128?|Z256?)(rr|rrk|rrkz)" + "(V?)PACK(SS|US)(DW|WB)(Y?|Z128?|Z256?)(rr|rrk|rrkz)" + )>; +def Zn4WritePACKZ: SchedWriteRes<[Zn4FPFMisc12]> { + let Latency = 2; + let ReleaseAtCycles = [2]; + let NumMicroOps = 1; +} +def : InstRW<[Zn4WritePACKZ], (instregex + "(V?)PACK(SS|US)(DW|WB)Z(rr|rrk|rrkz)" )>; // MAX and MIN Instructions diff --git a/llvm/lib/Target/X86/X86SpeculativeLoadHardening.cpp b/llvm/lib/Target/X86/X86SpeculativeLoadHardening.cpp index 4cc456ece77e..c28de14a9787 100644 --- a/llvm/lib/Target/X86/X86SpeculativeLoadHardening.cpp +++ b/llvm/lib/Target/X86/X86SpeculativeLoadHardening.cpp @@ -893,6 +893,7 @@ void X86SpeculativeLoadHardeningPass::unfoldCallAndJumpLoads( case X86::TAILJMPm64_REX: case X86::TAILJMPm: case X86::TCRETURNmi64: + case X86::TCRETURN_WINmi64: case X86::TCRETURNmi: { // Use the generic unfold logic now that we know we're dealing with // expected instructions. diff --git a/llvm/lib/Target/X86/X86TargetMachine.cpp b/llvm/lib/Target/X86/X86TargetMachine.cpp index 6d9c6cdedd9e..babbe95cc780 100644 --- a/llvm/lib/Target/X86/X86TargetMachine.cpp +++ b/llvm/lib/Target/X86/X86TargetMachine.cpp @@ -125,54 +125,6 @@ static std::unique_ptr<TargetLoweringObjectFile> createTLOF(const Triple &TT) { return std::make_unique<X86ELFTargetObjectFile>(); } -static std::string computeDataLayout(const Triple &TT) { - // X86 is little endian - std::string Ret = "e"; - - Ret += DataLayout::getManglingComponent(TT); - // X86 and x32 have 32 bit pointers. - if (!TT.isArch64Bit() || TT.isX32()) - Ret += "-p:32:32"; - - // Address spaces for 32 bit signed, 32 bit unsigned, and 64 bit pointers. - Ret += "-p270:32:32-p271:32:32-p272:64:64"; - - // Some ABIs align 64 bit integers and doubles to 64 bits, others to 32. - // 128 bit integers are not specified in the 32-bit ABIs but are used - // internally for lowering f128, so we match the alignment to that. - if (TT.isArch64Bit() || TT.isOSWindows()) - Ret += "-i64:64-i128:128"; - else if (TT.isOSIAMCU()) - Ret += "-i64:32-f64:32"; - else - Ret += "-i128:128-f64:32:64"; - - // Some ABIs align long double to 128 bits, others to 32. - if (TT.isOSIAMCU()) - ; // No f80 - else if (TT.isArch64Bit() || TT.isOSDarwin() || TT.isWindowsMSVCEnvironment()) - Ret += "-f80:128"; - else - Ret += "-f80:32"; - - if (TT.isOSIAMCU()) - Ret += "-f128:32"; - - // The registers can hold 8, 16, 32 or, in x86-64, 64 bits. - if (TT.isArch64Bit()) - Ret += "-n8:16:32:64"; - else - Ret += "-n8:16:32"; - - // The stack is aligned to 32 bits on some ABIs and 128 bits on others. - if ((!TT.isArch64Bit() && TT.isOSWindows()) || TT.isOSIAMCU()) - Ret += "-a:0:32-S32"; - else - Ret += "-S128"; - - return Ret; -} - static Reloc::Model getEffectiveRelocModel(const Triple &TT, bool JIT, std::optional<Reloc::Model> RM) { bool is64Bit = TT.getArch() == Triple::x86_64; @@ -236,7 +188,7 @@ X86TargetMachine::X86TargetMachine(const Target &T, const Triple &TT, std::optional<Reloc::Model> RM, std::optional<CodeModel::Model> CM, CodeGenOptLevel OL, bool JIT) - : CodeGenTargetMachineImpl(T, computeDataLayout(TT), TT, CPU, FS, Options, + : CodeGenTargetMachineImpl(T, TT.computeDataLayout(), TT, CPU, FS, Options, getEffectiveRelocModel(TT, JIT, RM), getEffectiveX86CodeModel(TT, CM, JIT), OL), TLOF(createTLOF(getTargetTriple())), IsJIT(JIT) { diff --git a/llvm/lib/Target/Xtensa/XtensaTargetMachine.cpp b/llvm/lib/Target/Xtensa/XtensaTargetMachine.cpp index c9f1ca8b46da..72cb61b5e864 100644 --- a/llvm/lib/Target/Xtensa/XtensaTargetMachine.cpp +++ b/llvm/lib/Target/Xtensa/XtensaTargetMachine.cpp @@ -32,13 +32,6 @@ extern "C" LLVM_EXTERNAL_VISIBILITY void LLVMInitializeXtensaTarget() { initializeXtensaAsmPrinterPass(PR); } -static std::string computeDataLayout(const Triple &TT, StringRef CPU, - const TargetOptions &Options, - bool IsLittle) { - std::string Ret = "e-m:e-p:32:32-i8:8:32-i16:16:32-i64:64-n32"; - return Ret; -} - static Reloc::Model getEffectiveRelocModel(bool JIT, std::optional<Reloc::Model> RM) { if (!RM || JIT) @@ -53,8 +46,7 @@ XtensaTargetMachine::XtensaTargetMachine(const Target &T, const Triple &TT, std::optional<CodeModel::Model> CM, CodeGenOptLevel OL, bool JIT, bool IsLittle) - : CodeGenTargetMachineImpl(T, computeDataLayout(TT, CPU, Options, IsLittle), - TT, CPU, FS, Options, + : CodeGenTargetMachineImpl(T, TT.computeDataLayout(), TT, CPU, FS, Options, getEffectiveRelocModel(JIT, RM), getEffectiveCodeModel(CM, CodeModel::Small), OL), TLOF(std::make_unique<TargetLoweringObjectFileELF>()) { diff --git a/llvm/lib/TargetParser/CMakeLists.txt b/llvm/lib/TargetParser/CMakeLists.txt index 5eecfbf80b2f..e1a30199e1ad 100644 --- a/llvm/lib/TargetParser/CMakeLists.txt +++ b/llvm/lib/TargetParser/CMakeLists.txt @@ -26,6 +26,7 @@ add_llvm_component_library(LLVMTargetParser SubtargetFeature.cpp TargetParser.cpp Triple.cpp + TargetDataLayout.cpp X86TargetParser.cpp XtensaTargetParser.cpp diff --git a/llvm/lib/TargetParser/RISCVISAInfo.cpp b/llvm/lib/TargetParser/RISCVISAInfo.cpp index 7617058aa102..9268df29f523 100644 --- a/llvm/lib/TargetParser/RISCVISAInfo.cpp +++ b/llvm/lib/TargetParser/RISCVISAInfo.cpp @@ -287,11 +287,6 @@ std::vector<std::string> RISCVISAInfo::toFeatures(bool AddAllExtensions, bool IgnoreUnknown) const { std::vector<std::string> Features; for (const auto &[ExtName, _] : Exts) { - // i is a base instruction set, not an extension (see - // https://github.com/riscv/riscv-isa-manual/blob/main/src/naming.adoc#base-integer-isa) - // and is not recognized in clang -cc1 - if (ExtName == "i") - continue; if (IgnoreUnknown && !isSupportedExtension(ExtName)) continue; diff --git a/llvm/lib/TargetParser/TargetDataLayout.cpp b/llvm/lib/TargetParser/TargetDataLayout.cpp new file mode 100644 index 000000000000..e222588ea389 --- /dev/null +++ b/llvm/lib/TargetParser/TargetDataLayout.cpp @@ -0,0 +1,629 @@ +//===--- TargetDataLayout.cpp - Map Triple to LLVM data layout string -----===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#include "llvm/ADT/StringRef.h" +#include "llvm/Support/CommandLine.h" +#include "llvm/TargetParser/ARMTargetParser.h" +#include "llvm/TargetParser/Triple.h" +#include <cstring> +using namespace llvm; + +static StringRef getManglingComponent(const Triple &T) { + if (T.isOSBinFormatGOFF()) + return "-m:l"; + if (T.isOSBinFormatMachO()) + return "-m:o"; + if ((T.isOSWindows() || T.isUEFI()) && T.isOSBinFormatCOFF()) + return T.getArch() == Triple::x86 ? "-m:x" : "-m:w"; + if (T.isOSBinFormatXCOFF()) + return "-m:a"; + return "-m:e"; +} + +static std::string computeARMDataLayout(const Triple &TT, StringRef ABIName) { + auto ABI = ARM::computeTargetABI(TT, ABIName); + std::string Ret; + + if (TT.isLittleEndian()) + // Little endian. + Ret += "e"; + else + // Big endian. + Ret += "E"; + + Ret += getManglingComponent(TT); + + // Pointers are 32 bits and aligned to 32 bits. + Ret += "-p:32:32"; + + // Function pointers are aligned to 8 bits (because the LSB stores the + // ARM/Thumb state). + Ret += "-Fi8"; + + // ABIs other than APCS have 64 bit integers with natural alignment. + if (ABI != ARM::ARM_ABI_APCS) + Ret += "-i64:64"; + + // We have 64 bits floats. The APCS ABI requires them to be aligned to 32 + // bits, others to 64 bits. We always try to align to 64 bits. + if (ABI == ARM::ARM_ABI_APCS) + Ret += "-f64:32:64"; + + // We have 128 and 64 bit vectors. The APCS ABI aligns them to 32 bits, others + // to 64. We always ty to give them natural alignment. + if (ABI == ARM::ARM_ABI_APCS) + Ret += "-v64:32:64-v128:32:128"; + else if (ABI != ARM::ARM_ABI_AAPCS16) + Ret += "-v128:64:128"; + + // Try to align aggregates to 32 bits (the default is 64 bits, which has no + // particular hardware support on 32-bit ARM). + Ret += "-a:0:32"; + + // Integer registers are 32 bits. + Ret += "-n32"; + + // The stack is 64 bit aligned on AAPCS and 32 bit aligned everywhere else. + if (ABI == ARM::ARM_ABI_AAPCS16) + Ret += "-S128"; + else if (ABI == ARM::ARM_ABI_AAPCS) + Ret += "-S64"; + else + Ret += "-S32"; + + return Ret; +} + +// Helper function to build a DataLayout string +static std::string computeAArch64DataLayout(const Triple &TT) { + if (TT.isOSBinFormatMachO()) { + if (TT.getArch() == Triple::aarch64_32) + return "e-m:o-p:32:32-p270:32:32-p271:32:32-p272:64:64-i64:64-i128:128-" + "n32:64-S128-Fn32"; + return "e-m:o-p270:32:32-p271:32:32-p272:64:64-i64:64-i128:128-n32:64-S128-" + "Fn32"; + } + if (TT.isOSBinFormatCOFF()) + return "e-m:w-p270:32:32-p271:32:32-p272:64:64-p:64:64-i32:32-i64:64-i128:" + "128-n32:64-S128-Fn32"; + std::string Endian = TT.isLittleEndian() ? "e" : "E"; + std::string Ptr32 = TT.getEnvironment() == Triple::GNUILP32 ? "-p:32:32" : ""; + return Endian + "-m:e" + Ptr32 + + "-p270:32:32-p271:32:32-p272:64:64-i8:8:32-i16:16:32-i64:64-i128:128-" + "n32:64-S128-Fn32"; +} + +// DataLayout: little or big endian +static std::string computeBPFDataLayout(const Triple &TT) { + if (TT.getArch() == Triple::bpfeb) + return "E-m:e-p:64:64-i64:64-i128:128-n32:64-S128"; + else + return "e-m:e-p:64:64-i64:64-i128:128-n32:64-S128"; +} + +static std::string computeCSKYDataLayout(const Triple &TT) { + // CSKY is always 32-bit target with the CSKYv2 ABI as prefer now. + // It's a 4-byte aligned stack with ELF mangling only. + // Only support little endian for now. + // TODO: Add support for big endian. + return "e-m:e-S32-p:32:32-i32:32:32-i64:32:32-f32:32:32-f64:32:32-v64:32:32" + "-v128:32:32-a:0:32-Fi32-n32"; +} + +static std::string computeLoongArchDataLayout(const Triple &TT) { + if (TT.isLoongArch64()) + return "e-m:e-p:64:64-i64:64-i128:128-n32:64-S128"; + assert(TT.isLoongArch32() && "only LA32 and LA64 are currently supported"); + return "e-m:e-p:32:32-i64:64-n32-S128"; +} + +static std::string computeM68kDataLayout(const Triple &TT) { + std::string Ret = ""; + // M68k is Big Endian + Ret += "E"; + + // FIXME how to wire it with the used object format? + Ret += "-m:e"; + + // M68k pointers are always 32 bit wide even for 16-bit CPUs. + // The ABI only specifies 16-bit alignment. + // On at least the 68020+ with a 32-bit bus, there is a performance benefit + // to having 32-bit alignment. + Ret += "-p:32:16:32"; + + // Bytes do not require special alignment, words are word aligned and + // long words are word aligned at minimum. + Ret += "-i8:8:8-i16:16:16-i32:16:32"; + + // FIXME no floats at the moment + + // The registers can hold 8, 16, 32 bits + Ret += "-n8:16:32"; + + Ret += "-a:0:16-S16"; + + return Ret; +} + +namespace { +enum class MipsABI { Unknown, O32, N32, N64 }; +} + +// FIXME: This duplicates MipsABIInfo::computeTargetABI, but duplicating this is +// preferable to violating layering rules. Ideally that information should live +// in LLVM TargetParser, but for now we just duplicate some ABI name string +// logic for simplicity. +static MipsABI getMipsABI(const Triple &TT, StringRef ABIName) { + if (ABIName.starts_with("o32")) + return MipsABI::O32; + if (ABIName.starts_with("n32")) + return MipsABI::N32; + if (ABIName.starts_with("n64")) + return MipsABI::N64; + if (TT.isABIN32()) + return MipsABI::N32; + assert(ABIName.empty() && "Unknown ABI option for MIPS"); + + if (TT.isMIPS64()) + return MipsABI::N64; + return MipsABI::O32; +} + +static std::string computeMipsDataLayout(const Triple &TT, StringRef ABIName) { + std::string Ret; + MipsABI ABI = getMipsABI(TT, ABIName); + + // There are both little and big endian mips. + if (TT.isLittleEndian()) + Ret += "e"; + else + Ret += "E"; + + if (ABI == MipsABI::O32) + Ret += "-m:m"; + else + Ret += "-m:e"; + + // Pointers are 32 bit on some ABIs. + if (ABI != MipsABI::N64) + Ret += "-p:32:32"; + + // 8 and 16 bit integers only need to have natural alignment, but try to + // align them to 32 bits. 64 bit integers have natural alignment. + Ret += "-i8:8:32-i16:16:32-i64:64"; + + // 32 bit registers are always available and the stack is at least 64 bit + // aligned. On N64 64 bit registers are also available and the stack is + // 128 bit aligned. + if (ABI == MipsABI::N64 || ABI == MipsABI::N32) + Ret += "-i128:128-n32:64-S128"; + else + Ret += "-n32-S64"; + + return Ret; +} + +static std::string computePowerDataLayout(const Triple &T) { + bool is64Bit = T.isPPC64(); + std::string Ret; + + // Most PPC* platforms are big endian, PPC(64)LE is little endian. + if (T.isLittleEndian()) + Ret = "e"; + else + Ret = "E"; + + Ret += getManglingComponent(T); + + // PPC32 has 32 bit pointers. The PS3 (OS Lv2) is a PPC64 machine with 32 bit + // pointers. + if (!is64Bit || T.getOS() == Triple::Lv2) + Ret += "-p:32:32"; + + // If the target ABI uses function descriptors, then the alignment of function + // pointers depends on the alignment used to emit the descriptor. Otherwise, + // function pointers are aligned to 32 bits because the instructions must be. + if ((T.getArch() == Triple::ppc64 && !T.isPPC64ELFv2ABI())) { + Ret += "-Fi64"; + } else if (T.isOSAIX()) { + Ret += is64Bit ? "-Fi64" : "-Fi32"; + } else { + Ret += "-Fn32"; + } + + // Note, the alignment values for f64 and i64 on ppc64 in Darwin + // documentation are wrong; these are correct (i.e. "what gcc does"). + Ret += "-i64:64"; + + // PPC64 has 32 and 64 bit registers, PPC32 has only 32 bit ones. + if (is64Bit) + Ret += "-i128:128-n32:64"; + else + Ret += "-n32"; + + // Specify the vector alignment explicitly. For v256i1 and v512i1, the + // calculated alignment would be 256*alignment(i1) and 512*alignment(i1), + // which is 256 and 512 bytes - way over aligned. + if (is64Bit && (T.isOSAIX() || T.isOSLinux())) + Ret += "-S128-v256:256:256-v512:512:512"; + + return Ret; +} + +static std::string computeAMDDataLayout(const Triple &TT) { + if (TT.getArch() == Triple::r600) { + // 32-bit pointers. + return "e-p:32:32-i64:64-v16:16-v24:32-v32:32-v48:64-v96:128" + "-v192:256-v256:256-v512:512-v1024:1024-v2048:2048-n32:64-S32-A5-G1"; + } + + // 32-bit private, local, and region pointers. 64-bit global, constant and + // flat. 160-bit non-integral fat buffer pointers that include a 128-bit + // buffer descriptor and a 32-bit offset, which are indexed by 32-bit values + // (address space 7), and 128-bit non-integral buffer resourcees (address + // space 8) which cannot be non-trivilally accessed by LLVM memory operations + // like getelementptr. + return "e-p:64:64-p1:64:64-p2:32:32-p3:32:32-p4:64:64-p5:32:32-p6:32:32" + "-p7:160:256:256:32-p8:128:128:128:48-p9:192:256:256:32-i64:64-" + "v16:16-v24:32-v32:32-v48:64-v96:128-v192:256-v256:256-v512:512-" + "v1024:1024-v2048:2048-n32:64-S32-A5-G1-ni:7:8:9"; +} + +static std::string computeRISCVDataLayout(const Triple &TT, StringRef ABIName) { + std::string Ret; + + if (TT.isLittleEndian()) + Ret += "e"; + else + Ret += "E"; + + Ret += "-m:e"; + + // Pointer and integer sizes. + if (TT.isRISCV64()) { + Ret += "-p:64:64-i64:64-i128:128"; + Ret += "-n32:64"; + } else { + assert(TT.isRISCV32() && "only RV32 and RV64 are currently supported"); + Ret += "-p:32:32-i64:64"; + Ret += "-n32"; + } + + // Stack alignment based on ABI. + StringRef ABI = ABIName; + if (ABI == "ilp32e") + Ret += "-S32"; + else if (ABI == "lp64e") + Ret += "-S64"; + else + Ret += "-S128"; + + return Ret; +} + +static std::string computeSparcDataLayout(const Triple &T) { + const bool Is64Bit = T.isSPARC64(); + + // Sparc is typically big endian, but some are little. + std::string Ret = T.getArch() == Triple::sparcel ? "e" : "E"; + Ret += "-m:e"; + + // Some ABIs have 32bit pointers. + if (!Is64Bit) + Ret += "-p:32:32"; + + // Alignments for 64 bit integers. + Ret += "-i64:64"; + + // Alignments for 128 bit integers. + // This is not specified in the ABI document but is the de facto standard. + Ret += "-i128:128"; + + // On SparcV9 128 floats are aligned to 128 bits, on others only to 64. + // On SparcV9 registers can hold 64 or 32 bits, on others only 32. + if (Is64Bit) + Ret += "-n32:64"; + else + Ret += "-f128:64-n32"; + + if (Is64Bit) + Ret += "-S128"; + else + Ret += "-S64"; + + return Ret; +} + +static std::string computeSystemZDataLayout(const Triple &TT) { + std::string Ret; + + // Big endian. + Ret += "E"; + + // Data mangling. + Ret += getManglingComponent(TT); + + // Special features for z/OS. + if (TT.isOSzOS()) { + // Custom address space for ptr32. + Ret += "-p1:32:32"; + } + + // Make sure that global data has at least 16 bits of alignment by + // default, so that we can refer to it using LARL. We don't have any + // special requirements for stack variables though. + Ret += "-i1:8:16-i8:8:16"; + + // 64-bit integers are naturally aligned. + Ret += "-i64:64"; + + // 128-bit floats are aligned only to 64 bits. + Ret += "-f128:64"; + + // The DataLayout string always holds a vector alignment of 64 bits, see + // comment in clang/lib/Basic/Targets/SystemZ.h. + Ret += "-v128:64"; + + // We prefer 16 bits of aligned for all globals; see above. + Ret += "-a:8:16"; + + // Integer registers are 32 or 64 bits. + Ret += "-n32:64"; + + return Ret; +} + +static std::string computeX86DataLayout(const Triple &TT) { + bool Is64Bit = TT.getArch() == Triple::x86_64; + + // X86 is little endian + std::string Ret = "e"; + + Ret += getManglingComponent(TT); + // X86 and x32 have 32 bit pointers. + if (!Is64Bit || TT.isX32()) + Ret += "-p:32:32"; + + // Address spaces for 32 bit signed, 32 bit unsigned, and 64 bit pointers. + Ret += "-p270:32:32-p271:32:32-p272:64:64"; + + // Some ABIs align 64 bit integers and doubles to 64 bits, others to 32. + // 128 bit integers are not specified in the 32-bit ABIs but are used + // internally for lowering f128, so we match the alignment to that. + if (Is64Bit || TT.isOSWindows()) + Ret += "-i64:64-i128:128"; + else if (TT.isOSIAMCU()) + Ret += "-i64:32-f64:32"; + else + Ret += "-i128:128-f64:32:64"; + + // Some ABIs align long double to 128 bits, others to 32. + if (TT.isOSIAMCU()) + ; // No f80 + else if (Is64Bit || TT.isOSDarwin() || TT.isWindowsMSVCEnvironment()) + Ret += "-f80:128"; + else + Ret += "-f80:32"; + + if (TT.isOSIAMCU()) + Ret += "-f128:32"; + + // The registers can hold 8, 16, 32 or, in x86-64, 64 bits. + if (Is64Bit) + Ret += "-n8:16:32:64"; + else + Ret += "-n8:16:32"; + + // The stack is aligned to 32 bits on some ABIs and 128 bits on others. + if ((!Is64Bit && TT.isOSWindows()) || TT.isOSIAMCU()) + Ret += "-a:0:32-S32"; + else + Ret += "-S128"; + + return Ret; +} + +static std::string computeNVPTXDataLayout(const Triple &T, StringRef ABIName) { + bool Is64Bit = T.getArch() == Triple::nvptx64; + std::string Ret = "e"; + + // Tensor Memory (addrspace:6) is always 32-bits. + // Distributed Shared Memory (addrspace:7) follows shared memory + // (addrspace:3). + if (!Is64Bit) + Ret += "-p:32:32-p6:32:32-p7:32:32"; + else if (ABIName == "shortptr") + Ret += "-p3:32:32-p4:32:32-p5:32:32-p6:32:32-p7:32:32"; + else + Ret += "-p6:32:32"; + + Ret += "-i64:64-i128:128-i256:256-v16:16-v32:32-n16:32:64"; + + return Ret; +} + +static std::string computeSPIRVDataLayout(const Triple &TT) { + const auto Arch = TT.getArch(); + // TODO: this probably needs to be revisited: + // Logical SPIR-V has no pointer size, so any fixed pointer size would be + // wrong. The choice to default to 32 or 64 is just motivated by another + // memory model used for graphics: PhysicalStorageBuffer64. But it shouldn't + // mean anything. + if (Arch == Triple::spirv32) + return "e-p:32:32-i64:64-v16:16-v24:32-v32:32-v48:64-v96:128-v192:256-" + "v256:256-v512:512-v1024:1024-n8:16:32:64-G1"; + if (Arch == Triple::spirv) + return "e-i64:64-v16:16-v24:32-v32:32-v48:64-v96:128-v192:256-v256:256-" + "v512:512-v1024:1024-n8:16:32:64-G10"; + if (TT.getVendor() == Triple::VendorType::AMD && + TT.getOS() == Triple::OSType::AMDHSA) + return "e-i64:64-v16:16-v24:32-v32:32-v48:64-v96:128-v192:256-v256:256-" + "v512:512-v1024:1024-n32:64-S32-G1-P4-A0"; + if (TT.getVendor() == Triple::VendorType::Intel) + return "e-i64:64-v16:16-v24:32-v32:32-v48:64-v96:128-v192:256-v256:256-" + "v512:512-v1024:1024-n8:16:32:64-G1-P9-A0"; + return "e-i64:64-v16:16-v24:32-v32:32-v48:64-v96:128-v192:256-v256:256-" + "v512:512-v1024:1024-n8:16:32:64-G1"; +} + +static std::string computeLanaiDataLayout() { + // Data layout (keep in sync with clang/lib/Basic/Targets.cpp) + return "E" // Big endian + "-m:e" // ELF name manging + "-p:32:32" // 32-bit pointers, 32 bit aligned + "-i64:64" // 64 bit integers, 64 bit aligned + "-a:0:32" // 32 bit alignment of objects of aggregate type + "-n32" // 32 bit native integer width + "-S64"; // 64 bit natural stack alignment +} + +static std::string computeWebAssemblyDataLayout(const Triple &TT) { + return TT.getArch() == Triple::wasm64 + ? (TT.isOSEmscripten() ? "e-m:e-p:64:64-p10:8:8-p20:8:8-i64:64-" + "i128:128-f128:64-n32:64-S128-ni:1:10:20" + : "e-m:e-p:64:64-p10:8:8-p20:8:8-i64:64-" + "i128:128-n32:64-S128-ni:1:10:20") + : (TT.isOSEmscripten() ? "e-m:e-p:32:32-p10:8:8-p20:8:8-i64:64-" + "i128:128-f128:64-n32:64-S128-ni:1:10:20" + : "e-m:e-p:32:32-p10:8:8-p20:8:8-i64:64-" + "i128:128-n32:64-S128-ni:1:10:20"); +} + +static std::string computeVEDataLayout(const Triple &T) { + // Aurora VE is little endian + std::string Ret = "e"; + + // Use ELF mangling + Ret += "-m:e"; + + // Alignments for 64 bit integers. + Ret += "-i64:64"; + + // VE supports 32 bit and 64 bits integer on registers + Ret += "-n32:64"; + + // Stack alignment is 128 bits + Ret += "-S128"; + + // Vector alignments are 64 bits + // Need to define all of them. Otherwise, each alignment becomes + // the size of each data by default. + Ret += "-v64:64:64"; // for v2f32 + Ret += "-v128:64:64"; + Ret += "-v256:64:64"; + Ret += "-v512:64:64"; + Ret += "-v1024:64:64"; + Ret += "-v2048:64:64"; + Ret += "-v4096:64:64"; + Ret += "-v8192:64:64"; + Ret += "-v16384:64:64"; // for v256f64 + + return Ret; +} + +std::string Triple::computeDataLayout(StringRef ABIName) const { + switch (getArch()) { + case Triple::arm: + case Triple::armeb: + case Triple::thumb: + case Triple::thumbeb: + return computeARMDataLayout(*this, ABIName); + case Triple::aarch64: + case Triple::aarch64_be: + case Triple::aarch64_32: + return computeAArch64DataLayout(*this); + case Triple::arc: + return "e-m:e-p:32:32-i1:8:32-i8:8:32-i16:16:32-i32:32:32-" + "f32:32:32-i64:32-f64:32-a:0:32-n32"; + case Triple::avr: + return "e-P1-p:16:8-i8:8-i16:8-i32:8-i64:8-f32:8-f64:8-n8:16-a:8"; + case Triple::bpfel: + case Triple::bpfeb: + return computeBPFDataLayout(*this); + case Triple::csky: + return computeCSKYDataLayout(*this); + case Triple::dxil: + return "e-m:e-p:32:32-i1:32-i8:8-i16:16-i32:32-i64:64-f16:16-" + "f32:32-f64:64-n8:16:32:64"; + case Triple::hexagon: + return "e-m:e-p:32:32:32-a:0-n16:32-" + "i64:64:64-i32:32:32-i16:16:16-i1:8:8-f32:32:32-f64:64:64-" + "v32:32:32-v64:64:64-v512:512:512-v1024:1024:1024-v2048:2048:2048"; + case Triple::loongarch32: + case Triple::loongarch64: + return computeLoongArchDataLayout(*this); + case Triple::m68k: + return computeM68kDataLayout(*this); + case Triple::mips: + case Triple::mipsel: + case Triple::mips64: + case Triple::mips64el: + return computeMipsDataLayout(*this, ABIName); + case Triple::msp430: + return "e-m:e-p:16:16-i32:16-i64:16-f32:16-f64:16-a:8-n8:16-S16"; + case Triple::ppc: + case Triple::ppcle: + case Triple::ppc64: + case Triple::ppc64le: + return computePowerDataLayout(*this); + case Triple::r600: + case Triple::amdgcn: + return computeAMDDataLayout(*this); + case Triple::riscv32: + case Triple::riscv64: + case Triple::riscv32be: + case Triple::riscv64be: + return computeRISCVDataLayout(*this, ABIName); + case Triple::sparc: + case Triple::sparcv9: + case Triple::sparcel: + return computeSparcDataLayout(*this); + case Triple::systemz: + return computeSystemZDataLayout(*this); + case Triple::tce: + case Triple::tcele: + case Triple::x86: + case Triple::x86_64: + return computeX86DataLayout(*this); + case Triple::xcore: + case Triple::xtensa: + return "e-m:e-p:32:32-i8:8:32-i16:16:32-i64:64-n32"; + case Triple::nvptx: + case Triple::nvptx64: + return computeNVPTXDataLayout(*this, ABIName); + case Triple::spir: + case Triple::spir64: + case Triple::spirv: + case Triple::spirv32: + case Triple::spirv64: + return computeSPIRVDataLayout(*this); + case Triple::lanai: + return computeLanaiDataLayout(); + case Triple::wasm32: + case Triple::wasm64: + return computeWebAssemblyDataLayout(*this); + case Triple::ve: + return computeVEDataLayout(*this); + + case Triple::amdil: + case Triple::amdil64: + case Triple::hsail: + case Triple::hsail64: + case Triple::kalimba: + case Triple::shave: + case Triple::renderscript32: + case Triple::renderscript64: + // These are all virtual ISAs with no LLVM backend, and therefore no fixed + // LLVM data layout. + return ""; + + case Triple::UnknownArch: + return ""; + } + llvm_unreachable("Invalid arch"); +} diff --git a/llvm/lib/TargetParser/Triple.cpp b/llvm/lib/TargetParser/Triple.cpp index 71517e5e9e83..ac3626db46ea 100644 --- a/llvm/lib/TargetParser/Triple.cpp +++ b/llvm/lib/TargetParser/Triple.cpp @@ -391,6 +391,8 @@ StringRef Triple::getEnvironmentTypeName(EnvironmentType Kind) { case Callable: return "callable"; case Mesh: return "mesh"; case Amplification: return "amplification"; + case RootSignature: + return "rootsignature"; case OpenCL: return "opencl"; case OpenHOS: return "ohos"; @@ -787,6 +789,7 @@ static Triple::EnvironmentType parseEnvironment(StringRef EnvironmentName) { .StartsWith("callable", Triple::Callable) .StartsWith("mesh", Triple::Mesh) .StartsWith("amplification", Triple::Amplification) + .StartsWith("rootsignature", Triple::RootSignature) .StartsWith("opencl", Triple::OpenCL) .StartsWith("ohos", Triple::OpenHOS) .StartsWith("pauthtest", Triple::PAuthTest) diff --git a/llvm/lib/Transforms/Coroutines/SpillUtils.cpp b/llvm/lib/Transforms/Coroutines/SpillUtils.cpp index d5d60a320b52..e474c07c8231 100644 --- a/llvm/lib/Transforms/Coroutines/SpillUtils.cpp +++ b/llvm/lib/Transforms/Coroutines/SpillUtils.cpp @@ -183,6 +183,16 @@ struct AllocaUseVisitor : PtrUseVisitor<AllocaUseVisitor> { handleAlias(I); } + void visitInsertElementInst(InsertElementInst &I) { + enqueueUsers(I); + handleAlias(I); + } + + void visitInsertValueInst(InsertValueInst &I) { + enqueueUsers(I); + handleAlias(I); + } + void visitStoreInst(StoreInst &SI) { // Regardless whether the alias of the alloca is the value operand or the // pointer operand, we need to assume the alloca is been written. diff --git a/llvm/lib/Transforms/IPO/AttributorAttributes.cpp b/llvm/lib/Transforms/IPO/AttributorAttributes.cpp index ff4da3378f82..6d1659958058 100644 --- a/llvm/lib/Transforms/IPO/AttributorAttributes.cpp +++ b/llvm/lib/Transforms/IPO/AttributorAttributes.cpp @@ -8561,7 +8561,8 @@ protected: /// Mapping from *single* memory location kinds, e.g., LOCAL_MEM with the /// value of NO_LOCAL_MEM, to the accesses encountered for this memory kind. using AccessSet = SmallSet<AccessInfo, 2, AccessInfo>; - std::array<AccessSet *, llvm::CTLog2<VALID_STATE>()> AccessKind2Accesses; + std::array<AccessSet *, llvm::ConstantLog2<VALID_STATE>()> + AccessKind2Accesses; /// Categorize the pointer arguments of CB that might access memory in /// AccessedLoc and update the state and access map accordingly. diff --git a/llvm/lib/Transforms/IPO/FunctionSpecialization.cpp b/llvm/lib/Transforms/IPO/FunctionSpecialization.cpp index a459a9eddbcf..20d399315fac 100644 --- a/llvm/lib/Transforms/IPO/FunctionSpecialization.cpp +++ b/llvm/lib/Transforms/IPO/FunctionSpecialization.cpp @@ -89,6 +89,8 @@ static cl::opt<bool> SpecializeLiteralConstant( "Enable specialization of functions that take a literal constant as an " "argument")); +extern cl::opt<bool> ProfcheckDisableMetadataFixes; + bool InstCostVisitor::canEliminateSuccessor(BasicBlock *BB, BasicBlock *Succ) const { unsigned I = 0; @@ -784,9 +786,31 @@ bool FunctionSpecializer::run() { // Update the known call sites to call the clone. for (CallBase *Call : S.CallSites) { + Function *Clone = S.Clone; LLVM_DEBUG(dbgs() << "FnSpecialization: Redirecting " << *Call - << " to call " << S.Clone->getName() << "\n"); + << " to call " << Clone->getName() << "\n"); Call->setCalledFunction(S.Clone); + auto &BFI = GetBFI(*Call->getFunction()); + std::optional<uint64_t> Count = + BFI.getBlockProfileCount(Call->getParent()); + if (Count && !ProfcheckDisableMetadataFixes) { + std::optional<llvm::Function::ProfileCount> MaybeCloneCount = + Clone->getEntryCount(); + assert(MaybeCloneCount && "Clone entry count was not set!"); + uint64_t CallCount = *Count + MaybeCloneCount->getCount(); + Clone->setEntryCount(CallCount); + if (std::optional<llvm::Function::ProfileCount> MaybeOriginalCount = + S.F->getEntryCount()) { + uint64_t OriginalCount = MaybeOriginalCount->getCount(); + if (OriginalCount >= *Count) { + S.F->setEntryCount(OriginalCount - *Count); + } else { + // This should generally not happen as that would mean there are + // more computed calls to the function than what was recorded. + LLVM_DEBUG(S.F->setEntryCount(0)); + } + } + } } Clones.push_back(S.Clone); @@ -1043,6 +1067,9 @@ Function *FunctionSpecializer::createSpecialization(Function *F, // clone must. Clone->setLinkage(GlobalValue::InternalLinkage); + if (F->getEntryCount() && !ProfcheckDisableMetadataFixes) + Clone->setEntryCount(0); + // Initialize the lattice state of the arguments of the function clone, // marking the argument on which we specialized the function constant // with the given value. diff --git a/llvm/lib/Transforms/IPO/GlobalOpt.cpp b/llvm/lib/Transforms/IPO/GlobalOpt.cpp index d7edd1288309..f88d51f443bc 100644 --- a/llvm/lib/Transforms/IPO/GlobalOpt.cpp +++ b/llvm/lib/Transforms/IPO/GlobalOpt.cpp @@ -2551,7 +2551,8 @@ static bool OptimizeNonTrivialIFuncs( })) continue; - assert(!Callees.empty() && "Expecting successful collection of versions"); + if (Callees.empty()) + continue; LLVM_DEBUG(dbgs() << "Statically resolving calls to function " << Resolver->getName() << "\n"); diff --git a/llvm/lib/Transforms/IPO/LowerTypeTests.cpp b/llvm/lib/Transforms/IPO/LowerTypeTests.cpp index 821a9d82ddb0..be6cba38d8b3 100644 --- a/llvm/lib/Transforms/IPO/LowerTypeTests.cpp +++ b/llvm/lib/Transforms/IPO/LowerTypeTests.cpp @@ -573,28 +573,11 @@ public: }; } // end anonymous namespace -/// Build a bit set for TypeId using the object layouts in -/// GlobalLayout. -static BitSetInfo -buildBitSet(Metadata *TypeId, - const DenseMap<GlobalTypeMember *, uint64_t> &GlobalLayout) { - BitSetBuilder BSB; - +/// Build a bit set for list of offsets. +static BitSetInfo buildBitSet(ArrayRef<uint64_t> Offsets) { // Compute the byte offset of each address associated with this type // identifier. - for (const auto &GlobalAndOffset : GlobalLayout) { - for (MDNode *Type : GlobalAndOffset.first->types()) { - if (Type->getOperand(1) != TypeId) - continue; - uint64_t Offset = - cast<ConstantInt>( - cast<ConstantAsMetadata>(Type->getOperand(0))->getValue()) - ->getZExtValue(); - BSB.addOffset(GlobalAndOffset.second + Offset); - } - } - - return BSB.build(); + return BitSetBuilder(Offsets).build(); } /// Build a test that bit BitOffset mod sizeof(Bits)*8 is set in @@ -1161,21 +1144,47 @@ void LowerTypeTestsModule::importFunction(Function *F, F->setVisibility(Visibility); } -void LowerTypeTestsModule::lowerTypeTestCalls( - ArrayRef<Metadata *> TypeIds, Constant *CombinedGlobalAddr, - const DenseMap<GlobalTypeMember *, uint64_t> &GlobalLayout) { - // For each type identifier in this disjoint set... +static auto +buildBitSets(ArrayRef<Metadata *> TypeIds, + const DenseMap<GlobalTypeMember *, uint64_t> &GlobalLayout) { + DenseMap<Metadata *, SmallVector<uint64_t, 16>> OffsetsByTypeID; + // Pre-populate the map with interesting type identifiers. + for (Metadata *TypeId : TypeIds) + OffsetsByTypeID[TypeId]; + for (const auto &[Mem, MemOff] : GlobalLayout) { + for (MDNode *Type : Mem->types()) { + auto It = OffsetsByTypeID.find(Type->getOperand(1)); + if (It == OffsetsByTypeID.end()) + continue; + uint64_t Offset = + cast<ConstantInt>( + cast<ConstantAsMetadata>(Type->getOperand(0))->getValue()) + ->getZExtValue(); + It->second.push_back(MemOff + Offset); + } + } + + SmallVector<std::pair<Metadata *, BitSetInfo>> BitSets; + BitSets.reserve(TypeIds.size()); for (Metadata *TypeId : TypeIds) { - // Build the bitset. - BitSetInfo BSI = buildBitSet(TypeId, GlobalLayout); + BitSets.emplace_back(TypeId, buildBitSet(OffsetsByTypeID[TypeId])); LLVM_DEBUG({ if (auto MDS = dyn_cast<MDString>(TypeId)) dbgs() << MDS->getString() << ": "; else dbgs() << "<unnamed>: "; - BSI.print(dbgs()); + BitSets.back().second.print(dbgs()); }); + } + + return BitSets; +} +void LowerTypeTestsModule::lowerTypeTestCalls( + ArrayRef<Metadata *> TypeIds, Constant *CombinedGlobalAddr, + const DenseMap<GlobalTypeMember *, uint64_t> &GlobalLayout) { + // For each type identifier in this disjoint set... + for (const auto &[TypeId, BSI] : buildBitSets(TypeIds, GlobalLayout)) { ByteArrayInfo *BAI = nullptr; TypeIdLowering TIL; diff --git a/llvm/lib/Transforms/IPO/WholeProgramDevirt.cpp b/llvm/lib/Transforms/IPO/WholeProgramDevirt.cpp index bfb25c806e53..09bffa7bf584 100644 --- a/llvm/lib/Transforms/IPO/WholeProgramDevirt.cpp +++ b/llvm/lib/Transforms/IPO/WholeProgramDevirt.cpp @@ -1495,7 +1495,7 @@ void DevirtModule::tryICallBranchFunnel( if (!JT->getEntryCount().has_value()) { // FIXME: we could pass through thinlto the necessary information. - setExplicitlyUnknownFunctionEntryCount(*JT); + setExplicitlyUnknownFunctionEntryCount(*JT, DEBUG_TYPE); } } diff --git a/llvm/lib/Transforms/InstCombine/InstCombineAddSub.cpp b/llvm/lib/Transforms/InstCombine/InstCombineAddSub.cpp index d934638c15e7..f9155cc66031 100644 --- a/llvm/lib/Transforms/InstCombine/InstCombineAddSub.cpp +++ b/llvm/lib/Transforms/InstCombine/InstCombineAddSub.cpp @@ -2115,6 +2115,7 @@ CommonPointerBase CommonPointerBase::compute(Value *LHS, Value *RHS) { } // Find common base and collect RHS GEPs. + bool First = true; while (true) { if (Ptrs.contains(RHS)) { Base.Ptr = RHS; @@ -2123,7 +2124,12 @@ CommonPointerBase CommonPointerBase::compute(Value *LHS, Value *RHS) { if (auto *GEP = dyn_cast<GEPOperator>(RHS)) { Base.RHSGEPs.push_back(GEP); - Base.RHSNW &= GEP->getNoWrapFlags(); + if (First) { + First = false; + Base.RHSNW = GEP->getNoWrapFlags(); + } else { + Base.RHSNW = Base.RHSNW.intersectForOffsetAdd(GEP->getNoWrapFlags()); + } RHS = GEP->getPointerOperand(); } else { // No common base. @@ -2132,13 +2138,19 @@ CommonPointerBase CommonPointerBase::compute(Value *LHS, Value *RHS) { } // Collect LHS GEPs. + First = true; while (true) { if (LHS == Base.Ptr) break; auto *GEP = cast<GEPOperator>(LHS); Base.LHSGEPs.push_back(GEP); - Base.LHSNW &= GEP->getNoWrapFlags(); + if (First) { + First = false; + Base.LHSNW = GEP->getNoWrapFlags(); + } else { + Base.LHSNW = Base.LHSNW.intersectForOffsetAdd(GEP->getNoWrapFlags()); + } LHS = GEP->getPointerOperand(); } diff --git a/llvm/lib/Transforms/InstCombine/InstCombineAndOrXor.cpp b/llvm/lib/Transforms/InstCombine/InstCombineAndOrXor.cpp index 8b9df62d7c65..2d7524e8018b 100644 --- a/llvm/lib/Transforms/InstCombine/InstCombineAndOrXor.cpp +++ b/llvm/lib/Transforms/InstCombine/InstCombineAndOrXor.cpp @@ -1801,10 +1801,14 @@ static Instruction *foldLogicCastConstant(BinaryOperator &Logic, CastInst *Cast, Value *X; auto &DL = IC.getDataLayout(); if (match(Cast, m_OneUse(m_ZExt(m_Value(X))))) { - if (Constant *TruncC = getLosslessUnsignedTrunc(C, SrcTy, DL)) { + PreservedCastFlags Flags; + if (Constant *TruncC = getLosslessUnsignedTrunc(C, SrcTy, DL, &Flags)) { // LogicOpc (zext X), C --> zext (LogicOpc X, C) Value *NewOp = IC.Builder.CreateBinOp(LogicOpc, X, TruncC); - return new ZExtInst(NewOp, DestTy); + auto *ZExt = new ZExtInst(NewOp, DestTy); + ZExt->setNonNeg(Flags.NNeg); + ZExt->andIRFlags(Cast); + return ZExt; } } diff --git a/llvm/lib/Transforms/InstCombine/InstCombineCasts.cpp b/llvm/lib/Transforms/InstCombine/InstCombineCasts.cpp index fdef49e310f8..ccf918f0b6db 100644 --- a/llvm/lib/Transforms/InstCombine/InstCombineCasts.cpp +++ b/llvm/lib/Transforms/InstCombine/InstCombineCasts.cpp @@ -11,11 +11,13 @@ //===----------------------------------------------------------------------===// #include "InstCombineInternal.h" +#include "llvm/ADT/APInt.h" #include "llvm/ADT/SetVector.h" #include "llvm/Analysis/ConstantFolding.h" #include "llvm/IR/DataLayout.h" #include "llvm/IR/DebugInfo.h" #include "llvm/IR/PatternMatch.h" +#include "llvm/IR/Value.h" #include "llvm/Support/KnownBits.h" #include "llvm/Transforms/InstCombine/InstCombiner.h" #include <optional> @@ -969,6 +971,25 @@ Instruction *InstCombinerImpl::visitTrunc(TruncInst &Trunc) { Changed = true; } + const APInt *C1; + Value *V1; + // OP = { lshr, ashr } + // trunc ( OP i8 C1, V1) to i1 -> icmp eq V1, log_2(C1) iff C1 is power of 2 + if (DestWidth == 1 && match(Src, m_Shr(m_Power2(C1), m_Value(V1)))) { + Value *Right = ConstantInt::get(V1->getType(), C1->countr_zero()); + Value *Icmp = Builder.CreateICmpEQ(V1, Right); + return replaceInstUsesWith(Trunc, Icmp); + } + + // OP = { lshr, ashr } + // trunc ( OP i8 C1, V1) to i1 -> icmp ult V1, log_2(C1 + 1) iff (C1 + 1) is + // power of 2 + if (DestWidth == 1 && match(Src, m_Shr(m_LowBitMask(C1), m_Value(V1)))) { + Value *Right = ConstantInt::get(V1->getType(), C1->countr_one()); + Value *Icmp = Builder.CreateICmpULT(V1, Right); + return replaceInstUsesWith(Trunc, Icmp); + } + return Changed ? &Trunc : nullptr; } diff --git a/llvm/lib/Transforms/InstCombine/InstCombineCompares.cpp b/llvm/lib/Transforms/InstCombine/InstCombineCompares.cpp index 07da12a3ab2a..99ea04816681 100644 --- a/llvm/lib/Transforms/InstCombine/InstCombineCompares.cpp +++ b/llvm/lib/Transforms/InstCombine/InstCombineCompares.cpp @@ -19,6 +19,7 @@ #include "llvm/Analysis/CmpInstAnalysis.h" #include "llvm/Analysis/ConstantFolding.h" #include "llvm/Analysis/InstructionSimplify.h" +#include "llvm/Analysis/Loads.h" #include "llvm/Analysis/Utils/Local.h" #include "llvm/Analysis/VectorUtils.h" #include "llvm/IR/ConstantRange.h" @@ -110,30 +111,27 @@ static bool isSignTest(ICmpInst::Predicate &Pred, const APInt &C) { /// If AndCst is non-null, then the loaded value is masked with that constant /// before doing the comparison. This handles cases like "A[i]&4 == 0". Instruction *InstCombinerImpl::foldCmpLoadFromIndexedGlobal( - LoadInst *LI, GetElementPtrInst *GEP, GlobalVariable *GV, CmpInst &ICI, - ConstantInt *AndCst) { - if (LI->isVolatile() || !GV->isConstant() || !GV->hasDefinitiveInitializer()) + LoadInst *LI, GetElementPtrInst *GEP, CmpInst &ICI, ConstantInt *AndCst) { + auto *GV = dyn_cast<GlobalVariable>(getUnderlyingObject(GEP)); + if (LI->isVolatile() || !GV || !GV->isConstant() || + !GV->hasDefinitiveInitializer()) return nullptr; - Constant *Init = GV->getInitializer(); - TypeSize GlobalSize = DL.getTypeAllocSize(Init->getType()); Type *EltTy = LI->getType(); TypeSize EltSize = DL.getTypeStoreSize(EltTy); if (EltSize.isScalable()) return nullptr; - unsigned IndexBW = DL.getIndexTypeSizeInBits(GEP->getType()); - SmallMapVector<Value *, APInt, 4> VarOffsets; - APInt ConstOffset(IndexBW, 0); - if (!GEP->collectOffset(DL, IndexBW, VarOffsets, ConstOffset) || - VarOffsets.size() != 1 || IndexBW > 64) + LinearExpression Expr = decomposeLinearExpression(DL, GEP); + if (!Expr.Index || Expr.BasePtr != GV || Expr.Offset.getBitWidth() > 64) return nullptr; - Value *Idx = VarOffsets.front().first; - const APInt &Stride = VarOffsets.front().second; - // If the index type is non-canonical, wait for it to be canonicalized. - if (Idx->getType()->getScalarSizeInBits() != IndexBW) - return nullptr; + Constant *Init = GV->getInitializer(); + TypeSize GlobalSize = DL.getTypeAllocSize(Init->getType()); + + Value *Idx = Expr.Index; + const APInt &Stride = Expr.Scale; + const APInt &ConstOffset = Expr.Offset; // Allow an additional context offset, but only within the stride. if (!ConstOffset.ult(Stride)) @@ -280,7 +278,7 @@ Instruction *InstCombinerImpl::foldCmpLoadFromIndexedGlobal( // comparison is false if Idx was 0x80..00. // We need to erase the highest countTrailingZeros(ElementSize) bits of Idx. auto MaskIdx = [&](Value *Idx) { - if (!GEP->isInBounds() && Stride.countr_zero() != 0) { + if (!Expr.Flags.isInBounds() && Stride.countr_zero() != 0) { Value *Mask = Constant::getAllOnesValue(Idx->getType()); Mask = Builder.CreateLShr(Mask, Stride.countr_zero()); Idx = Builder.CreateAnd(Idx, Mask); @@ -1958,10 +1956,8 @@ Instruction *InstCombinerImpl::foldICmpAndConstant(ICmpInst &Cmp, if (auto *C2 = dyn_cast<ConstantInt>(Y)) if (auto *LI = dyn_cast<LoadInst>(X)) if (auto *GEP = dyn_cast<GetElementPtrInst>(LI->getOperand(0))) - if (auto *GV = dyn_cast<GlobalVariable>(GEP->getOperand(0))) - if (Instruction *Res = - foldCmpLoadFromIndexedGlobal(LI, GEP, GV, Cmp, C2)) - return Res; + if (Instruction *Res = foldCmpLoadFromIndexedGlobal(LI, GEP, Cmp, C2)) + return Res; if (!Cmp.isEquality()) return nullptr; @@ -4314,10 +4310,9 @@ Instruction *InstCombinerImpl::foldICmpInstWithConstantNotInt(ICmpInst &I) { // Try to optimize things like "A[i] > 4" to index computations. if (GetElementPtrInst *GEP = dyn_cast<GetElementPtrInst>(LHSI->getOperand(0))) - if (GlobalVariable *GV = dyn_cast<GlobalVariable>(GEP->getOperand(0))) - if (Instruction *Res = - foldCmpLoadFromIndexedGlobal(cast<LoadInst>(LHSI), GEP, GV, I)) - return Res; + if (Instruction *Res = + foldCmpLoadFromIndexedGlobal(cast<LoadInst>(LHSI), GEP, I)) + return Res; break; } @@ -8798,10 +8793,9 @@ Instruction *InstCombinerImpl::visitFCmpInst(FCmpInst &I) { break; case Instruction::Load: if (auto *GEP = dyn_cast<GetElementPtrInst>(LHSI->getOperand(0))) - if (auto *GV = dyn_cast<GlobalVariable>(GEP->getOperand(0))) - if (Instruction *Res = foldCmpLoadFromIndexedGlobal( - cast<LoadInst>(LHSI), GEP, GV, I)) - return Res; + if (Instruction *Res = + foldCmpLoadFromIndexedGlobal(cast<LoadInst>(LHSI), GEP, I)) + return Res; break; case Instruction::FPTrunc: if (Instruction *NV = foldFCmpFpTrunc(I, *LHSI, *RHSC)) diff --git a/llvm/lib/Transforms/InstCombine/InstCombineInternal.h b/llvm/lib/Transforms/InstCombine/InstCombineInternal.h index d3d23130b6fc..7a979c16da50 100644 --- a/llvm/lib/Transforms/InstCombine/InstCombineInternal.h +++ b/llvm/lib/Transforms/InstCombine/InstCombineInternal.h @@ -693,7 +693,7 @@ public: bool foldAllocaCmp(AllocaInst *Alloca); Instruction *foldCmpLoadFromIndexedGlobal(LoadInst *LI, GetElementPtrInst *GEP, - GlobalVariable *GV, CmpInst &ICI, + CmpInst &ICI, ConstantInt *AndCst = nullptr); Instruction *foldFCmpIntToFPConst(FCmpInst &I, Instruction *LHSI, Constant *RHSC); diff --git a/llvm/lib/Transforms/InstCombine/InstructionCombining.cpp b/llvm/lib/Transforms/InstCombine/InstructionCombining.cpp index a74f292524b4..c2f045a2ab02 100644 --- a/llvm/lib/Transforms/InstCombine/InstructionCombining.cpp +++ b/llvm/lib/Transforms/InstCombine/InstructionCombining.cpp @@ -2744,125 +2744,53 @@ Instruction *InstCombinerImpl::visitGEPOfGEP(GetElementPtrInst &GEP, if (auto *I = combineConstantOffsets(GEP, *this)) return I; - // For constant GEPs, use a more general offset-based folding approach. - Type *PtrTy = Src->getType()->getScalarType(); - if (GEP.hasAllConstantIndices() && - (Src->hasOneUse() || Src->hasAllConstantIndices())) { - // Split Src into a variable part and a constant suffix. - gep_type_iterator GTI = gep_type_begin(*Src); - Type *BaseType = GTI.getIndexedType(); - bool IsFirstType = true; - unsigned NumVarIndices = 0; - for (auto Pair : enumerate(Src->indices())) { - if (!isa<ConstantInt>(Pair.value())) { - BaseType = GTI.getIndexedType(); - IsFirstType = false; - NumVarIndices = Pair.index() + 1; - } - ++GTI; - } - - // Determine the offset for the constant suffix of Src. - APInt Offset(DL.getIndexTypeSizeInBits(PtrTy), 0); - if (NumVarIndices != Src->getNumIndices()) { - // FIXME: getIndexedOffsetInType() does not handled scalable vectors. - if (BaseType->isScalableTy()) - return nullptr; - - SmallVector<Value *> ConstantIndices; - if (!IsFirstType) - ConstantIndices.push_back( - Constant::getNullValue(Type::getInt32Ty(GEP.getContext()))); - append_range(ConstantIndices, drop_begin(Src->indices(), NumVarIndices)); - Offset += DL.getIndexedOffsetInType(BaseType, ConstantIndices); - } - - // Add the offset for GEP (which is fully constant). - if (!GEP.accumulateConstantOffset(DL, Offset)) - return nullptr; - - // Convert the total offset back into indices. - SmallVector<APInt> ConstIndices = - DL.getGEPIndicesForOffset(BaseType, Offset); - if (!Offset.isZero() || (!IsFirstType && !ConstIndices[0].isZero())) - return nullptr; - - GEPNoWrapFlags NW = getMergedGEPNoWrapFlags(*Src, *cast<GEPOperator>(&GEP)); - SmallVector<Value *> Indices( - drop_end(Src->indices(), Src->getNumIndices() - NumVarIndices)); - for (const APInt &Idx : drop_begin(ConstIndices, !IsFirstType)) { - Indices.push_back(ConstantInt::get(GEP.getContext(), Idx)); - // Even if the total offset is inbounds, we may end up representing it - // by first performing a larger negative offset, and then a smaller - // positive one. The large negative offset might go out of bounds. Only - // preserve inbounds if all signs are the same. - if (Idx.isNonNegative() != ConstIndices[0].isNonNegative()) - NW = NW.withoutNoUnsignedSignedWrap(); - if (!Idx.isNonNegative()) - NW = NW.withoutNoUnsignedWrap(); - } - - return replaceInstUsesWith( - GEP, Builder.CreateGEP(Src->getSourceElementType(), Src->getOperand(0), - Indices, "", NW)); - } - if (Src->getResultElementType() != GEP.getSourceElementType()) return nullptr; - SmallVector<Value*, 8> Indices; - // Find out whether the last index in the source GEP is a sequential idx. bool EndsWithSequential = false; for (gep_type_iterator I = gep_type_begin(*Src), E = gep_type_end(*Src); I != E; ++I) EndsWithSequential = I.isSequential(); + if (!EndsWithSequential) + return nullptr; - // Can we combine the two pointer arithmetics offsets? - if (EndsWithSequential) { - // Replace: gep (gep %P, long B), long A, ... - // With: T = long A+B; gep %P, T, ... - Value *SO1 = Src->getOperand(Src->getNumOperands()-1); - Value *GO1 = GEP.getOperand(1); - - // If they aren't the same type, then the input hasn't been processed - // by the loop above yet (which canonicalizes sequential index types to - // intptr_t). Just avoid transforming this until the input has been - // normalized. - if (SO1->getType() != GO1->getType()) - return nullptr; + // Replace: gep (gep %P, long B), long A, ... + // With: T = long A+B; gep %P, T, ... + Value *SO1 = Src->getOperand(Src->getNumOperands() - 1); + Value *GO1 = GEP.getOperand(1); - Value *Sum = - simplifyAddInst(GO1, SO1, false, false, SQ.getWithInstruction(&GEP)); - // Only do the combine when we are sure the cost after the - // merge is never more than that before the merge. - if (Sum == nullptr) - return nullptr; + // If they aren't the same type, then the input hasn't been processed + // by the loop above yet (which canonicalizes sequential index types to + // intptr_t). Just avoid transforming this until the input has been + // normalized. + if (SO1->getType() != GO1->getType()) + return nullptr; - Indices.append(Src->op_begin()+1, Src->op_end()-1); - Indices.push_back(Sum); - Indices.append(GEP.op_begin()+2, GEP.op_end()); - } else if (isa<Constant>(*GEP.idx_begin()) && - cast<Constant>(*GEP.idx_begin())->isNullValue() && - Src->getNumOperands() != 1) { - // Otherwise we can do the fold if the first index of the GEP is a zero - Indices.append(Src->op_begin()+1, Src->op_end()); - Indices.append(GEP.idx_begin()+1, GEP.idx_end()); - } - - // Don't create GEPs with more than one variable index. - unsigned NumVarIndices = - count_if(Indices, [](Value *Idx) { return !isa<Constant>(Idx); }); - if (NumVarIndices > 1) + Value *Sum = + simplifyAddInst(GO1, SO1, false, false, SQ.getWithInstruction(&GEP)); + // Only do the combine when we are sure the cost after the + // merge is never more than that before the merge. + if (Sum == nullptr) return nullptr; - if (!Indices.empty()) - return replaceInstUsesWith( - GEP, Builder.CreateGEP( - Src->getSourceElementType(), Src->getOperand(0), Indices, "", - getMergedGEPNoWrapFlags(*Src, *cast<GEPOperator>(&GEP)))); + SmallVector<Value *, 8> Indices; + Indices.append(Src->op_begin() + 1, Src->op_end() - 1); + Indices.push_back(Sum); + Indices.append(GEP.op_begin() + 2, GEP.op_end()); - return nullptr; + // Don't create GEPs with more than one non-zero index. + unsigned NumNonZeroIndices = count_if(Indices, [](Value *Idx) { + auto *C = dyn_cast<Constant>(Idx); + return !C || !C->isNullValue(); + }); + if (NumNonZeroIndices > 1) + return nullptr; + + return replaceInstUsesWith( + GEP, Builder.CreateGEP( + Src->getSourceElementType(), Src->getOperand(0), Indices, "", + getMergedGEPNoWrapFlags(*Src, *cast<GEPOperator>(&GEP)))); } Value *InstCombiner::getFreelyInvertedImpl(Value *V, bool WillInvertAllUses, @@ -3334,17 +3262,18 @@ Instruction *InstCombinerImpl::visitGetElementPtrInst(GetElementPtrInst &GEP) { return replaceInstUsesWith(GEP, Res); } - bool SeenVarIndex = false; + bool SeenNonZeroIndex = false; for (auto [IdxNum, Idx] : enumerate(Indices)) { - if (isa<Constant>(Idx)) + auto *C = dyn_cast<Constant>(Idx); + if (C && C->isNullValue()) continue; - if (!SeenVarIndex) { - SeenVarIndex = true; + if (!SeenNonZeroIndex) { + SeenNonZeroIndex = true; continue; } - // GEP has multiple variable indices: Split it. + // GEP has multiple non-zero indices: Split it. ArrayRef<Value *> FrontIndices = ArrayRef(Indices).take_front(IdxNum); Value *FrontGEP = Builder.CreateGEP(GEPEltType, PtrOp, FrontIndices, diff --git a/llvm/lib/Transforms/Instrumentation/MemorySanitizer.cpp b/llvm/lib/Transforms/Instrumentation/MemorySanitizer.cpp index a50e8b3db4d2..3ea790ad1839 100644 --- a/llvm/lib/Transforms/Instrumentation/MemorySanitizer.cpp +++ b/llvm/lib/Transforms/Instrumentation/MemorySanitizer.cpp @@ -3684,6 +3684,15 @@ struct MemorySanitizerVisitor : public InstVisitor<MemorySanitizerVisitor> { case Intrinsic::x86_mmx_packssdw: return Intrinsic::x86_mmx_packssdw; + + case Intrinsic::x86_avx512_packssdw_512: + case Intrinsic::x86_avx512_packusdw_512: + return Intrinsic::x86_avx512_packssdw_512; + + case Intrinsic::x86_avx512_packsswb_512: + case Intrinsic::x86_avx512_packuswb_512: + return Intrinsic::x86_avx512_packsswb_512; + default: llvm_unreachable("unexpected intrinsic id"); } @@ -3696,6 +3705,8 @@ struct MemorySanitizerVisitor : public InstVisitor<MemorySanitizerVisitor> { // Shadow is propagated with the signed variant of the same intrinsic applied // to sext(Sa != zeroinitializer), sext(Sb != zeroinitializer). // MMXEltSizeInBits is used only for x86mmx arguments. + // + // TODO: consider using GetMinMaxUnsigned() to handle saturation precisely void handleVectorPackIntrinsic(IntrinsicInst &I, unsigned MMXEltSizeInBits = 0) { assert(I.arg_size() == 2); @@ -3861,7 +3872,7 @@ struct MemorySanitizerVisitor : public InstVisitor<MemorySanitizerVisitor> { // // Three operands: // <4 x i32> @llvm.x86.avx512.vpdpbusd.128 - // (<4 x i32> %s, <4 x i32> %a, <4 x i32> %b) + // (<4 x i32> %s, <16 x i8> %a, <16 x i8> %b) // (this is equivalent to multiply-add on %a and %b, followed by // adding/"accumulating" %s. "Accumulation" stores the result in one // of the source registers, but this accumulate vs. add distinction @@ -3903,8 +3914,9 @@ struct MemorySanitizerVisitor : public InstVisitor<MemorySanitizerVisitor> { ReturnType->getPrimitiveSizeInBits()); if (I.arg_size() == 3) { - assert(ParamType == ReturnType); - assert(ParamType == I.getArgOperand(0)->getType()); + [[maybe_unused]] auto *AccumulatorType = + cast<FixedVectorType>(I.getOperand(0)->getType()); + assert(AccumulatorType == ReturnType); } FixedVectorType *ImplicitReturnType = ReturnType; @@ -5553,6 +5565,7 @@ struct MemorySanitizerVisitor : public InstVisitor<MemorySanitizerVisitor> { handleVectorShiftIntrinsic(I, /* Variable */ true); break; + // Pack with Signed/Unsigned Saturation case Intrinsic::x86_sse2_packsswb_128: case Intrinsic::x86_sse2_packssdw_128: case Intrinsic::x86_sse2_packuswb_128: @@ -5561,6 +5574,15 @@ struct MemorySanitizerVisitor : public InstVisitor<MemorySanitizerVisitor> { case Intrinsic::x86_avx2_packssdw: case Intrinsic::x86_avx2_packuswb: case Intrinsic::x86_avx2_packusdw: + // e.g., <64 x i8> @llvm.x86.avx512.packsswb.512 + // (<32 x i16> %a, <32 x i16> %b) + // <32 x i16> @llvm.x86.avx512.packssdw.512 + // (<16 x i32> %a, <16 x i32> %b) + // Note: AVX512 masked variants are auto-upgraded by LLVM. + case Intrinsic::x86_avx512_packsswb_512: + case Intrinsic::x86_avx512_packssdw_512: + case Intrinsic::x86_avx512_packuswb_512: + case Intrinsic::x86_avx512_packusdw_512: handleVectorPackIntrinsic(I); break; @@ -5642,19 +5664,19 @@ struct MemorySanitizerVisitor : public InstVisitor<MemorySanitizerVisitor> { // // Multiply and Add Packed Signed and Unsigned Bytes // < 4 x i32> @llvm.x86.avx512.vpdpbusd.128 - // (< 4 x i32>, < 4 x i32>, < 4 x i32>) + // (< 4 x i32>, <16 x i8>, <16 x i8>) // < 8 x i32> @llvm.x86.avx512.vpdpbusd.256 - // (< 8 x i32>, < 8 x i32>, < 8 x i32>) + // (< 8 x i32>, <32 x i8>, <32 x i8>) // <16 x i32> @llvm.x86.avx512.vpdpbusd.512 - // (<16 x i32>, <16 x i32>, <16 x i32>) + // (<16 x i32>, <64 x i8>, <64 x i8>) // // Multiply and Add Unsigned and Signed Bytes With Saturation // < 4 x i32> @llvm.x86.avx512.vpdpbusds.128 - // (< 4 x i32>, < 4 x i32>, < 4 x i32>) + // (< 4 x i32>, <16 x i8>, <16 x i8>) // < 8 x i32> @llvm.x86.avx512.vpdpbusds.256 - // (< 8 x i32>, < 8 x i32>, < 8 x i32>) + // (< 8 x i32>, <32 x i8>, <32 x i8>) // <16 x i32> @llvm.x86.avx512.vpdpbusds.512 - // (<16 x i32>, <16 x i32>, <16 x i32>) + // (<16 x i32>, <64 x i8>, <64 x i8>) // // < 4 x i32> @llvm.x86.avx2.vpdpbssd.128 // (< 4 x i32>, < 4 x i32>, < 4 x i32>) @@ -5673,30 +5695,30 @@ struct MemorySanitizerVisitor : public InstVisitor<MemorySanitizerVisitor> { // // These intrinsics are auto-upgraded into non-masked forms: // <4 x i32> @llvm.x86.avx512.mask.vpdpbusd.128 - // (<4 x i32>, <4 x i32>, <4 x i32>, i8) + // (<4 x i32>, <16 x i8>, <16 x i8>, i8) // <4 x i32> @llvm.x86.avx512.maskz.vpdpbusd.128 - // (<4 x i32>, <4 x i32>, <4 x i32>, i8) + // (<4 x i32>, <16 x i8>, <16 x i8>, i8) // <8 x i32> @llvm.x86.avx512.mask.vpdpbusd.256 - // (<8 x i32>, <8 x i32>, <8 x i32>, i8) + // (<8 x i32>, <32 x i8>, <32 x i8>, i8) // <8 x i32> @llvm.x86.avx512.maskz.vpdpbusd.256 - // (<8 x i32>, <8 x i32>, <8 x i32>, i8) + // (<8 x i32>, <32 x i8>, <32 x i8>, i8) // <16 x i32> @llvm.x86.avx512.mask.vpdpbusd.512 - // (<16 x i32>, <16 x i32>, <16 x i32>, i16) + // (<16 x i32>, <64 x i8>, <64 x i8>, i16) // <16 x i32> @llvm.x86.avx512.maskz.vpdpbusd.512 - // (<16 x i32>, <16 x i32>, <16 x i32>, i16) + // (<16 x i32>, <64 x i8>, <64 x i8>, i16) // // <4 x i32> @llvm.x86.avx512.mask.vpdpbusds.128 - // (<4 x i32>, <4 x i32>, <4 x i32>, i8) + // (<4 x i32>, <16 x i8>, <16 x i8>, i8) // <4 x i32> @llvm.x86.avx512.maskz.vpdpbusds.128 - // (<4 x i32>, <4 x i32>, <4 x i32>, i8) + // (<4 x i32>, <16 x i8>, <16 x i8>, i8) // <8 x i32> @llvm.x86.avx512.mask.vpdpbusds.256 - // (<8 x i32>, <8 x i32>, <8 x i32>, i8) + // (<8 x i32>, <32 x i8>, <32 x i8>, i8) // <8 x i32> @llvm.x86.avx512.maskz.vpdpbusds.256 - // (<8 x i32>, <8 x i32>, <8 x i32>, i8) + // (<8 x i32>, <32 x i8>, <32 x i8>, i8) // <16 x i32> @llvm.x86.avx512.mask.vpdpbusds.512 - // (<16 x i32>, <16 x i32>, <16 x i32>, i16) + // (<16 x i32>, <64 x i8>, <64 x i8>, i16) // <16 x i32> @llvm.x86.avx512.maskz.vpdpbusds.512 - // (<16 x i32>, <16 x i32>, <16 x i32>, i16) + // (<16 x i32>, <64 x i8>, <64 x i8>, i16) case Intrinsic::x86_avx512_vpdpbusd_128: case Intrinsic::x86_avx512_vpdpbusd_256: case Intrinsic::x86_avx512_vpdpbusd_512: diff --git a/llvm/lib/Transforms/Scalar/DFAJumpThreading.cpp b/llvm/lib/Transforms/Scalar/DFAJumpThreading.cpp index 22cb38565264..944b253e0f5e 100644 --- a/llvm/lib/Transforms/Scalar/DFAJumpThreading.cpp +++ b/llvm/lib/Transforms/Scalar/DFAJumpThreading.cpp @@ -581,15 +581,17 @@ struct AllSwitchPaths { VisitedBlocks VB; // Get paths from the determinator BBs to SwitchPhiDefBB std::vector<ThreadingPath> PathsToPhiDef = - getPathsFromStateDefMap(StateDef, SwitchPhi, VB); - if (SwitchPhiDefBB == SwitchBlock) { + getPathsFromStateDefMap(StateDef, SwitchPhi, VB, MaxNumPaths); + if (SwitchPhiDefBB == SwitchBlock || PathsToPhiDef.empty()) { TPaths = std::move(PathsToPhiDef); return; } + assert(MaxNumPaths >= PathsToPhiDef.size() && !PathsToPhiDef.empty()); + auto PathsLimit = MaxNumPaths / PathsToPhiDef.size(); // Find and append paths from SwitchPhiDefBB to SwitchBlock. PathsType PathsToSwitchBB = - paths(SwitchPhiDefBB, SwitchBlock, VB, /* PathDepth = */ 1); + paths(SwitchPhiDefBB, SwitchBlock, VB, /* PathDepth = */ 1, PathsLimit); if (PathsToSwitchBB.empty()) return; @@ -610,13 +612,16 @@ private: typedef DenseMap<const BasicBlock *, const PHINode *> StateDefMap; std::vector<ThreadingPath> getPathsFromStateDefMap(StateDefMap &StateDef, PHINode *Phi, - VisitedBlocks &VB) { + VisitedBlocks &VB, + unsigned PathsLimit) { std::vector<ThreadingPath> Res; auto *PhiBB = Phi->getParent(); VB.insert(PhiBB); VisitedBlocks UniqueBlocks; for (auto *IncomingBB : Phi->blocks()) { + if (Res.size() >= PathsLimit) + break; if (!UniqueBlocks.insert(IncomingBB).second) continue; if (!SwitchOuterLoop->contains(IncomingBB)) @@ -652,8 +657,9 @@ private: // Direct predecessor, just add to the path. if (IncomingPhiDefBB == IncomingBB) { - std::vector<ThreadingPath> PredPaths = - getPathsFromStateDefMap(StateDef, IncomingPhi, VB); + assert(PathsLimit > Res.size()); + std::vector<ThreadingPath> PredPaths = getPathsFromStateDefMap( + StateDef, IncomingPhi, VB, PathsLimit - Res.size()); for (ThreadingPath &Path : PredPaths) { Path.push_back(PhiBB); Res.push_back(std::move(Path)); @@ -666,13 +672,17 @@ private: continue; PathsType IntermediatePaths; - IntermediatePaths = - paths(IncomingPhiDefBB, IncomingBB, VB, /* PathDepth = */ 1); + assert(PathsLimit > Res.size()); + auto InterPathLimit = PathsLimit - Res.size(); + IntermediatePaths = paths(IncomingPhiDefBB, IncomingBB, VB, + /* PathDepth = */ 1, InterPathLimit); if (IntermediatePaths.empty()) continue; + assert(InterPathLimit >= IntermediatePaths.size()); + auto PredPathLimit = InterPathLimit / IntermediatePaths.size(); std::vector<ThreadingPath> PredPaths = - getPathsFromStateDefMap(StateDef, IncomingPhi, VB); + getPathsFromStateDefMap(StateDef, IncomingPhi, VB, PredPathLimit); for (const ThreadingPath &Path : PredPaths) { for (const PathType &IPath : IntermediatePaths) { ThreadingPath NewPath(Path); @@ -687,7 +697,7 @@ private: } PathsType paths(BasicBlock *BB, BasicBlock *ToBB, VisitedBlocks &Visited, - unsigned PathDepth) { + unsigned PathDepth, unsigned PathsLimit) { PathsType Res; // Stop exploring paths after visiting MaxPathLength blocks @@ -714,6 +724,8 @@ private: // is used to prevent a duplicate path from being generated SmallPtrSet<BasicBlock *, 4> Successors; for (BasicBlock *Succ : successors(BB)) { + if (Res.size() >= PathsLimit) + break; if (!Successors.insert(Succ).second) continue; @@ -735,14 +747,12 @@ private: // coverage and compile time. if (LI->getLoopFor(Succ) != CurrLoop) continue; - - PathsType SuccPaths = paths(Succ, ToBB, Visited, PathDepth + 1); + assert(PathsLimit > Res.size()); + PathsType SuccPaths = + paths(Succ, ToBB, Visited, PathDepth + 1, PathsLimit - Res.size()); for (PathType &Path : SuccPaths) { Path.push_front(BB); Res.push_back(Path); - if (Res.size() >= MaxNumPaths) { - return Res; - } } } // This block could now be visited again from a different predecessor. Note diff --git a/llvm/lib/Transforms/Scalar/JumpTableToSwitch.cpp b/llvm/lib/Transforms/Scalar/JumpTableToSwitch.cpp index b0478ee1fd94..2025fbbf0597 100644 --- a/llvm/lib/Transforms/Scalar/JumpTableToSwitch.cpp +++ b/llvm/lib/Transforms/Scalar/JumpTableToSwitch.cpp @@ -186,7 +186,7 @@ expandToSwitch(CallBase *CB, const JumpTableTy &JT, DomTreeUpdater &DTU, setBranchWeights(*Switch, downscaleWeights(BranchWeights), /*IsExpected=*/false); } else - setExplicitlyUnknownBranchWeights(*Switch); + setExplicitlyUnknownBranchWeights(*Switch, DEBUG_TYPE); if (PHI) CB->replaceAllUsesWith(PHI); CB->eraseFromParent(); diff --git a/llvm/lib/Transforms/Scalar/JumpThreading.cpp b/llvm/lib/Transforms/Scalar/JumpThreading.cpp index c2a737d8f9a4..c7d71eb5633e 100644 --- a/llvm/lib/Transforms/Scalar/JumpThreading.cpp +++ b/llvm/lib/Transforms/Scalar/JumpThreading.cpp @@ -1437,9 +1437,18 @@ bool JumpThreadingPass::simplifyPartiallyRedundantLoad(LoadInst *LoadI) { // AvailablePreds vector as we go so that all of the PHI entries for this // predecessor use the same bitcast. Value *&PredV = I->second; - if (PredV->getType() != LoadI->getType()) + if (PredV->getType() != LoadI->getType()) { PredV = CastInst::CreateBitOrPointerCast( PredV, LoadI->getType(), "", P->getTerminator()->getIterator()); + // The new cast is producing the value used to replace the load + // instruction, so uses the load's debug location. If P does not always + // branch to the load BB however then the debug location must be dropped, + // as it is hoisted past a conditional branch. + DebugLoc DL = P->getTerminator()->getNumSuccessors() == 1 + ? LoadI->getDebugLoc() + : DebugLoc::getDropped(); + cast<CastInst>(PredV)->setDebugLoc(DL); + } PN->addIncoming(PredV, I->first); } diff --git a/llvm/lib/Transforms/Scalar/LoopUnrollPass.cpp b/llvm/lib/Transforms/Scalar/LoopUnrollPass.cpp index f7d2258e1c28..2bda9d83236e 100644 --- a/llvm/lib/Transforms/Scalar/LoopUnrollPass.cpp +++ b/llvm/lib/Transforms/Scalar/LoopUnrollPass.cpp @@ -220,6 +220,7 @@ TargetTransformInfo::UnrollingPreferences llvm::gatherUnrollingPreferences( UP.MaxIterationsCountToAnalyze = UnrollMaxIterationsCountToAnalyze; UP.SCEVExpansionBudget = SCEVCheapExpansionBudget; UP.RuntimeUnrollMultiExit = false; + UP.AddAdditionalAccumulators = false; // Override with any target specific settings TTI.getUnrollingPreferences(L, SE, UP, &ORE); @@ -1354,6 +1355,7 @@ tryToUnrollLoop(Loop *L, DominatorTree &DT, LoopInfo *LI, ScalarEvolution &SE, ULO.Heart = getLoopConvergenceHeart(L); ULO.SCEVExpansionBudget = UP.SCEVExpansionBudget; ULO.RuntimeUnrollMultiExit = UP.RuntimeUnrollMultiExit; + ULO.AddAdditionalAccumulators = UP.AddAdditionalAccumulators; LoopUnrollResult UnrollResult = UnrollLoop( L, ULO, LI, &SE, &DT, &AC, &TTI, &ORE, PreserveLCSSA, &RemainderLoop, AA); if (UnrollResult == LoopUnrollResult::Unmodified) diff --git a/llvm/lib/Transforms/Utils/BasicBlockUtils.cpp b/llvm/lib/Transforms/Utils/BasicBlockUtils.cpp index cad0b4c12b54..d2391e166f94 100644 --- a/llvm/lib/Transforms/Utils/BasicBlockUtils.cpp +++ b/llvm/lib/Transforms/Utils/BasicBlockUtils.cpp @@ -58,6 +58,19 @@ static cl::opt<unsigned> MaxDeoptOrUnreachableSuccessorCheckDepth( "is followed by a block that either has a terminating " "deoptimizing call or is terminated with an unreachable")); +static void replaceFuncletPadsRetWithUnreachable(Instruction &I) { + assert(isa<FuncletPadInst>(I) && "Instruction must be a funclet pad!"); + for (User *User : make_early_inc_range(I.users())) { + Instruction *ReturnInstr = dyn_cast<Instruction>(User); + if (isa<CatchReturnInst>(ReturnInstr) || + isa<CleanupReturnInst>(ReturnInstr)) { + BasicBlock *ReturnInstrBB = ReturnInstr->getParent(); + ReturnInstr->eraseFromParent(); + new UnreachableInst(ReturnInstrBB->getContext(), ReturnInstrBB); + } + } +} + void llvm::detachDeadBlocks( ArrayRef<BasicBlock *> BBs, SmallVectorImpl<DominatorTree::UpdateType> *Updates, @@ -75,7 +88,36 @@ void llvm::detachDeadBlocks( // Zap all the instructions in the block. while (!BB->empty()) { Instruction &I = BB->back(); - // If this instruction is used, replace uses with an arbitrary value. + // Exception handling funclets need to be explicitly addressed. + // These funclets must begin with cleanuppad or catchpad and end with + // cleanupred or catchret. The return instructions can be in different + // basic blocks than the pad instruction. If we would only delete the + // first block, the we would have possible cleanupret and catchret + // instructions with poison arguments, which wouldn't be valid. + if (isa<FuncletPadInst>(I)) + replaceFuncletPadsRetWithUnreachable(I); + + // Catchswitch instructions have handlers, that must be catchpads and + // an unwind label, that is either a catchpad or catchswitch. + if (CatchSwitchInst *CSI = dyn_cast<CatchSwitchInst>(&I)) { + // Iterating over the handlers and the unwind basic block and processing + // catchpads. If the unwind label is a catchswitch, we just replace the + // label with poison later on. + for (unsigned I = 0; I < CSI->getNumSuccessors(); I++) { + BasicBlock *SucBlock = CSI->getSuccessor(I); + Instruction &SucFstInst = *(SucBlock->getFirstNonPHIIt()); + if (isa<FuncletPadInst>(SucFstInst)) { + replaceFuncletPadsRetWithUnreachable(SucFstInst); + // There may be catchswitch instructions using the catchpad. + // Just replace those with poison. + if (!SucFstInst.use_empty()) + SucFstInst.replaceAllUsesWith( + PoisonValue::get(SucFstInst.getType())); + SucFstInst.eraseFromParent(); + } + } + } + // Because control flow can't get here, we don't care what we replace the // value with. Note that since this block is unreachable, and all values // contained within it must dominate their uses, that all uses will diff --git a/llvm/lib/Transforms/Utils/LoopUnroll.cpp b/llvm/lib/Transforms/Utils/LoopUnroll.cpp index 8a6c7789d137..b18aceaa67d7 100644 --- a/llvm/lib/Transforms/Utils/LoopUnroll.cpp +++ b/llvm/lib/Transforms/Utils/LoopUnroll.cpp @@ -671,8 +671,10 @@ llvm::UnrollLoop(Loop *L, UnrollLoopOptions ULO, LoopInfo *LI, // to not exit. DenseMap<PHINode *, RecurrenceDescriptor> Reductions; bool CanAddAdditionalAccumulators = - UnrollAddParallelReductions && !CompletelyUnroll && - L->getNumBlocks() == 1 && + (UnrollAddParallelReductions.getNumOccurrences() > 0 + ? UnrollAddParallelReductions + : ULO.AddAdditionalAccumulators) && + !CompletelyUnroll && L->getNumBlocks() == 1 && (ULO.Runtime || (ExitInfos.contains(Header) && ((ExitInfos[Header].TripCount != 0 && ExitInfos[Header].BreakoutTrip == 0)))); diff --git a/llvm/lib/Transforms/Utils/LoopUtils.cpp b/llvm/lib/Transforms/Utils/LoopUtils.cpp index 843364eb34f8..7b1a7ce6995f 100644 --- a/llvm/lib/Transforms/Utils/LoopUtils.cpp +++ b/llvm/lib/Transforms/Utils/LoopUtils.cpp @@ -804,26 +804,51 @@ static BranchInst *getExpectedExitLoopLatchBranch(Loop *L) { return LatchBR; } -/// Return the estimated trip count for any exiting branch which dominates -/// the loop latch. -static std::optional<unsigned> getEstimatedTripCount(BranchInst *ExitingBranch, - Loop *L, - uint64_t &OrigExitWeight) { +struct DbgLoop { + const Loop *L; + explicit DbgLoop(const Loop *L) : L(L) {} +}; + +#ifndef NDEBUG +static inline raw_ostream &operator<<(raw_ostream &OS, DbgLoop D) { + OS << "function "; + D.L->getHeader()->getParent()->printAsOperand(OS, /*PrintType=*/false); + return OS << " " << *D.L; +} +#endif // NDEBUG + +static std::optional<unsigned> estimateLoopTripCount(Loop *L) { + // Currently we take the estimate exit count only from the loop latch, + // ignoring other exiting blocks. This can overestimate the trip count + // if we exit through another exit, but can never underestimate it. + // TODO: incorporate information from other exits + BranchInst *ExitingBranch = getExpectedExitLoopLatchBranch(L); + if (!ExitingBranch) { + LLVM_DEBUG(dbgs() << "estimateLoopTripCount: Failed to find exiting " + << "latch branch of required form in " << DbgLoop(L) + << "\n"); + return std::nullopt; + } + // To estimate the number of times the loop body was executed, we want to // know the number of times the backedge was taken, vs. the number of times // we exited the loop. uint64_t LoopWeight, ExitWeight; - if (!extractBranchWeights(*ExitingBranch, LoopWeight, ExitWeight)) + if (!extractBranchWeights(*ExitingBranch, LoopWeight, ExitWeight)) { + LLVM_DEBUG(dbgs() << "estimateLoopTripCount: Failed to extract branch " + << "weights for " << DbgLoop(L) << "\n"); return std::nullopt; + } if (L->contains(ExitingBranch->getSuccessor(1))) std::swap(LoopWeight, ExitWeight); - if (!ExitWeight) + if (!ExitWeight) { // Don't have a way to return predicated infinite + LLVM_DEBUG(dbgs() << "estimateLoopTripCount: Failed because of zero exit " + << "probability for " << DbgLoop(L) << "\n"); return std::nullopt; - - OrigExitWeight = ExitWeight; + } // Estimated exit count is a ratio of the loop weight by the weight of the // edge exiting the loop, rounded to nearest. @@ -834,43 +859,102 @@ static std::optional<unsigned> getEstimatedTripCount(BranchInst *ExitingBranch, return std::numeric_limits<unsigned>::max(); // Estimated trip count is one plus estimated exit count. - return ExitCount + 1; + uint64_t TC = ExitCount + 1; + LLVM_DEBUG(dbgs() << "estimateLoopTripCount: Estimated trip count of " << TC + << " for " << DbgLoop(L) << "\n"); + return TC; } std::optional<unsigned> llvm::getLoopEstimatedTripCount(Loop *L, unsigned *EstimatedLoopInvocationWeight) { - // Currently we take the estimate exit count only from the loop latch, - // ignoring other exiting blocks. This can overestimate the trip count - // if we exit through another exit, but can never underestimate it. - // TODO: incorporate information from other exits - if (BranchInst *LatchBranch = getExpectedExitLoopLatchBranch(L)) { - uint64_t ExitWeight; - if (std::optional<uint64_t> EstTripCount = - getEstimatedTripCount(LatchBranch, L, ExitWeight)) { - if (EstimatedLoopInvocationWeight) - *EstimatedLoopInvocationWeight = ExitWeight; - return *EstTripCount; - } + // If EstimatedLoopInvocationWeight, we do not support this loop if + // getExpectedExitLoopLatchBranch returns nullptr. + // + // FIXME: Also, this is a stop-gap solution for nested loops. It avoids + // mistaking LLVMLoopEstimatedTripCount metadata to be for an outer loop when + // it was created for an inner loop. The problem is that loop metadata is + // attached to the branch instruction in the loop latch block, but that can be + // shared by the loops. A solution is to attach loop metadata to loop headers + // instead, but that would be a large change to LLVM. + // + // Until that happens, we work around the problem as follows. + // getExpectedExitLoopLatchBranch (which also guards + // setLoopEstimatedTripCount) returns nullptr for a loop unless the loop has + // one latch and that latch has exactly two successors one of which is an exit + // from the loop. If the latch is shared by nested loops, then that condition + // might hold for the inner loop but cannot hold for the outer loop: + // - Because the latch is shared, it must have at least two successors: the + // inner loop header and the outer loop header, which is also an exit for + // the inner loop. That satisifies the condition for the inner loop. + // - To satsify the condition for the outer loop, the latch must have a third + // successor that is an exit for the outer loop. But that violates the + // condition for both loops. + BranchInst *ExitingBranch = getExpectedExitLoopLatchBranch(L); + if (!ExitingBranch) + return std::nullopt; + + // If requested, either compute *EstimatedLoopInvocationWeight or return + // nullopt if cannot. + // + // TODO: Eventually, once all passes have migrated away from setting branch + // weights to indicate estimated trip counts, this function will drop the + // EstimatedLoopInvocationWeight parameter. + if (EstimatedLoopInvocationWeight) { + uint64_t LoopWeight = 0, ExitWeight = 0; // Inits expected to be unused. + if (!extractBranchWeights(*ExitingBranch, LoopWeight, ExitWeight)) + return std::nullopt; + if (L->contains(ExitingBranch->getSuccessor(1))) + std::swap(LoopWeight, ExitWeight); + if (!ExitWeight) + return std::nullopt; + *EstimatedLoopInvocationWeight = ExitWeight; } - return std::nullopt; + + // Return the estimated trip count from metadata unless the metadata is + // missing or has no value. + if (auto TC = getOptionalIntLoopAttribute(L, LLVMLoopEstimatedTripCount)) { + LLVM_DEBUG(dbgs() << "getLoopEstimatedTripCount: " + << LLVMLoopEstimatedTripCount << " metadata has trip " + << "count of " << *TC << " for " << DbgLoop(L) << "\n"); + return TC; + } + + // Estimate the trip count from latch branch weights. + return estimateLoopTripCount(L); } -bool llvm::setLoopEstimatedTripCount(Loop *L, unsigned EstimatedTripCount, - unsigned EstimatedloopInvocationWeight) { - // At the moment, we currently support changing the estimate trip count of - // the latch branch only. We could extend this API to manipulate estimated - // trip counts for any exit. +bool llvm::setLoopEstimatedTripCount( + Loop *L, unsigned EstimatedTripCount, + std::optional<unsigned> EstimatedloopInvocationWeight) { + // If EstimatedLoopInvocationWeight, we do not support this loop if + // getExpectedExitLoopLatchBranch returns nullptr. + // + // FIXME: See comments in getLoopEstimatedTripCount for why this is required + // here regardless of EstimatedLoopInvocationWeight. BranchInst *LatchBranch = getExpectedExitLoopLatchBranch(L); if (!LatchBranch) return false; + // Set the metadata. + addStringMetadataToLoop(L, LLVMLoopEstimatedTripCount, EstimatedTripCount); + + // At the moment, we currently support changing the estimated trip count in + // the latch branch's branch weights only. We could extend this API to + // manipulate estimated trip counts for any exit. + // + // TODO: Eventually, once all passes have migrated away from setting branch + // weights to indicate estimated trip counts, we will not set branch weights + // here at all. + if (!EstimatedloopInvocationWeight) + return true; + // Calculate taken and exit weights. unsigned LatchExitWeight = 0; unsigned BackedgeTakenWeight = 0; - if (EstimatedTripCount > 0) { - LatchExitWeight = EstimatedloopInvocationWeight; + if (EstimatedTripCount != 0) { + LatchExitWeight = *EstimatedloopInvocationWeight; BackedgeTakenWeight = (EstimatedTripCount - 1) * LatchExitWeight; } @@ -2032,6 +2116,7 @@ Value *llvm::addRuntimeChecks( MemoryRuntimeCheck = IsConflict; } + Exp.eraseDeadInstructions(MemoryRuntimeCheck); return MemoryRuntimeCheck; } @@ -2077,6 +2162,7 @@ Value *llvm::addDiffRuntimeChecks( MemoryRuntimeCheck = IsConflict; } + Expander.eraseDeadInstructions(MemoryRuntimeCheck); return MemoryRuntimeCheck; } diff --git a/llvm/lib/Transforms/Utils/LoopVersioning.cpp b/llvm/lib/Transforms/Utils/LoopVersioning.cpp index 1711163fb9f5..ec2e6c1ab796 100644 --- a/llvm/lib/Transforms/Utils/LoopVersioning.cpp +++ b/llvm/lib/Transforms/Utils/LoopVersioning.cpp @@ -81,6 +81,8 @@ void LoopVersioning::versionLoop( } else RuntimeCheck = MemRuntimeCheck ? MemRuntimeCheck : SCEVRuntimeCheck; + Exp.eraseDeadInstructions(SCEVRuntimeCheck); + assert(RuntimeCheck && "called even though we don't need " "any runtime checks"); diff --git a/llvm/lib/Transforms/Utils/PromoteMemoryToRegister.cpp b/llvm/lib/Transforms/Utils/PromoteMemoryToRegister.cpp index 10c162bc6463..d93a4d87f30f 100644 --- a/llvm/lib/Transforms/Utils/PromoteMemoryToRegister.cpp +++ b/llvm/lib/Transforms/Utils/PromoteMemoryToRegister.cpp @@ -849,9 +849,12 @@ void PromoteMem2Reg::run() { for (unsigned i = 0, e = Allocas.size(); i != e; ++i) IncomingVals.init(i, UndefValue::get(Allocas[i]->getAllocatedType())); - // When handling debug info, treat all incoming values as if they have unknown - // locations until proven otherwise. + // When handling debug info, treat all incoming values as if they have + // compiler-generated (empty) locations, representing the uninitialized + // alloca, until proven otherwise. IncomingLocs.resize(Allocas.size()); + for (unsigned i = 0, e = Allocas.size(); i != e; ++i) + IncomingLocs.init(i, DebugLoc::getCompilerGenerated()); // The renamer uses the Visited set to avoid infinite loops. Visited.resize(F.getMaxBlockNumber(), false); diff --git a/llvm/lib/Transforms/Utils/ScalarEvolutionExpander.cpp b/llvm/lib/Transforms/Utils/ScalarEvolutionExpander.cpp index 7c12dac73f3d..45cee1e7da62 100644 --- a/llvm/lib/Transforms/Utils/ScalarEvolutionExpander.cpp +++ b/llvm/lib/Transforms/Utils/ScalarEvolutionExpander.cpp @@ -26,6 +26,7 @@ #include "llvm/IR/PatternMatch.h" #include "llvm/Support/CommandLine.h" #include "llvm/Support/raw_ostream.h" +#include "llvm/Transforms/Utils/Local.h" #include "llvm/Transforms/Utils/LoopUtils.h" #if LLVM_ENABLE_ABI_BREAKING_CHECKS @@ -175,6 +176,26 @@ SCEVExpander::findInsertPointAfter(Instruction *I, return IP; } +void SCEVExpander::eraseDeadInstructions(Value *Root) { + SmallVector<Value *> WorkList; + SmallPtrSet<Value *, 8> DeletedValues; + append_range(WorkList, getAllInsertedInstructions()); + while (!WorkList.empty()) { + Value *V = WorkList.pop_back_val(); + if (DeletedValues.contains(V)) + continue; + auto *I = dyn_cast<Instruction>(V); + if (!I || I == Root || !isInsertedInstruction(I) || + !isInstructionTriviallyDead(I)) + continue; + append_range(WorkList, I->operands()); + InsertedValues.erase(I); + InsertedPostIncValues.erase(I); + DeletedValues.insert(I); + I->eraseFromParent(); + } +} + BasicBlock::iterator SCEVExpander::GetOptimalInsertionPointForCastOf(Value *V) const { // Cast the argument at the beginning of the entry block, after @@ -2201,20 +2222,11 @@ Value *SCEVExpander::generateOverflowCheck(const SCEVAddRecExpr *AR, // Get the backedge taken count and truncate or extended to the AR type. Value *TruncTripCount = Builder.CreateZExtOrTrunc(TripCountVal, Ty); - Value *MulV, *OfMul; - if (Step->isOne()) { - // Special-case Step of one. Potentially-costly `umul_with_overflow` isn't - // needed, there is never an overflow, so to avoid artificially inflating - // the cost of the check, directly emit the optimized IR. - MulV = TruncTripCount; - OfMul = ConstantInt::getFalse(MulV->getContext()); - } else { - CallInst *Mul = Builder.CreateIntrinsic(Intrinsic::umul_with_overflow, Ty, - {AbsStep, TruncTripCount}, - /*FMFSource=*/nullptr, "mul"); - MulV = Builder.CreateExtractValue(Mul, 0, "mul.result"); - OfMul = Builder.CreateExtractValue(Mul, 1, "mul.overflow"); - } + CallInst *Mul = Builder.CreateIntrinsic(Intrinsic::umul_with_overflow, Ty, + {AbsStep, TruncTripCount}, + /*FMFSource=*/nullptr, "mul"); + Value *MulV = Builder.CreateExtractValue(Mul, 0, "mul.result"); + Value *OfMul = Builder.CreateExtractValue(Mul, 1, "mul.overflow"); Value *Add = nullptr, *Sub = nullptr; bool NeedPosCheck = !SE.isKnownNegative(Step); diff --git a/llvm/lib/Transforms/Utils/SimplifyCFG.cpp b/llvm/lib/Transforms/Utils/SimplifyCFG.cpp index 39edc4d87126..850e57e6b0b1 100644 --- a/llvm/lib/Transforms/Utils/SimplifyCFG.cpp +++ b/llvm/lib/Transforms/Utils/SimplifyCFG.cpp @@ -203,6 +203,8 @@ static cl::opt<unsigned> MaxJumpThreadingLiveBlocks( cl::desc("Limit number of blocks a define in a threaded block is allowed " "to be live in")); +extern cl::opt<bool> ProfcheckDisableMetadataFixes; + STATISTIC(NumBitMaps, "Number of switch instructions turned into bitmaps"); STATISTIC(NumLinearMaps, "Number of switch instructions turned into linear mapping"); @@ -4438,6 +4440,20 @@ static bool mergeConditionalStoreToAddress( auto *T = SplitBlockAndInsertIfThen(CombinedPred, InsertPt, /*Unreachable=*/false, /*BranchWeights=*/nullptr, DTU); + if (hasBranchWeightMD(*PBranch) && hasBranchWeightMD(*QBranch) && + !ProfcheckDisableMetadataFixes) { + SmallVector<uint32_t, 2> PWeights, QWeights; + extractBranchWeights(*PBranch, PWeights); + extractBranchWeights(*QBranch, QWeights); + if (InvertPCond) + std::swap(PWeights[0], PWeights[1]); + if (InvertQCond) + std::swap(QWeights[0], QWeights[1]); + auto CombinedWeights = getDisjunctionWeights(PWeights, QWeights); + setBranchWeights(PostBB->getTerminator(), CombinedWeights[0], + CombinedWeights[1], + /*IsExpected=*/false); + } QB.SetInsertPoint(T); StoreInst *SI = cast<StoreInst>(QB.CreateStore(QPHI, Address)); @@ -6458,6 +6474,9 @@ public: /// Return the default value of the switch. Constant *getDefaultValue(); + /// Return true if the replacement is a lookup table. + bool isLookupTable(); + private: // Depending on the switch, there are different alternatives. enum { @@ -6605,7 +6624,6 @@ SwitchReplacement::SwitchReplacement( (void)M.smul_ov(APInt(M.getBitWidth(), TableSize - 1), MayWrap); LinearMapValWrapped = NonMonotonic || MayWrap; Kind = LinearMapKind; - ++NumLinearMaps; return; } } @@ -6625,7 +6643,6 @@ SwitchReplacement::SwitchReplacement( BitMap = ConstantInt::get(M.getContext(), TableInt); BitMapElementTy = IT; Kind = BitMapKind; - ++NumBitMaps; return; } @@ -6642,6 +6659,7 @@ Value *SwitchReplacement::replaceSwitch(Value *Index, IRBuilder<> &Builder, case SingleValueKind: return SingleValue; case LinearMapKind: { + ++NumLinearMaps; // Derive the result value from the input value. Value *Result = Builder.CreateIntCast(Index, LinearMultiplier->getType(), false, "switch.idx.cast"); @@ -6657,6 +6675,7 @@ Value *SwitchReplacement::replaceSwitch(Value *Index, IRBuilder<> &Builder, return Result; } case BitMapKind: { + ++NumBitMaps; // Type of the bitmap (e.g. i59). IntegerType *MapTy = BitMap->getIntegerType(); @@ -6679,6 +6698,7 @@ Value *SwitchReplacement::replaceSwitch(Value *Index, IRBuilder<> &Builder, return Builder.CreateTrunc(DownShifted, BitMapElementTy, "switch.masked"); } case LookupTableKind: { + ++NumLookupTables; auto *Table = new GlobalVariable(*Func->getParent(), Initializer->getType(), /*isConstant=*/true, GlobalVariable::PrivateLinkage, @@ -6745,6 +6765,8 @@ static bool isTypeLegalForLookupTable(Type *Ty, const TargetTransformInfo &TTI, Constant *SwitchReplacement::getDefaultValue() { return DefaultValue; } +bool SwitchReplacement::isLookupTable() { return Kind == LookupTableKind; } + static bool isSwitchDense(uint64_t NumCases, uint64_t CaseRange) { // 40% is the default density for building a jump table in optsize/minsize // mode. See also TargetLoweringBase::isSuitableForJumpTable(), which this @@ -6915,11 +6937,6 @@ static bool simplifySwitchLookup(SwitchInst *SI, IRBuilder<> &Builder, BasicBlock *BB = SI->getParent(); Function *Fn = BB->getParent(); - // Only build lookup table when we have a target that supports it or the - // attribute is not set. - if (!TTI.shouldBuildLookupTables() || - (Fn->getFnAttribute("no-jump-tables").getValueAsBool())) - return false; // FIXME: If the switch is too sparse for a lookup table, perhaps we could // split off a dense part and build a lookup table for that. @@ -7078,6 +7095,20 @@ static bool simplifySwitchLookup(SwitchInst *SI, IRBuilder<> &Builder, PhiToReplacementMap.insert({PHI, Replacement}); } + bool AnyLookupTables = any_of( + PhiToReplacementMap, [](auto &KV) { return KV.second.isLookupTable(); }); + + // A few conditions prevent the generation of lookup tables: + // 1. The target does not support lookup tables. + // 2. The "no-jump-tables" function attribute is set. + // However, these objections do not apply to other switch replacements, like + // the bitmap, so we only stop here if any of these conditions are met and we + // want to create a LUT. Otherwise, continue with the switch replacement. + if (AnyLookupTables && + (!TTI.shouldBuildLookupTables() || + Fn->getFnAttribute("no-jump-tables").getValueAsBool())) + return false; + Builder.SetInsertPoint(SI); // TableIndex is the switch condition - TableIndexOffset if we don't // use the condition directly @@ -7219,7 +7250,6 @@ static bool simplifySwitchLookup(SwitchInst *SI, IRBuilder<> &Builder, if (DTU) DTU->applyUpdates(Updates); - ++NumLookupTables; if (NeedMask) ++NumLookupTablesHoles; return true; diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorizationLegality.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorizationLegality.cpp index 528ae3822d07..85a6fcaf3ecd 100644 --- a/llvm/lib/Transforms/Vectorize/LoopVectorizationLegality.cpp +++ b/llvm/lib/Transforms/Vectorize/LoopVectorizationLegality.cpp @@ -15,8 +15,10 @@ // #include "llvm/Transforms/Vectorize/LoopVectorizationLegality.h" +#include "llvm/Analysis/AliasAnalysis.h" #include "llvm/Analysis/Loads.h" #include "llvm/Analysis/LoopInfo.h" +#include "llvm/Analysis/MustExecute.h" #include "llvm/Analysis/OptimizationRemarkEmitter.h" #include "llvm/Analysis/ScalarEvolutionExpressions.h" #include "llvm/Analysis/TargetLibraryInfo.h" @@ -1223,8 +1225,18 @@ bool LoopVectorizationLegality::canVectorizeMemory() { }); } - if (!LAI->canVectorizeMemory()) + if (!LAI->canVectorizeMemory()) { + if (hasUncountableExitWithSideEffects()) { + reportVectorizationFailure( + "Cannot vectorize unsafe dependencies in uncountable exit loop with " + "side effects", + "CantVectorizeUnsafeDependencyForEELoopWithSideEffects", ORE, + TheLoop); + return false; + } + return canVectorizeIndirectUnsafeDependences(); + } if (LAI->hasLoadStoreDependenceInvolvingLoopInvariantAddress()) { reportVectorizationFailure("We don't allow storing to uniform addresses", @@ -1755,16 +1767,24 @@ bool LoopVectorizationLegality::isVectorizableEarlyExitLoop() { } }; + bool HasSideEffects = false; for (auto *BB : TheLoop->blocks()) for (auto &I : *BB) { if (I.mayWriteToMemory()) { - // We don't support writes to memory. + if (isa<StoreInst>(&I) && cast<StoreInst>(&I)->isSimple()) { + HasSideEffects = true; + continue; + } + + // We don't support complex writes to memory. reportVectorizationFailure( - "Writes to memory unsupported in early exit loops", - "Cannot vectorize early exit loop with writes to memory", + "Complex writes to memory unsupported in early exit loops", + "Cannot vectorize early exit loop with complex writes to memory", "WritesInEarlyExitLoop", ORE, TheLoop); return false; - } else if (!IsSafeOperation(&I)) { + } + + if (!IsSafeOperation(&I)) { reportVectorizationFailure("Early exit loop contains operations that " "cannot be speculatively executed", "UnsafeOperationsEarlyExitLoop", ORE, @@ -1777,15 +1797,22 @@ bool LoopVectorizationLegality::isVectorizableEarlyExitLoop() { assert(LatchBB->getUniquePredecessor() == SingleUncountableExitingBlock && "Expected latch predecessor to be the early exiting block"); - Predicates.clear(); SmallVector<LoadInst *, 4> NonDerefLoads; - if (!isReadOnlyLoop(TheLoop, PSE.getSE(), DT, AC, NonDerefLoads, - &Predicates)) { - reportVectorizationFailure("Loop may fault", - "Cannot vectorize non-read-only early exit loop", - "NonReadOnlyEarlyExitLoop", ORE, TheLoop); + // TODO: Handle loops that may fault. + if (!HasSideEffects) { + // Read-only loop. + Predicates.clear(); + if (!isReadOnlyLoop(TheLoop, PSE.getSE(), DT, AC, NonDerefLoads, + &Predicates)) { + reportVectorizationFailure( + "Loop may fault", "Cannot vectorize non-read-only early exit loop", + "NonReadOnlyEarlyExitLoop", ORE, TheLoop); + return false; + } + } else if (!canUncountableExitConditionLoadBeMoved( + SingleUncountableExitingBlock)) return false; - } + // Check non-dereferenceable loads if any. for (LoadInst *LI : NonDerefLoads) { // Only support unit-stride access for now. @@ -1813,6 +1840,99 @@ bool LoopVectorizationLegality::isVectorizableEarlyExitLoop() { "backedge taken count: " << *SymbolicMaxBTC << '\n'); UncountableExitingBB = SingleUncountableExitingBlock; + UncountableExitWithSideEffects = HasSideEffects; + return true; +} + +bool LoopVectorizationLegality::canUncountableExitConditionLoadBeMoved( + BasicBlock *ExitingBlock) { + // Try to find a load in the critical path for the uncountable exit condition. + // This is currently matching about the simplest form we can, expecting + // only one in-loop load, the result of which is directly compared against + // a loop-invariant value. + // FIXME: We're insisting on a single use for now, because otherwise we will + // need to make PHI nodes for other users. That can be done once the initial + // transform code lands. + auto *Br = cast<BranchInst>(ExitingBlock->getTerminator()); + + using namespace llvm::PatternMatch; + Instruction *L = nullptr; + Value *Ptr = nullptr; + Value *R = nullptr; + if (!match(Br->getCondition(), + m_OneUse(m_ICmp(m_OneUse(m_Instruction(L, m_Load(m_Value(Ptr)))), + m_Value(R))))) { + reportVectorizationFailure( + "Early exit loop with store but no supported condition load", + "NoConditionLoadForEarlyExitLoop", ORE, TheLoop); + return false; + } + + // FIXME: Don't rely on operand ordering for the comparison. + if (!TheLoop->isLoopInvariant(R)) { + reportVectorizationFailure( + "Early exit loop with store but no supported condition load", + "NoConditionLoadForEarlyExitLoop", ORE, TheLoop); + return false; + } + + // Make sure that the load address is not loop invariant; we want an + // address calculation that we can rotate to the next vector iteration. + const auto *AR = dyn_cast<SCEVAddRecExpr>(PSE.getSE()->getSCEV(Ptr)); + if (!AR || AR->getLoop() != TheLoop || !AR->isAffine()) { + reportVectorizationFailure( + "Uncountable exit condition depends on load with an address that is " + "not an add recurrence in the loop", + "EarlyExitLoadInvariantAddress", ORE, TheLoop); + return false; + } + + // FIXME: Support gathers after first-faulting load support lands. + SmallVector<const SCEVPredicate *, 4> Predicates; + LoadInst *Load = cast<LoadInst>(L); + if (!isDereferenceableAndAlignedInLoop(Load, TheLoop, *PSE.getSE(), *DT, AC, + &Predicates)) { + reportVectorizationFailure( + "Loop may fault", + "Cannot vectorize potentially faulting early exit loop", + "PotentiallyFaultingEarlyExitLoop", ORE, TheLoop); + return false; + } + + ICFLoopSafetyInfo SafetyInfo; + SafetyInfo.computeLoopSafetyInfo(TheLoop); + // We need to know that load will be executed before we can hoist a + // copy out to run just before the first iteration. + // FIXME: Currently, other restrictions prevent us from reaching this point + // with a loop where the uncountable exit condition is determined + // by a conditional load. + assert(SafetyInfo.isGuaranteedToExecute(*Load, DT, TheLoop) && + "Unhandled control flow in uncountable exit loop with side effects"); + + // Prohibit any potential aliasing with any instruction in the loop which + // might store to memory. + // FIXME: Relax this constraint where possible. + for (auto *BB : TheLoop->blocks()) { + for (auto &I : *BB) { + if (&I == Load) + continue; + + if (I.mayWriteToMemory()) { + if (auto *SI = dyn_cast<StoreInst>(&I)) { + AliasResult AR = AA->alias(Ptr, SI->getPointerOperand()); + if (AR == AliasResult::NoAlias) + continue; + } + + reportVectorizationFailure( + "Cannot determine whether critical uncountable exit load address " + "does not alias with a memory write", + "CantVectorizeAliasWithCriticalUncountableExitLoad", ORE, TheLoop); + return false; + } + } + } + return true; } @@ -1885,6 +2005,7 @@ bool LoopVectorizationLegality::canVectorize(bool UseVPlanNativePath) { } else { if (!isVectorizableEarlyExitLoop()) { assert(!hasUncountableEarlyExit() && + !hasUncountableExitWithSideEffects() && "Must be false without vectorizable early-exit loop"); if (DoExtraAnalysis) Result = false; @@ -1903,6 +2024,15 @@ bool LoopVectorizationLegality::canVectorize(bool UseVPlanNativePath) { return false; } + // Bail out for state-changing loops with uncountable exits for now. + if (UncountableExitWithSideEffects) { + reportVectorizationFailure( + "Writes to memory unsupported in early exit loops", + "Cannot vectorize early exit loop with writes to memory", + "WritesInEarlyExitLoop", ORE, TheLoop); + return false; + } + if (Result) { LLVM_DEBUG(dbgs() << "LV: We can vectorize this loop" << (LAI->getRuntimePointerChecking()->Need diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp index d78e190e8bf7..b4acda80cfb9 100644 --- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp +++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp @@ -1848,6 +1848,8 @@ public: "claimed checks are required"); } + SCEVExp.eraseDeadInstructions(SCEVCheckCond); + if (!MemCheckBlock && !SCEVCheckBlock) return; @@ -6905,6 +6907,16 @@ static bool planContainsAdditionalSimplifications(VPlan &Plan, if (isa<VPPartialReductionRecipe>(&R)) return true; + // The VPlan-based cost model can analyze if recipes are scalar + // recursively, but the legacy cost model cannot. + if (auto *WidenMemR = dyn_cast<VPWidenMemoryRecipe>(&R)) { + auto *AddrI = dyn_cast<Instruction>( + getLoadStorePointerOperand(&WidenMemR->getIngredient())); + if (AddrI && vputils::isSingleScalar(WidenMemR->getAddr()) != + CostCtx.isLegacyUniformAfterVectorization(AddrI, VF)) + return true; + } + /// If a VPlan transform folded a recipe to one producing a single-scalar, /// but the original instruction wasn't uniform-after-vectorization in the /// legacy cost model, the legacy cost overestimates the actual cost. @@ -9808,7 +9820,7 @@ bool LoopVectorizePass::processLoop(Loop *L) { // Check if it is legal to vectorize the loop. LoopVectorizationRequirements Requirements; LoopVectorizationLegality LVL(L, PSE, DT, TTI, TLI, F, *LAIs, LI, ORE, - &Requirements, &Hints, DB, AC, BFI, PSI); + &Requirements, &Hints, DB, AC, BFI, PSI, AA); if (!LVL.canVectorize(EnableVPlanNativePath)) { LLVM_DEBUG(dbgs() << "LV: Not vectorizing: Cannot prove legality.\n"); Hints.emitRemarkWithHints(); @@ -10245,6 +10257,7 @@ PreservedAnalyses LoopVectorizePass::run(Function &F, DB = &AM.getResult<DemandedBitsAnalysis>(F); ORE = &AM.getResult<OptimizationRemarkEmitterAnalysis>(F); LAIs = &AM.getResult<LoopAccessAnalysis>(F); + AA = &AM.getResult<AAManager>(F); auto &MAMProxy = AM.getResult<ModuleAnalysisManagerFunctionProxy>(F); PSI = MAMProxy.getCachedResult<ProfileSummaryAnalysis>(*F.getParent()); diff --git a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp index 1cfcd3ffbd66..75cace77ec53 100644 --- a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp +++ b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp @@ -11287,9 +11287,6 @@ void BoUpSLP::buildTreeRec(ArrayRef<Value *> VLRef, unsigned Depth, if (!canBuildSplitNode(VL, LocalState, Op1, Op2, ReorderIndices)) return false; - SmallVector<Value *> NewVL(VL.size()); - copy(Op1, NewVL.begin()); - copy(Op2, std::next(NewVL.begin(), Op1.size())); auto Invalid = ScheduleBundle::invalid(); auto *TE = newTreeEntry(VL, TreeEntry::SplitVectorize, Invalid, LocalState, UserTreeIdx, {}, ReorderIndices); @@ -20788,6 +20785,14 @@ BoUpSLP::BlockScheduling::tryScheduleBundle(ArrayRef<Value *> VL, BoUpSLP *SLP, continue; } auto *SD = cast<ScheduleData>(SE); + if (SD->hasValidDependencies() && + (!S.areInstructionsWithCopyableElements() || + !S.isCopyableElement(SD->getInst())) && + !getScheduleCopyableData(SD->getInst()).empty() && EI.UserTE && + EI.UserTE->hasState() && + (!EI.UserTE->hasCopyableElements() || + !EI.UserTE->isCopyableElement(SD->getInst()))) + SD->clearDirectDependencies(); for (const Use &U : SD->getInst()->operands()) { unsigned &NumOps = UserOpToNumOps diff --git a/llvm/lib/Transforms/Vectorize/VPlan.h b/llvm/lib/Transforms/Vectorize/VPlan.h index b93bdf244237..53291a931530 100644 --- a/llvm/lib/Transforms/Vectorize/VPlan.h +++ b/llvm/lib/Transforms/Vectorize/VPlan.h @@ -721,6 +721,10 @@ public: AllFlags = Other.AllFlags; } + /// Only keep flags also present in \p Other. \p Other must have the same + /// OpType as the current object. + void intersectFlags(const VPIRFlags &Other); + /// Drop all poison-generating flags. void dropPoisonGeneratingFlags() { // NOTE: This needs to be kept in-sync with diff --git a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp index 46162a927646..bf5148954309 100644 --- a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp +++ b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp @@ -392,6 +392,42 @@ void VPPartialReductionRecipe::print(raw_ostream &O, const Twine &Indent, } #endif +void VPIRFlags::intersectFlags(const VPIRFlags &Other) { + assert(OpType == Other.OpType && "OpType must match"); + switch (OpType) { + case OperationType::OverflowingBinOp: + WrapFlags.HasNUW &= Other.WrapFlags.HasNUW; + WrapFlags.HasNSW &= Other.WrapFlags.HasNSW; + break; + case OperationType::Trunc: + TruncFlags.HasNUW &= Other.TruncFlags.HasNUW; + TruncFlags.HasNSW &= Other.TruncFlags.HasNSW; + break; + case OperationType::DisjointOp: + DisjointFlags.IsDisjoint &= Other.DisjointFlags.IsDisjoint; + break; + case OperationType::PossiblyExactOp: + ExactFlags.IsExact &= Other.ExactFlags.IsExact; + break; + case OperationType::GEPOp: + GEPFlags &= Other.GEPFlags; + break; + case OperationType::FPMathOp: + FMFs.NoNaNs &= Other.FMFs.NoNaNs; + FMFs.NoInfs &= Other.FMFs.NoInfs; + break; + case OperationType::NonNegOp: + NonNegFlags.NonNeg &= Other.NonNegFlags.NonNeg; + break; + case OperationType::Cmp: + assert(CmpPredicate == Other.CmpPredicate && "Cannot drop CmpPredicate"); + break; + case OperationType::Other: + assert(AllFlags == Other.AllFlags && "Cannot drop other flags"); + break; + } +} + FastMathFlags VPIRFlags::getFastMathFlags() const { assert(OpType == OperationType::FPMathOp && "recipe doesn't have fast math flags"); diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp index 10b2f5df2e23..2cac5557daee 100644 --- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp +++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp @@ -349,7 +349,7 @@ static VPRegionBlock *createReplicateRegion(VPReplicateRecipe *PredRecipe, auto *BlockInMask = PredRecipe->getMask(); auto *MaskDef = BlockInMask->getDefiningRecipe(); auto *BOMRecipe = new VPBranchOnMaskRecipe( - BlockInMask, MaskDef ? MaskDef->getDebugLoc() : DebugLoc()); + BlockInMask, MaskDef ? MaskDef->getDebugLoc() : DebugLoc::getUnknown()); auto *Entry = Plan.createVPBasicBlock(Twine(RegionName) + ".entry", BOMRecipe); @@ -864,8 +864,8 @@ static VPValue *optimizeLatchExitInductionUser( Type *StepTy = TypeInfo.inferScalarType(Step); auto *Zero = Plan.getOrAddLiveIn(ConstantInt::get(StepTy, 0)); return B.createPtrAdd(EndValue, - B.createNaryOp(Instruction::Sub, {Zero, Step}), {}, - "ind.escape"); + B.createNaryOp(Instruction::Sub, {Zero, Step}), + DebugLoc::getUnknown(), "ind.escape"); } if (ScalarTy->isFloatingPointTy()) { const auto &ID = WideIV->getInductionDescriptor(); @@ -1270,11 +1270,12 @@ static void simplifyRecipe(VPRecipeBase &R, VPTypeAnalysis &TypeInfo) { return; } - VPInstruction *OpVPI; - if (match(Def, m_ExtractLastElement(m_VPInstruction(OpVPI))) && - OpVPI->isVectorToScalar()) { - Def->replaceAllUsesWith(OpVPI); - return; + if (match(Def, + m_VPInstruction<VPInstruction::ExtractLastElement>(m_VPValue(A))) && + vputils::isSingleScalar(A) && all_of(A->users(), [Def, A](VPUser *U) { + return U->usesScalars(A) || Def == U; + })) { + return Def->replaceAllUsesWith(A); } } @@ -1307,6 +1308,20 @@ static void narrowToSingleScalarRecipes(VPlan &Plan) { continue; auto *RepOrWidenR = cast<VPSingleDefRecipe>(&R); + if (RepR && isa<StoreInst>(RepR->getUnderlyingInstr()) && + vputils::isSingleScalar(RepR->getOperand(1))) { + auto *Clone = new VPReplicateRecipe( + RepOrWidenR->getUnderlyingInstr(), RepOrWidenR->operands(), + true /*IsSingleScalar*/, nullptr /*Mask*/, *RepR /*Metadata*/); + Clone->insertBefore(RepOrWidenR); + auto *Ext = new VPInstruction(VPInstruction::ExtractLastElement, + {Clone->getOperand(0)}); + Ext->insertBefore(Clone); + Clone->setOperand(0, Ext); + RepR->eraseFromParent(); + continue; + } + // Skip recipes that aren't single scalars or don't have only their // scalar results used. In the latter case, we would introduce extra // broadcasts. @@ -2042,9 +2057,9 @@ void VPlanTransforms::cse(VPlan &Plan) { // V must dominate Def for a valid replacement. if (!VPDT.dominates(V->getParent(), VPBB)) continue; - // Drop poison-generating flags when reusing a value. + // Only keep flags present on both V and Def. if (auto *RFlags = dyn_cast<VPRecipeWithIRFlags>(V)) - RFlags->dropPoisonGeneratingFlags(); + RFlags->intersectFlags(*cast<VPRecipeWithIRFlags>(Def)); Def->replaceAllUsesWith(V); continue; } @@ -2214,10 +2229,10 @@ void VPlanTransforms::optimize(VPlan &Plan) { runPass(removeRedundantInductionCasts, Plan); runPass(simplifyRecipes, Plan); - runPass(simplifyBlends, Plan); runPass(removeDeadRecipes, Plan); - runPass(narrowToSingleScalarRecipes, Plan); + runPass(simplifyBlends, Plan); runPass(legalizeAndOptimizeInductions, Plan); + runPass(narrowToSingleScalarRecipes, Plan); runPass(removeRedundantExpandSCEVRecipes, Plan); runPass(simplifyRecipes, Plan); runPass(removeBranchOnConst, Plan); @@ -2311,7 +2326,8 @@ static VPActiveLaneMaskPHIRecipe *addVPLaneMaskPhiAndUpdateExitBranch( // Now create the ActiveLaneMaskPhi recipe in the main loop using the // preheader ActiveLaneMask instruction. - auto *LaneMaskPhi = new VPActiveLaneMaskPHIRecipe(EntryALM, DebugLoc()); + auto *LaneMaskPhi = + new VPActiveLaneMaskPHIRecipe(EntryALM, DebugLoc::getUnknown()); LaneMaskPhi->insertAfter(CanonicalIVPHI); // Create the active lane mask for the next iteration of the loop before the @@ -2538,11 +2554,11 @@ static void transformRecipestoEVLRecipes(VPlan &Plan, VPValue &EVL) { VPBuilder Builder(LoopRegion->getPreheaderVPBB()); MaxEVL = Builder.createScalarZExtOrTrunc( MaxEVL, Type::getInt32Ty(Plan.getContext()), - TypeInfo.inferScalarType(MaxEVL), DebugLoc()); + TypeInfo.inferScalarType(MaxEVL), DebugLoc::getUnknown()); Builder.setInsertPoint(Header, Header->getFirstNonPhi()); - VPValue *PrevEVL = - Builder.createScalarPhi({MaxEVL, &EVL}, DebugLoc(), "prev.evl"); + VPValue *PrevEVL = Builder.createScalarPhi( + {MaxEVL, &EVL}, DebugLoc::getUnknown(), "prev.evl"); for (VPBasicBlock *VPBB : VPBlockUtils::blocksOnly<VPBasicBlock>( vp_depth_first_deep(Plan.getVectorLoopRegion()->getEntry()))) { @@ -2672,7 +2688,7 @@ void VPlanTransforms::addExplicitVectorLength( VPValue *StartV = CanonicalIVPHI->getStartValue(); // Create the ExplicitVectorLengthPhi recipe in the main loop. - auto *EVLPhi = new VPEVLBasedIVPHIRecipe(StartV, DebugLoc()); + auto *EVLPhi = new VPEVLBasedIVPHIRecipe(StartV, DebugLoc::getUnknown()); EVLPhi->insertAfter(CanonicalIVPHI); VPBuilder Builder(Header, Header->getFirstNonPhi()); // Create the AVL (application vector length), starting from TC -> 0 in steps @@ -2686,10 +2702,11 @@ void VPlanTransforms::addExplicitVectorLength( VPValue *AVLSafe = Plan.getOrAddLiveIn(ConstantInt::get(CanIVTy, *MaxSafeElements)); VPValue *Cmp = Builder.createICmp(ICmpInst::ICMP_ULT, AVL, AVLSafe); - AVL = Builder.createSelect(Cmp, AVL, AVLSafe, DebugLoc(), "safe_avl"); + AVL = Builder.createSelect(Cmp, AVL, AVLSafe, DebugLoc::getUnknown(), + "safe_avl"); } auto *VPEVL = Builder.createNaryOp(VPInstruction::ExplicitVectorLength, AVL, - DebugLoc()); + DebugLoc::getUnknown()); auto *CanonicalIVIncrement = cast<VPInstruction>(CanonicalIVPHI->getBackedgeValue()); @@ -3116,8 +3133,8 @@ expandVPWidenIntOrFpInduction(VPWidenIntOrFpInductionRecipe *WidenIVR, VPValue *SplatStep = Builder.createNaryOp(VPInstruction::Broadcast, Step); Init = Builder.createNaryOp(MulOp, {Init, SplatStep}, Flags); - Init = - Builder.createNaryOp(AddOp, {SplatStart, Init}, Flags, {}, "induction"); + Init = Builder.createNaryOp(AddOp, {SplatStart, Init}, Flags, + DebugLoc::getUnknown(), "induction"); // Create the widened phi of the vector IV. auto *WidePHI = new VPWidenPHIRecipe(WidenIVR->getPHINode(), nullptr, @@ -4014,9 +4031,10 @@ void VPlanTransforms::narrowInterleaveGroups(VPlan &Plan, ElementCount VF, return; // Convert InterleaveGroup \p R to a single VPWidenLoadRecipe. - auto NarrowOp = [](VPValue *V) -> VPValue * { + SmallPtrSet<VPValue *, 4> NarrowedOps; + auto NarrowOp = [&NarrowedOps](VPValue *V) -> VPValue * { auto *R = V->getDefiningRecipe(); - if (!R) + if (!R || NarrowedOps.contains(V)) return V; if (auto *LoadGroup = dyn_cast<VPInterleaveRecipe>(R)) { // Narrow interleave group to wide load, as transformed VPlan will only @@ -4026,6 +4044,7 @@ void VPlanTransforms::narrowInterleaveGroups(VPlan &Plan, ElementCount VF, LoadGroup->getAddr(), LoadGroup->getMask(), /*Consecutive=*/true, /*Reverse=*/false, {}, LoadGroup->getDebugLoc()); L->insertBefore(LoadGroup); + NarrowedOps.insert(L); return L; } @@ -4033,6 +4052,7 @@ void VPlanTransforms::narrowInterleaveGroups(VPlan &Plan, ElementCount VF, assert(RepR->isSingleScalar() && isa<LoadInst>(RepR->getUnderlyingInstr()) && "must be a single scalar load"); + NarrowedOps.insert(RepR); return RepR; } auto *WideLoad = cast<VPWidenLoadRecipe>(R); @@ -4046,6 +4066,7 @@ void VPlanTransforms::narrowInterleaveGroups(VPlan &Plan, ElementCount VF, /*IsUniform*/ true, /*Mask*/ nullptr, *WideLoad); N->insertBefore(WideLoad); + NarrowedOps.insert(N); return N; }; diff --git a/llvm/lib/Transforms/Vectorize/VPlanUnroll.cpp b/llvm/lib/Transforms/Vectorize/VPlanUnroll.cpp index 7a63d20825a3..443df167378b 100644 --- a/llvm/lib/Transforms/Vectorize/VPlanUnroll.cpp +++ b/llvm/lib/Transforms/Vectorize/VPlanUnroll.cpp @@ -536,16 +536,9 @@ void VPlanTransforms::replicateByVF(VPlan &Plan, ElementCount VF) { VPBuilder Builder(RepR); if (RepR->getNumUsers() == 0) { - if (isa<StoreInst>(RepR->getUnderlyingInstr()) && - vputils::isSingleScalar(RepR->getOperand(1))) { - // Stores to invariant addresses need to store the last lane only. - cloneForLane(Plan, Builder, IdxTy, RepR, VPLane::getLastLaneForVF(VF), - Def2LaneDefs); - } else { - // Create single-scalar version of RepR for all lanes. - for (unsigned I = 0; I != VF.getKnownMinValue(); ++I) - cloneForLane(Plan, Builder, IdxTy, RepR, VPLane(I), Def2LaneDefs); - } + // Create single-scalar version of RepR for all lanes. + for (unsigned I = 0; I != VF.getKnownMinValue(); ++I) + cloneForLane(Plan, Builder, IdxTy, RepR, VPLane(I), Def2LaneDefs); RepR->eraseFromParent(); continue; } diff --git a/llvm/lib/Transforms/Vectorize/VectorCombine.cpp b/llvm/lib/Transforms/Vectorize/VectorCombine.cpp index 17cb18a22336..a84b6f59971c 100644 --- a/llvm/lib/Transforms/Vectorize/VectorCombine.cpp +++ b/llvm/lib/Transforms/Vectorize/VectorCombine.cpp @@ -962,6 +962,9 @@ bool VectorCombine::foldBitOpOfCastConstant(Instruction &I) { // Only handle supported cast operations switch (CastOpcode) { case Instruction::BitCast: + case Instruction::ZExt: + case Instruction::SExt: + case Instruction::Trunc: break; default: return false; |