diff options
| author | Mingming Liu <mingmingl@google.com> | 2025-09-10 15:25:31 -0700 |
|---|---|---|
| committer | GitHub <noreply@github.com> | 2025-09-10 15:25:31 -0700 |
| commit | 1417dafa1db9cb1b2b09438aa9f53ea5ab6e36e2 (patch) | |
| tree | 57f4b1f313c8cf74eed8819870f39c36ea263c68 /llvm/lib/Target/AArch64 | |
| parent | 898b813bc8a6d0276bf0f4769f5f2f64b34e632d (diff) | |
| parent | b8cefcb601ddaa18482555c4ff363c01a270c2fe (diff) | |
Merge branch 'main' into users/mingmingl-llvm/samplefdo-profile-formatusers/mingmingl-llvm/samplefdo-profile-format
Diffstat (limited to 'llvm/lib/Target/AArch64')
41 files changed, 2751 insertions, 1587 deletions
diff --git a/llvm/lib/Target/AArch64/AArch64AsmPrinter.cpp b/llvm/lib/Target/AArch64/AArch64AsmPrinter.cpp index c52487ab8a79..c31a090bba77 100644 --- a/llvm/lib/Target/AArch64/AArch64AsmPrinter.cpp +++ b/llvm/lib/Target/AArch64/AArch64AsmPrinter.cpp @@ -307,6 +307,7 @@ private: /// Emit instruction to set float register to zero. void emitFMov0(const MachineInstr &MI); + void emitFMov0AsFMov(const MachineInstr &MI, Register DestReg); using MInstToMCSymbol = std::map<const MachineInstr *, MCSymbol *>; @@ -734,7 +735,7 @@ void AArch64AsmPrinter::emitHwasanMemaccessSymbols(Module &M) { const Triple &TT = TM.getTargetTriple(); assert(TT.isOSBinFormatELF()); std::unique_ptr<MCSubtargetInfo> STI( - TM.getTarget().createMCSubtargetInfo(TT.str(), "", "")); + TM.getTarget().createMCSubtargetInfo(TT, "", "")); assert(STI && "Unable to create subtarget info"); this->STI = static_cast<const AArch64Subtarget *>(&*STI); @@ -1829,45 +1830,77 @@ void AArch64AsmPrinter::emitMOVK(Register Dest, uint64_t Imm, unsigned Shift) { void AArch64AsmPrinter::emitFMov0(const MachineInstr &MI) { Register DestReg = MI.getOperand(0).getReg(); - if (STI->hasZeroCycleZeroingFP() && !STI->hasZeroCycleZeroingFPWorkaround() && - STI->isNeonAvailable()) { - // Convert H/S register to corresponding D register - if (AArch64::H0 <= DestReg && DestReg <= AArch64::H31) - DestReg = AArch64::D0 + (DestReg - AArch64::H0); - else if (AArch64::S0 <= DestReg && DestReg <= AArch64::S31) - DestReg = AArch64::D0 + (DestReg - AArch64::S0); - else - assert(AArch64::D0 <= DestReg && DestReg <= AArch64::D31); + if (!STI->hasZeroCycleZeroingFPWorkaround() && STI->isNeonAvailable()) { + if (STI->hasZeroCycleZeroingFPR64()) { + // Convert H/S register to corresponding D register + const AArch64RegisterInfo *TRI = STI->getRegisterInfo(); + if (AArch64::FPR16RegClass.contains(DestReg)) + DestReg = TRI->getMatchingSuperReg(DestReg, AArch64::hsub, + &AArch64::FPR64RegClass); + else if (AArch64::FPR32RegClass.contains(DestReg)) + DestReg = TRI->getMatchingSuperReg(DestReg, AArch64::ssub, + &AArch64::FPR64RegClass); + else + assert(AArch64::FPR64RegClass.contains(DestReg)); + + MCInst MOVI; + MOVI.setOpcode(AArch64::MOVID); + MOVI.addOperand(MCOperand::createReg(DestReg)); + MOVI.addOperand(MCOperand::createImm(0)); + EmitToStreamer(*OutStreamer, MOVI); + } else if (STI->hasZeroCycleZeroingFPR128()) { + // Convert H/S/D register to corresponding Q register + const AArch64RegisterInfo *TRI = STI->getRegisterInfo(); + if (AArch64::FPR16RegClass.contains(DestReg)) { + DestReg = TRI->getMatchingSuperReg(DestReg, AArch64::hsub, + &AArch64::FPR128RegClass); + } else if (AArch64::FPR32RegClass.contains(DestReg)) { + DestReg = TRI->getMatchingSuperReg(DestReg, AArch64::ssub, + &AArch64::FPR128RegClass); + } else { + assert(AArch64::FPR64RegClass.contains(DestReg)); + DestReg = TRI->getMatchingSuperReg(DestReg, AArch64::dsub, + &AArch64::FPR128RegClass); + } - MCInst MOVI; - MOVI.setOpcode(AArch64::MOVID); - MOVI.addOperand(MCOperand::createReg(DestReg)); - MOVI.addOperand(MCOperand::createImm(0)); - EmitToStreamer(*OutStreamer, MOVI); - } else { - MCInst FMov; - switch (MI.getOpcode()) { - default: llvm_unreachable("Unexpected opcode"); - case AArch64::FMOVH0: - FMov.setOpcode(STI->hasFullFP16() ? AArch64::FMOVWHr : AArch64::FMOVWSr); - if (!STI->hasFullFP16()) - DestReg = (AArch64::S0 + (DestReg - AArch64::H0)); - FMov.addOperand(MCOperand::createReg(DestReg)); - FMov.addOperand(MCOperand::createReg(AArch64::WZR)); - break; - case AArch64::FMOVS0: - FMov.setOpcode(AArch64::FMOVWSr); - FMov.addOperand(MCOperand::createReg(DestReg)); - FMov.addOperand(MCOperand::createReg(AArch64::WZR)); - break; - case AArch64::FMOVD0: - FMov.setOpcode(AArch64::FMOVXDr); - FMov.addOperand(MCOperand::createReg(DestReg)); - FMov.addOperand(MCOperand::createReg(AArch64::XZR)); - break; + MCInst MOVI; + MOVI.setOpcode(AArch64::MOVIv2d_ns); + MOVI.addOperand(MCOperand::createReg(DestReg)); + MOVI.addOperand(MCOperand::createImm(0)); + EmitToStreamer(*OutStreamer, MOVI); + } else { + emitFMov0AsFMov(MI, DestReg); } - EmitToStreamer(*OutStreamer, FMov); + } else { + emitFMov0AsFMov(MI, DestReg); + } +} + +void AArch64AsmPrinter::emitFMov0AsFMov(const MachineInstr &MI, + Register DestReg) { + MCInst FMov; + switch (MI.getOpcode()) { + default: + llvm_unreachable("Unexpected opcode"); + case AArch64::FMOVH0: + FMov.setOpcode(STI->hasFullFP16() ? AArch64::FMOVWHr : AArch64::FMOVWSr); + if (!STI->hasFullFP16()) + DestReg = (AArch64::S0 + (DestReg - AArch64::H0)); + FMov.addOperand(MCOperand::createReg(DestReg)); + FMov.addOperand(MCOperand::createReg(AArch64::WZR)); + break; + case AArch64::FMOVS0: + FMov.setOpcode(AArch64::FMOVWSr); + FMov.addOperand(MCOperand::createReg(DestReg)); + FMov.addOperand(MCOperand::createReg(AArch64::WZR)); + break; + case AArch64::FMOVD0: + FMov.setOpcode(AArch64::FMOVXDr); + FMov.addOperand(MCOperand::createReg(DestReg)); + FMov.addOperand(MCOperand::createReg(AArch64::XZR)); + break; } + EmitToStreamer(*OutStreamer, FMov); } Register AArch64AsmPrinter::emitPtrauthDiscriminator(uint16_t Disc, @@ -2229,13 +2262,24 @@ void AArch64AsmPrinter::emitPtrauthBranch(const MachineInstr *MI) { if (BrTarget == AddrDisc) report_fatal_error("Branch target is signed with its own value"); - // If we are printing BLRA pseudo instruction, then x16 and x17 are - // implicit-def'ed by the MI and AddrDisc is not used as any other input, so - // try to save one MOV by setting MayUseAddrAsScratch. + // If we are printing BLRA pseudo, try to save one MOV by making use of the + // fact that x16 and x17 are described as clobbered by the MI instruction and + // AddrDisc is not used as any other input. + // + // Back in the day, emitPtrauthDiscriminator was restricted to only returning + // either x16 or x17, meaning the returned register is always among the + // implicit-def'ed registers of BLRA pseudo. Now this property can be violated + // if isX16X17Safer predicate is false, thus manually check if AddrDisc is + // among x16 and x17 to prevent clobbering unexpected registers. + // // Unlike BLRA, BRA pseudo is used to perform computed goto, and thus not // declared as clobbering x16/x17. + // + // FIXME: Make use of `killed` flags and register masks instead. + bool AddrDiscIsImplicitDef = + IsCall && (AddrDisc == AArch64::X16 || AddrDisc == AArch64::X17); Register DiscReg = emitPtrauthDiscriminator(Disc, AddrDisc, AArch64::X17, - /*MayUseAddrAsScratch=*/IsCall); + AddrDiscIsImplicitDef); bool IsZeroDisc = DiscReg == AArch64::XZR; unsigned Opc; @@ -2862,7 +2906,7 @@ void AArch64AsmPrinter::emitInstruction(const MachineInstr *MI) { MCInst TmpInst; TmpInst.setOpcode(AArch64::MOVIv16b_ns); TmpInst.addOperand(MCOperand::createReg(MI->getOperand(0).getReg())); - TmpInst.addOperand(MCOperand::createImm(MI->getOperand(1).getImm())); + TmpInst.addOperand(MCOperand::createImm(0)); EmitToStreamer(*OutStreamer, TmpInst); return; } @@ -2968,8 +3012,15 @@ void AArch64AsmPrinter::emitInstruction(const MachineInstr *MI) { // See the comments in emitPtrauthBranch. if (Callee == AddrDisc) report_fatal_error("Call target is signed with its own value"); + + // After isX16X17Safer predicate was introduced, emitPtrauthDiscriminator is + // no longer restricted to only reusing AddrDisc when it is X16 or X17 + // (which are implicit-def'ed by AUTH_TCRETURN pseudos), thus impose this + // restriction manually not to clobber an unexpected register. + bool AddrDiscIsImplicitDef = + AddrDisc == AArch64::X16 || AddrDisc == AArch64::X17; Register DiscReg = emitPtrauthDiscriminator(Disc, AddrDisc, ScratchReg, - /*MayUseAddrAsScratch=*/true); + AddrDiscIsImplicitDef); const bool IsZero = DiscReg == AArch64::XZR; const unsigned Opcodes[2][2] = {{AArch64::BRAA, AArch64::BRAAZ}, diff --git a/llvm/lib/Target/AArch64/AArch64BranchTargets.cpp b/llvm/lib/Target/AArch64/AArch64BranchTargets.cpp index 3436dc9ef452..137ff898e86a 100644 --- a/llvm/lib/Target/AArch64/AArch64BranchTargets.cpp +++ b/llvm/lib/Target/AArch64/AArch64BranchTargets.cpp @@ -30,6 +30,14 @@ using namespace llvm; #define AARCH64_BRANCH_TARGETS_NAME "AArch64 Branch Targets" namespace { +// BTI HINT encoding: base (32) plus 'c' (2) and/or 'j' (4). +enum : unsigned { + BTIBase = 32, // Base immediate for BTI HINT + BTIC = 1u << 1, // 2 + BTIJ = 1u << 2, // 4 + BTIMask = BTIC | BTIJ, +}; + class AArch64BranchTargets : public MachineFunctionPass { public: static char ID; @@ -42,6 +50,7 @@ private: void addBTI(MachineBasicBlock &MBB, bool CouldCall, bool CouldJump, bool NeedsWinCFI); }; + } // end anonymous namespace char AArch64BranchTargets::ID = 0; @@ -62,9 +71,8 @@ bool AArch64BranchTargets::runOnMachineFunction(MachineFunction &MF) { if (!MF.getInfo<AArch64FunctionInfo>()->branchTargetEnforcement()) return false; - LLVM_DEBUG( - dbgs() << "********** AArch64 Branch Targets **********\n" - << "********** Function: " << MF.getName() << '\n'); + LLVM_DEBUG(dbgs() << "********** AArch64 Branch Targets **********\n" + << "********** Function: " << MF.getName() << '\n'); const Function &F = MF.getFunction(); // LLVM does not consider basic blocks which are the targets of jump tables @@ -103,6 +111,12 @@ bool AArch64BranchTargets::runOnMachineFunction(MachineFunction &MF) { JumpTableTargets.count(&MBB)) CouldJump = true; + if (MBB.isEHPad()) { + if (HasWinCFI && (MBB.isEHFuncletEntry() || MBB.isCleanupFuncletEntry())) + CouldCall = true; + else + CouldJump = true; + } if (CouldCall || CouldJump) { addBTI(MBB, CouldCall, CouldJump, HasWinCFI); MadeChange = true; @@ -130,7 +144,12 @@ void AArch64BranchTargets::addBTI(MachineBasicBlock &MBB, bool CouldCall, auto MBBI = MBB.begin(); - // Skip the meta instructions, those will be removed anyway. + // If the block starts with EH_LABEL(s), skip them first. + while (MBBI != MBB.end() && MBBI->isEHLabel()) { + ++MBBI; + } + + // Skip meta/CFI/etc. (and EMITBKEY) to reach the first executable insn. for (; MBBI != MBB.end() && (MBBI->isMetaInstruction() || MBBI->getOpcode() == AArch64::EMITBKEY); ++MBBI) @@ -138,16 +157,21 @@ void AArch64BranchTargets::addBTI(MachineBasicBlock &MBB, bool CouldCall, // SCTLR_EL1.BT[01] is set to 0 by default which means // PACI[AB]SP are implicitly BTI C so no BTI C instruction is needed there. - if (MBBI != MBB.end() && HintNum == 34 && + if (MBBI != MBB.end() && ((HintNum & BTIMask) == BTIC) && (MBBI->getOpcode() == AArch64::PACIASP || MBBI->getOpcode() == AArch64::PACIBSP)) return; - if (HasWinCFI && MBBI->getFlag(MachineInstr::FrameSetup)) { - BuildMI(MBB, MBB.begin(), MBB.findDebugLoc(MBB.begin()), - TII->get(AArch64::SEH_Nop)); + // Insert BTI exactly at the first executable instruction. + const DebugLoc DL = MBB.findDebugLoc(MBBI); + MachineInstr *BTI = BuildMI(MBB, MBBI, DL, TII->get(AArch64::HINT)) + .addImm(HintNum) + .getInstr(); + + // WinEH: put .seh_nop after BTI when the first real insn is FrameSetup. + if (HasWinCFI && MBBI != MBB.end() && + MBBI->getFlag(MachineInstr::FrameSetup)) { + auto AfterBTI = std::next(MachineBasicBlock::iterator(BTI)); + BuildMI(MBB, AfterBTI, DL, TII->get(AArch64::SEH_Nop)); } - BuildMI(MBB, MBB.begin(), MBB.findDebugLoc(MBB.begin()), - TII->get(AArch64::HINT)) - .addImm(HintNum); } diff --git a/llvm/lib/Target/AArch64/AArch64Combine.td b/llvm/lib/Target/AArch64/AArch64Combine.td index 5f499e5e9700..076a6235eef0 100644 --- a/llvm/lib/Target/AArch64/AArch64Combine.td +++ b/llvm/lib/Target/AArch64/AArch64Combine.td @@ -369,5 +369,5 @@ def AArch64PostLegalizerCombiner commute_constant_to_rhs, extract_vec_elt_combines, push_freeze_to_prevent_poison_from_propagating, combine_mul_cmlt, combine_use_vector_truncate, - extmultomull, truncsat_combines]> { + extmultomull, truncsat_combines, lshr_of_trunc_of_lshr]> { } diff --git a/llvm/lib/Target/AArch64/AArch64ExpandPseudoInsts.cpp b/llvm/lib/Target/AArch64/AArch64ExpandPseudoInsts.cpp index 57dcd68595ff..79655e1c9529 100644 --- a/llvm/lib/Target/AArch64/AArch64ExpandPseudoInsts.cpp +++ b/llvm/lib/Target/AArch64/AArch64ExpandPseudoInsts.cpp @@ -1688,6 +1688,7 @@ bool AArch64ExpandPseudo::expandMI(MachineBasicBlock &MBB, } case AArch64::InOutZAUsePseudo: case AArch64::RequiresZASavePseudo: + case AArch64::SMEStateAllocPseudo: case AArch64::COALESCER_BARRIER_FPR16: case AArch64::COALESCER_BARRIER_FPR32: case AArch64::COALESCER_BARRIER_FPR64: diff --git a/llvm/lib/Target/AArch64/AArch64Features.td b/llvm/lib/Target/AArch64/AArch64Features.td index c1c1f0a1024d..46f5f0c1ca9d 100644 --- a/llvm/lib/Target/AArch64/AArch64Features.td +++ b/llvm/lib/Target/AArch64/AArch64Features.td @@ -621,25 +621,30 @@ def FeatureZCRegMoveGPR64 : SubtargetFeature<"zcm-gpr64", "HasZeroCycleRegMoveGP def FeatureZCRegMoveGPR32 : SubtargetFeature<"zcm-gpr32", "HasZeroCycleRegMoveGPR32", "true", "Has zero-cycle register moves for GPR32 registers">; +def FeatureZCRegMoveFPR128 : SubtargetFeature<"zcm-fpr128", "HasZeroCycleRegMoveFPR128", "true", + "Has zero-cycle register moves for FPR128 registers">; + def FeatureZCRegMoveFPR64 : SubtargetFeature<"zcm-fpr64", "HasZeroCycleRegMoveFPR64", "true", "Has zero-cycle register moves for FPR64 registers">; def FeatureZCRegMoveFPR32 : SubtargetFeature<"zcm-fpr32", "HasZeroCycleRegMoveFPR32", "true", "Has zero-cycle register moves for FPR32 registers">; -def FeatureZCZeroingGP : SubtargetFeature<"zcz-gp", "HasZeroCycleZeroingGP", "true", - "Has zero-cycle zeroing instructions for generic registers">; +def FeatureZCZeroingGPR64 : SubtargetFeature<"zcz-gpr64", "HasZeroCycleZeroingGPR64", "true", + "Has zero-cycle zeroing instructions for GPR64 registers">; + +def FeatureZCZeroingGPR32 : SubtargetFeature<"zcz-gpr32", "HasZeroCycleZeroingGPR32", "true", + "Has zero-cycle zeroing instructions for GPR32 registers">; + +def FeatureZCZeroingFPR128 : SubtargetFeature<"zcz-fpr128", "HasZeroCycleZeroingFPR128", "true", + "Has zero-cycle zeroing instructions for FPR128 registers">; // It is generally beneficial to rewrite "fmov s0, wzr" to "movi d0, #0". // as movi is more efficient across all cores. Newer cores can eliminate // fmovs early and there is no difference with movi, but this not true for // all implementations. -def FeatureNoZCZeroingFP : SubtargetFeature<"no-zcz-fp", "HasZeroCycleZeroingFP", "false", - "Has no zero-cycle zeroing instructions for FP registers">; - -def FeatureZCZeroing : SubtargetFeature<"zcz", "HasZeroCycleZeroing", "true", - "Has zero-cycle zeroing instructions", - [FeatureZCZeroingGP]>; +def FeatureNoZCZeroingFPR64 : SubtargetFeature<"no-zcz-fpr64", "HasZeroCycleZeroingFPR64", "false", + "Has no zero-cycle zeroing instructions for FPR64 registers">; /// ... but the floating-point version doesn't quite work in rare cases on older /// CPUs. @@ -730,9 +735,13 @@ def FeatureFuseArithmeticLogic : SubtargetFeature< "fuse-arith-logic", "HasFuseArithmeticLogic", "true", "CPU fuses arithmetic and logic operations">; -def FeatureFuseCCSelect : SubtargetFeature< - "fuse-csel", "HasFuseCCSelect", "true", - "CPU fuses conditional select operations">; +def FeatureFuseCmpCSel : SubtargetFeature< + "fuse-csel", "HasFuseCmpCSel", "true", + "CPU can fuse CMP and CSEL operations">; + +def FeatureFuseCmpCSet : SubtargetFeature< + "fuse-cset", "HasFuseCmpCSet", "true", + "CPU can fuse CMP and CSET operations">; def FeatureFuseCryptoEOR : SubtargetFeature< "fuse-crypto-eor", "HasFuseCryptoEOR", "true", diff --git a/llvm/lib/Target/AArch64/AArch64FrameLowering.cpp b/llvm/lib/Target/AArch64/AArch64FrameLowering.cpp index 7725fa4f1ccb..175b5e04d82f 100644 --- a/llvm/lib/Target/AArch64/AArch64FrameLowering.cpp +++ b/llvm/lib/Target/AArch64/AArch64FrameLowering.cpp @@ -211,6 +211,7 @@ #include "AArch64FrameLowering.h" #include "AArch64InstrInfo.h" #include "AArch64MachineFunctionInfo.h" +#include "AArch64PrologueEpilogue.h" #include "AArch64RegisterInfo.h" #include "AArch64Subtarget.h" #include "MCTargetDesc/AArch64AddressingModes.h" @@ -218,7 +219,6 @@ #include "Utils/AArch64SMEAttributes.h" #include "llvm/ADT/ScopeExit.h" #include "llvm/ADT/SmallVector.h" -#include "llvm/ADT/Statistic.h" #include "llvm/Analysis/ValueTracking.h" #include "llvm/CodeGen/CFIInstBuilder.h" #include "llvm/CodeGen/LivePhysRegs.h" @@ -293,8 +293,6 @@ static cl::opt<bool> DisableMultiVectorSpillFill( cl::desc("Disable use of LD/ST pairs for SME2 or SVE2p1"), cl::init(false), cl::Hidden); -STATISTIC(NumRedZoneFunctions, "Number of functions using red zone"); - /// Returns how much of the incoming argument stack area (in bytes) we should /// clean up in an epilogue. For the C calling convention this will be 0, for /// guaranteed tail call conventions it can be positive (a normal return or a @@ -328,23 +326,20 @@ static int64_t getArgumentStackToRestore(MachineFunction &MF, return ArgumentPopSize; } -static bool produceCompactUnwindFrame(MachineFunction &MF); -static bool needsWinCFI(const MachineFunction &MF); -static StackOffset getSVEStackSize(const MachineFunction &MF); -static Register findScratchNonCalleeSaveRegister(MachineBasicBlock *MBB, - bool HasCall = false); -static bool requiresSaveVG(const MachineFunction &MF); +static bool produceCompactUnwindFrame(const AArch64FrameLowering &, + MachineFunction &MF); // Conservatively, returns true if the function is likely to have an SVE vectors // on the stack. This function is safe to be called before callee-saves or // object offsets have been determined. -static bool isLikelyToHaveSVEStack(const MachineFunction &MF) { +static bool isLikelyToHaveSVEStack(const AArch64FrameLowering &AFL, + const MachineFunction &MF) { auto *AFI = MF.getInfo<AArch64FunctionInfo>(); if (AFI->isSVECC()) return true; if (AFI->hasCalculatedStackSizeSVE()) - return bool(getSVEStackSize(MF)); + return bool(AFL.getSVEStackSize(MF)); const MachineFrameInfo &MFI = MF.getFrameInfo(); for (int FI = MFI.getObjectIndexBegin(); FI < MFI.getObjectIndexEnd(); FI++) { @@ -372,7 +367,7 @@ bool AArch64FrameLowering::homogeneousPrologEpilog( return false; // TODO: SVE is not supported yet. - if (isLikelyToHaveSVEStack(MF)) + if (isLikelyToHaveSVEStack(*this, MF)) return false; // Bail on stack adjustment needed on return for simplicity. @@ -409,7 +404,7 @@ bool AArch64FrameLowering::homogeneousPrologEpilog( /// Returns true if CSRs should be paired. bool AArch64FrameLowering::producePairRegisters(MachineFunction &MF) const { - return produceCompactUnwindFrame(MF) || homogeneousPrologEpilog(MF); + return produceCompactUnwindFrame(*this, MF) || homogeneousPrologEpilog(MF); } /// This is the biggest offset to the stack pointer we can encode in aarch64 @@ -451,11 +446,10 @@ AArch64FrameLowering::getStackIDForScalableVectors() const { return TargetStackID::ScalableVector; } -/// Returns the size of the fixed object area (allocated next to sp on entry) -/// On Win64 this may include a var args area and an UnwindHelp object for EH. -static unsigned getFixedObjectSize(const MachineFunction &MF, - const AArch64FunctionInfo *AFI, bool IsWin64, - bool IsFunclet) { +unsigned +AArch64FrameLowering::getFixedObjectSize(const MachineFunction &MF, + const AArch64FunctionInfo *AFI, + bool IsWin64, bool IsFunclet) const { assert(AFI->getTailCallReservedStack() % 16 == 0 && "Tail call reserved stack must be aligned to 16 bytes"); if (!IsWin64 || IsFunclet) { @@ -494,7 +488,8 @@ static unsigned getFixedObjectSize(const MachineFunction &MF, } /// Returns the size of the entire SVE stackframe (calleesaves + spills). -static StackOffset getSVEStackSize(const MachineFunction &MF) { +StackOffset +AArch64FrameLowering::getSVEStackSize(const MachineFunction &MF) const { const AArch64FunctionInfo *AFI = MF.getInfo<AArch64FunctionInfo>(); return StackOffset::getScalable((int64_t)AFI->getStackSizeSVE()); } @@ -683,70 +678,6 @@ MachineBasicBlock::iterator AArch64FrameLowering::eliminateCallFramePseudoInstr( return MBB.erase(I); } -void AArch64FrameLowering::emitCalleeSavedGPRLocations( - MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI) const { - MachineFunction &MF = *MBB.getParent(); - MachineFrameInfo &MFI = MF.getFrameInfo(); - - const std::vector<CalleeSavedInfo> &CSI = MFI.getCalleeSavedInfo(); - if (CSI.empty()) - return; - - CFIInstBuilder CFIBuilder(MBB, MBBI, MachineInstr::FrameSetup); - for (const auto &Info : CSI) { - unsigned FrameIdx = Info.getFrameIdx(); - if (MFI.getStackID(FrameIdx) == TargetStackID::ScalableVector) - continue; - - assert(!Info.isSpilledToReg() && "Spilling to registers not implemented"); - int64_t Offset = MFI.getObjectOffset(FrameIdx) - getOffsetOfLocalArea(); - CFIBuilder.buildOffset(Info.getReg(), Offset); - } -} - -void AArch64FrameLowering::emitCalleeSavedSVELocations( - MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI) const { - MachineFunction &MF = *MBB.getParent(); - MachineFrameInfo &MFI = MF.getFrameInfo(); - - // Add callee saved registers to move list. - const std::vector<CalleeSavedInfo> &CSI = MFI.getCalleeSavedInfo(); - if (CSI.empty()) - return; - - const TargetSubtargetInfo &STI = MF.getSubtarget(); - const TargetRegisterInfo &TRI = *STI.getRegisterInfo(); - AArch64FunctionInfo &AFI = *MF.getInfo<AArch64FunctionInfo>(); - CFIInstBuilder CFIBuilder(MBB, MBBI, MachineInstr::FrameSetup); - - std::optional<int64_t> IncomingVGOffsetFromDefCFA; - if (requiresSaveVG(MF)) { - auto IncomingVG = *find_if( - reverse(CSI), [](auto &Info) { return Info.getReg() == AArch64::VG; }); - IncomingVGOffsetFromDefCFA = - MFI.getObjectOffset(IncomingVG.getFrameIdx()) - getOffsetOfLocalArea(); - } - - for (const auto &Info : CSI) { - if (MFI.getStackID(Info.getFrameIdx()) != TargetStackID::ScalableVector) - continue; - - // Not all unwinders may know about SVE registers, so assume the lowest - // common denominator. - assert(!Info.isSpilledToReg() && "Spilling to registers not implemented"); - MCRegister Reg = Info.getReg(); - if (!static_cast<const AArch64RegisterInfo &>(TRI).regNeedsCFI(Reg, Reg)) - continue; - - StackOffset Offset = - StackOffset::getScalable(MFI.getObjectOffset(Info.getFrameIdx())) - - StackOffset::getFixed(AFI.getCalleeSavedStackSize(MFI)); - - CFIBuilder.insertCFIInst( - createCFAOffset(TRI, Reg, Offset, IncomingVGOffsetFromDefCFA)); - } -} - void AArch64FrameLowering::resetCFIToInitialState( MachineBasicBlock &MBB) const { @@ -1088,8 +1019,8 @@ void AArch64FrameLowering::emitZeroCallUsedRegs(BitVector RegsToZero, } } -static bool windowsRequiresStackProbe(const MachineFunction &MF, - uint64_t StackSizeInBytes) { +bool AArch64FrameLowering::windowsRequiresStackProbe( + const MachineFunction &MF, uint64_t StackSizeInBytes) const { const AArch64Subtarget &Subtarget = MF.getSubtarget<AArch64Subtarget>(); const AArch64FunctionInfo &MFI = *MF.getInfo<AArch64FunctionInfo>(); // TODO: When implementing stack protectors, take that into account @@ -1108,19 +1039,9 @@ static void getLiveRegsForEntryMBB(LivePhysRegs &LiveRegs, LiveRegs.addReg(CSRegs[i]); } -// Find a scratch register that we can use at the start of the prologue to -// re-align the stack pointer. We avoid using callee-save registers since they -// may appear to be free when this is called from canUseAsPrologue (during -// shrink wrapping), but then no longer be free when this is called from -// emitPrologue. -// -// FIXME: This is a bit conservative, since in the above case we could use one -// of the callee-save registers as a scratch temp to re-align the stack pointer, -// but we would then have to make sure that we were in fact saving at least one -// callee-save register in the prologue, which is additional complexity that -// doesn't seem worth the benefit. -static Register findScratchNonCalleeSaveRegister(MachineBasicBlock *MBB, - bool HasCall) { +Register +AArch64FrameLowering::findScratchNonCalleeSaveRegister(MachineBasicBlock *MBB, + bool HasCall) const { MachineFunction *MF = MBB->getParent(); // If MBB is an entry block, use X9 as the scratch register @@ -1193,13 +1114,14 @@ bool AArch64FrameLowering::canUseAsPrologue( return true; } -static bool needsWinCFI(const MachineFunction &MF) { +bool AArch64FrameLowering::needsWinCFI(const MachineFunction &MF) const { const Function &F = MF.getFunction(); return MF.getTarget().getMCAsmInfo()->usesWindowsCFI() && F.needsUnwindTableEntry(); } -static bool shouldSignReturnAddressEverywhere(const MachineFunction &MF) { +bool AArch64FrameLowering::shouldSignReturnAddressEverywhere( + const MachineFunction &MF) const { // FIXME: With WinCFI, extra care should be taken to place SEH_PACSignLR // and SEH_EpilogEnd instructions in the correct order. if (MF.getTarget().getMCAsmInfo()->usesWindowsCFI()) @@ -1475,13 +1397,13 @@ static void fixupSEHOpcode(MachineBasicBlock::iterator MBBI, ImmOpnd->setImm(ImmOpnd->getImm() + LocalStackSize); } -bool requiresGetVGCall(MachineFunction &MF) { - AArch64FunctionInfo *AFI = MF.getInfo<AArch64FunctionInfo>(); +bool AArch64FrameLowering::requiresGetVGCall(const MachineFunction &MF) const { + auto *AFI = MF.getInfo<AArch64FunctionInfo>(); return AFI->hasStreamingModeChanges() && !MF.getSubtarget<AArch64Subtarget>().hasSVE(); } -static bool requiresSaveVG(const MachineFunction &MF) { +bool AArch64FrameLowering::requiresSaveVG(const MachineFunction &MF) const { const AArch64FunctionInfo *AFI = MF.getInfo<AArch64FunctionInfo>(); if (!AFI->needsDwarfUnwindInfo(MF) || !AFI->hasStreamingModeChanges()) return false; @@ -1499,8 +1421,8 @@ static bool matchLibcall(const TargetLowering &TLI, const MachineOperand &MO, StringRef(TLI.getLibcallName(LC)) == MO.getSymbolName(); } -bool isVGInstruction(MachineBasicBlock::iterator MBBI, - const TargetLowering &TLI) { +bool AArch64FrameLowering::isVGInstruction(MachineBasicBlock::iterator MBBI, + const TargetLowering &TLI) const { unsigned Opc = MBBI->getOpcode(); if (Opc == AArch64::CNTD_XPiI) return true; @@ -1514,15 +1436,12 @@ bool isVGInstruction(MachineBasicBlock::iterator MBBI, return Opc == TargetOpcode::COPY; } -// Convert callee-save register save/restore instruction to do stack pointer -// decrement/increment to allocate/deallocate the callee-save stack area by -// converting store/load to use pre/post increment version. -static MachineBasicBlock::iterator convertCalleeSaveRestoreToSPPrePostIncDec( +MachineBasicBlock::iterator +AArch64FrameLowering::convertCalleeSaveRestoreToSPPrePostIncDec( MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI, const DebugLoc &DL, const TargetInstrInfo *TII, int CSStackSizeInc, bool NeedsWinCFI, bool *HasWinCFI, bool EmitCFI, - MachineInstr::MIFlag FrameFlag = MachineInstr::FrameSetup, - int CFAOffset = 0) { + MachineInstr::MIFlag FrameFlag, int CFAOffset) const { unsigned NewOpc; // If the function contains streaming mode changes, we expect instructions @@ -1643,12 +1562,9 @@ static MachineBasicBlock::iterator convertCalleeSaveRestoreToSPPrePostIncDec( return std::prev(MBB.erase(MBBI)); } -// Fixup callee-save register save/restore instructions to take into account -// combined SP bump by adding the local stack size to the stack offsets. -static void fixupCalleeSaveRestoreStackOffset(MachineInstr &MI, - uint64_t LocalStackSize, - bool NeedsWinCFI, - bool *HasWinCFI) { +void AArch64FrameLowering::fixupCalleeSaveRestoreStackOffset( + MachineInstr &MI, uint64_t LocalStackSize, bool NeedsWinCFI, + bool *HasWinCFI) const { if (AArch64InstrInfo::isSEHInstruction(MI)) return; @@ -1703,7 +1619,8 @@ static unsigned getStackHazardSize(const MachineFunction &MF) { } // Convenience function to determine whether I is an SVE callee save. -static bool IsSVECalleeSave(MachineBasicBlock::iterator I) { +bool AArch64FrameLowering::isSVECalleeSave( + MachineBasicBlock::iterator I) const { switch (I->getOpcode()) { default: return false; @@ -1725,42 +1642,6 @@ static bool IsSVECalleeSave(MachineBasicBlock::iterator I) { } } -static void emitShadowCallStackPrologue(const TargetInstrInfo &TII, - MachineFunction &MF, - MachineBasicBlock &MBB, - MachineBasicBlock::iterator MBBI, - const DebugLoc &DL, bool NeedsWinCFI, - bool NeedsUnwindInfo) { - // Shadow call stack prolog: str x30, [x18], #8 - BuildMI(MBB, MBBI, DL, TII.get(AArch64::STRXpost)) - .addReg(AArch64::X18, RegState::Define) - .addReg(AArch64::LR) - .addReg(AArch64::X18) - .addImm(8) - .setMIFlag(MachineInstr::FrameSetup); - - // This instruction also makes x18 live-in to the entry block. - MBB.addLiveIn(AArch64::X18); - - if (NeedsWinCFI) - BuildMI(MBB, MBBI, DL, TII.get(AArch64::SEH_Nop)) - .setMIFlag(MachineInstr::FrameSetup); - - if (NeedsUnwindInfo) { - // Emit a CFI instruction that causes 8 to be subtracted from the value of - // x18 when unwinding past this frame. - static const char CFIInst[] = { - dwarf::DW_CFA_val_expression, - 18, // register - 2, // length - static_cast<char>(unsigned(dwarf::DW_OP_breg18)), - static_cast<char>(-8) & 0x7f, // addend (sleb128) - }; - CFIInstBuilder(MBB, MBBI, MachineInstr::FrameSetup) - .buildEscape(StringRef(CFIInst, sizeof(CFIInst))); - } -} - static void emitShadowCallStackEpilogue(const TargetInstrInfo &TII, MachineFunction &MF, MachineBasicBlock &MBB, @@ -1783,36 +1664,6 @@ static void emitShadowCallStackEpilogue(const TargetInstrInfo &TII, .buildRestore(AArch64::X18); } -// Define the current CFA rule to use the provided FP. -static void emitDefineCFAWithFP(MachineFunction &MF, MachineBasicBlock &MBB, - MachineBasicBlock::iterator MBBI, - unsigned FixedObject) { - const AArch64Subtarget &STI = MF.getSubtarget<AArch64Subtarget>(); - const AArch64RegisterInfo *TRI = STI.getRegisterInfo(); - AArch64FunctionInfo *AFI = MF.getInfo<AArch64FunctionInfo>(); - - const int OffsetToFirstCalleeSaveFromFP = - AFI->getCalleeSaveBaseToFrameRecordOffset() - - AFI->getCalleeSavedStackSize(); - Register FramePtr = TRI->getFrameRegister(MF); - CFIInstBuilder(MBB, MBBI, MachineInstr::FrameSetup) - .buildDefCFA(FramePtr, FixedObject - OffsetToFirstCalleeSaveFromFP); -} - -#ifndef NDEBUG -/// Collect live registers from the end of \p MI's parent up to (including) \p -/// MI in \p LiveRegs. -static void getLivePhysRegsUpTo(MachineInstr &MI, const TargetRegisterInfo &TRI, - LivePhysRegs &LiveRegs) { - - MachineBasicBlock &MBB = *MI.getParent(); - LiveRegs.addLiveOuts(MBB); - for (const MachineInstr &MI : - reverse(make_range(MI.getIterator(), MBB.instr_end()))) - LiveRegs.stepBackward(MI); -} -#endif - void AArch64FrameLowering::emitPacRetPlusLeafHardening( MachineFunction &MF) const { const AArch64Subtarget &Subtarget = MF.getSubtarget<AArch64Subtarget>(); @@ -1848,616 +1699,8 @@ void AArch64FrameLowering::emitPacRetPlusLeafHardening( void AArch64FrameLowering::emitPrologue(MachineFunction &MF, MachineBasicBlock &MBB) const { - MachineBasicBlock::iterator MBBI = MBB.begin(); - const MachineFrameInfo &MFI = MF.getFrameInfo(); - const Function &F = MF.getFunction(); - const AArch64Subtarget &Subtarget = MF.getSubtarget<AArch64Subtarget>(); - const AArch64RegisterInfo *RegInfo = Subtarget.getRegisterInfo(); - const TargetInstrInfo *TII = Subtarget.getInstrInfo(); - - AArch64FunctionInfo *AFI = MF.getInfo<AArch64FunctionInfo>(); - bool EmitCFI = AFI->needsDwarfUnwindInfo(MF); - bool EmitAsyncCFI = AFI->needsAsyncDwarfUnwindInfo(MF); - bool HasFP = hasFP(MF); - bool NeedsWinCFI = needsWinCFI(MF); - bool HasWinCFI = false; - auto Cleanup = make_scope_exit([&]() { MF.setHasWinCFI(HasWinCFI); }); - - MachineBasicBlock::iterator End = MBB.end(); -#ifndef NDEBUG - const TargetRegisterInfo *TRI = MF.getSubtarget().getRegisterInfo(); - // Collect live register from the end of MBB up to the start of the existing - // frame setup instructions. - MachineBasicBlock::iterator NonFrameStart = MBB.begin(); - while (NonFrameStart != End && - NonFrameStart->getFlag(MachineInstr::FrameSetup)) - ++NonFrameStart; - - LivePhysRegs LiveRegs(*TRI); - if (NonFrameStart != MBB.end()) { - getLivePhysRegsUpTo(*NonFrameStart, *TRI, LiveRegs); - // Ignore registers used for stack management for now. - LiveRegs.removeReg(AArch64::SP); - LiveRegs.removeReg(AArch64::X19); - LiveRegs.removeReg(AArch64::FP); - LiveRegs.removeReg(AArch64::LR); - - // X0 will be clobbered by a call to __arm_get_current_vg in the prologue. - // This is necessary to spill VG if required where SVE is unavailable, but - // X0 is preserved around this call. - if (requiresGetVGCall(MF)) - LiveRegs.removeReg(AArch64::X0); - } - - auto VerifyClobberOnExit = make_scope_exit([&]() { - if (NonFrameStart == MBB.end()) - return; - // Check if any of the newly instructions clobber any of the live registers. - for (MachineInstr &MI : - make_range(MBB.instr_begin(), NonFrameStart->getIterator())) { - for (auto &Op : MI.operands()) - if (Op.isReg() && Op.isDef()) - assert(!LiveRegs.contains(Op.getReg()) && - "live register clobbered by inserted prologue instructions"); - } - }); -#endif - - bool IsFunclet = MBB.isEHFuncletEntry(); - - // At this point, we're going to decide whether or not the function uses a - // redzone. In most cases, the function doesn't have a redzone so let's - // assume that's false and set it to true in the case that there's a redzone. - AFI->setHasRedZone(false); - - // Debug location must be unknown since the first debug location is used - // to determine the end of the prologue. - DebugLoc DL; - - const auto &MFnI = *MF.getInfo<AArch64FunctionInfo>(); - if (MFnI.shouldSignReturnAddress(MF)) { - // If pac-ret+leaf is in effect, PAUTH_PROLOGUE pseudo instructions - // are inserted by emitPacRetPlusLeafHardening(). - if (!shouldSignReturnAddressEverywhere(MF)) { - BuildMI(MBB, MBBI, DL, TII->get(AArch64::PAUTH_PROLOGUE)) - .setMIFlag(MachineInstr::FrameSetup); - } - // AArch64PointerAuth pass will insert SEH_PACSignLR - HasWinCFI |= NeedsWinCFI; - } - - if (MFnI.needsShadowCallStackPrologueEpilogue(MF)) { - emitShadowCallStackPrologue(*TII, MF, MBB, MBBI, DL, NeedsWinCFI, - MFnI.needsDwarfUnwindInfo(MF)); - HasWinCFI |= NeedsWinCFI; - } - - if (EmitCFI && MFnI.isMTETagged()) { - BuildMI(MBB, MBBI, DL, TII->get(AArch64::EMITMTETAGGED)) - .setMIFlag(MachineInstr::FrameSetup); - } - - // We signal the presence of a Swift extended frame to external tools by - // storing FP with 0b0001 in bits 63:60. In normal userland operation a simple - // ORR is sufficient, it is assumed a Swift kernel would initialize the TBI - // bits so that is still true. - if (HasFP && AFI->hasSwiftAsyncContext()) { - switch (MF.getTarget().Options.SwiftAsyncFramePointer) { - case SwiftAsyncFramePointerMode::DeploymentBased: - if (Subtarget.swiftAsyncContextIsDynamicallySet()) { - // The special symbol below is absolute and has a *value* that can be - // combined with the frame pointer to signal an extended frame. - BuildMI(MBB, MBBI, DL, TII->get(AArch64::LOADgot), AArch64::X16) - .addExternalSymbol("swift_async_extendedFramePointerFlags", - AArch64II::MO_GOT); - if (NeedsWinCFI) { - BuildMI(MBB, MBBI, DL, TII->get(AArch64::SEH_Nop)) - .setMIFlags(MachineInstr::FrameSetup); - HasWinCFI = true; - } - BuildMI(MBB, MBBI, DL, TII->get(AArch64::ORRXrs), AArch64::FP) - .addUse(AArch64::FP) - .addUse(AArch64::X16) - .addImm(Subtarget.isTargetILP32() ? 32 : 0); - if (NeedsWinCFI) { - BuildMI(MBB, MBBI, DL, TII->get(AArch64::SEH_Nop)) - .setMIFlags(MachineInstr::FrameSetup); - HasWinCFI = true; - } - break; - } - [[fallthrough]]; - - case SwiftAsyncFramePointerMode::Always: - // ORR x29, x29, #0x1000_0000_0000_0000 - BuildMI(MBB, MBBI, DL, TII->get(AArch64::ORRXri), AArch64::FP) - .addUse(AArch64::FP) - .addImm(0x1100) - .setMIFlag(MachineInstr::FrameSetup); - if (NeedsWinCFI) { - BuildMI(MBB, MBBI, DL, TII->get(AArch64::SEH_Nop)) - .setMIFlags(MachineInstr::FrameSetup); - HasWinCFI = true; - } - break; - - case SwiftAsyncFramePointerMode::Never: - break; - } - } - - // All calls are tail calls in GHC calling conv, and functions have no - // prologue/epilogue. - if (MF.getFunction().getCallingConv() == CallingConv::GHC) - return; - - // Set tagged base pointer to the requested stack slot. - // Ideally it should match SP value after prologue. - std::optional<int> TBPI = AFI->getTaggedBasePointerIndex(); - if (TBPI) - AFI->setTaggedBasePointerOffset(-MFI.getObjectOffset(*TBPI)); - else - AFI->setTaggedBasePointerOffset(MFI.getStackSize()); - - const StackOffset &SVEStackSize = getSVEStackSize(MF); - - // getStackSize() includes all the locals in its size calculation. We don't - // include these locals when computing the stack size of a funclet, as they - // are allocated in the parent's stack frame and accessed via the frame - // pointer from the funclet. We only save the callee saved registers in the - // funclet, which are really the callee saved registers of the parent - // function, including the funclet. - int64_t NumBytes = - IsFunclet ? getWinEHFuncletFrameSize(MF) : MFI.getStackSize(); - if (!AFI->hasStackFrame() && !windowsRequiresStackProbe(MF, NumBytes)) { - assert(!HasFP && "unexpected function without stack frame but with FP"); - assert(!SVEStackSize && - "unexpected function without stack frame but with SVE objects"); - // All of the stack allocation is for locals. - AFI->setLocalStackSize(NumBytes); - if (!NumBytes) { - if (NeedsWinCFI && HasWinCFI) { - BuildMI(MBB, MBBI, DL, TII->get(AArch64::SEH_PrologEnd)) - .setMIFlag(MachineInstr::FrameSetup); - } - return; - } - // REDZONE: If the stack size is less than 128 bytes, we don't need - // to actually allocate. - if (canUseRedZone(MF)) { - AFI->setHasRedZone(true); - ++NumRedZoneFunctions; - } else { - emitFrameOffset(MBB, MBBI, DL, AArch64::SP, AArch64::SP, - StackOffset::getFixed(-NumBytes), TII, - MachineInstr::FrameSetup, false, NeedsWinCFI, &HasWinCFI); - if (EmitCFI) { - // Label used to tie together the PROLOG_LABEL and the MachineMoves. - MCSymbol *FrameLabel = MF.getContext().createTempSymbol(); - // Encode the stack size of the leaf function. - CFIInstBuilder(MBB, MBBI, MachineInstr::FrameSetup) - .buildDefCFAOffset(NumBytes, FrameLabel); - } - } - - if (NeedsWinCFI) { - HasWinCFI = true; - BuildMI(MBB, MBBI, DL, TII->get(AArch64::SEH_PrologEnd)) - .setMIFlag(MachineInstr::FrameSetup); - } - - return; - } - - bool IsWin64 = Subtarget.isCallingConvWin64(F.getCallingConv(), F.isVarArg()); - unsigned FixedObject = getFixedObjectSize(MF, AFI, IsWin64, IsFunclet); - - // Windows unwind can't represent the required stack adjustments if we have - // both SVE callee-saves and dynamic stack allocations, and the frame - // pointer is before the SVE spills. The allocation of the frame pointer - // must be the last instruction in the prologue so the unwinder can restore - // the stack pointer correctly. (And there isn't any unwind opcode for - // `addvl sp, x29, -17`.) - // - // Because of this, we do spills in the opposite order on Windows: first SVE, - // then GPRs. The main side-effect of this is that it makes accessing - // parameters passed on the stack more expensive. - // - // We could consider rearranging the spills for simpler cases. - bool FPAfterSVECalleeSaves = - Subtarget.isTargetWindows() && AFI->getSVECalleeSavedStackSize(); - - if (FPAfterSVECalleeSaves && AFI->hasStackHazardSlotIndex()) - reportFatalUsageError("SME hazard padding is not supported on Windows"); - - auto PrologueSaveSize = AFI->getCalleeSavedStackSize() + FixedObject; - // All of the remaining stack allocations are for locals. - AFI->setLocalStackSize(NumBytes - PrologueSaveSize); - bool CombineSPBump = shouldCombineCSRLocalStackBump(MF, NumBytes); - bool HomPrologEpilog = homogeneousPrologEpilog(MF); - if (FPAfterSVECalleeSaves) { - // If we're doing SVE saves first, we need to immediately allocate space - // for fixed objects, then space for the SVE callee saves. - // - // Windows unwind requires that the scalable size is a multiple of 16; - // that's handled when the callee-saved size is computed. - auto SaveSize = - StackOffset::getScalable(AFI->getSVECalleeSavedStackSize()) + - StackOffset::getFixed(FixedObject); - allocateStackSpace(MBB, MBBI, 0, SaveSize, NeedsWinCFI, &HasWinCFI, - /*EmitCFI=*/false, StackOffset{}, - /*FollowupAllocs=*/true); - NumBytes -= FixedObject; - - // Now allocate space for the GPR callee saves. - while (MBBI != End && IsSVECalleeSave(MBBI)) - ++MBBI; - MBBI = convertCalleeSaveRestoreToSPPrePostIncDec( - MBB, MBBI, DL, TII, -AFI->getCalleeSavedStackSize(), NeedsWinCFI, - &HasWinCFI, EmitAsyncCFI); - NumBytes -= AFI->getCalleeSavedStackSize(); - } else if (CombineSPBump) { - assert(!SVEStackSize && "Cannot combine SP bump with SVE"); - emitFrameOffset(MBB, MBBI, DL, AArch64::SP, AArch64::SP, - StackOffset::getFixed(-NumBytes), TII, - MachineInstr::FrameSetup, false, NeedsWinCFI, &HasWinCFI, - EmitAsyncCFI); - NumBytes = 0; - } else if (HomPrologEpilog) { - // Stack has been already adjusted. - NumBytes -= PrologueSaveSize; - } else if (PrologueSaveSize != 0) { - MBBI = convertCalleeSaveRestoreToSPPrePostIncDec( - MBB, MBBI, DL, TII, -PrologueSaveSize, NeedsWinCFI, &HasWinCFI, - EmitAsyncCFI); - NumBytes -= PrologueSaveSize; - } - assert(NumBytes >= 0 && "Negative stack allocation size!?"); - - // Move past the saves of the callee-saved registers, fixing up the offsets - // and pre-inc if we decided to combine the callee-save and local stack - // pointer bump above. - auto &TLI = *MF.getSubtarget().getTargetLowering(); - while (MBBI != End && MBBI->getFlag(MachineInstr::FrameSetup) && - !IsSVECalleeSave(MBBI)) { - if (CombineSPBump && - // Only fix-up frame-setup load/store instructions. - (!requiresSaveVG(MF) || !isVGInstruction(MBBI, TLI))) - fixupCalleeSaveRestoreStackOffset(*MBBI, AFI->getLocalStackSize(), - NeedsWinCFI, &HasWinCFI); - ++MBBI; - } - - // For funclets the FP belongs to the containing function. - if (!IsFunclet && HasFP) { - // Only set up FP if we actually need to. - int64_t FPOffset = AFI->getCalleeSaveBaseToFrameRecordOffset(); - - if (CombineSPBump) - FPOffset += AFI->getLocalStackSize(); - - if (AFI->hasSwiftAsyncContext()) { - // Before we update the live FP we have to ensure there's a valid (or - // null) asynchronous context in its slot just before FP in the frame - // record, so store it now. - const auto &Attrs = MF.getFunction().getAttributes(); - bool HaveInitialContext = Attrs.hasAttrSomewhere(Attribute::SwiftAsync); - if (HaveInitialContext) - MBB.addLiveIn(AArch64::X22); - Register Reg = HaveInitialContext ? AArch64::X22 : AArch64::XZR; - BuildMI(MBB, MBBI, DL, TII->get(AArch64::StoreSwiftAsyncContext)) - .addUse(Reg) - .addUse(AArch64::SP) - .addImm(FPOffset - 8) - .setMIFlags(MachineInstr::FrameSetup); - if (NeedsWinCFI) { - // WinCFI and arm64e, where StoreSwiftAsyncContext is expanded - // to multiple instructions, should be mutually-exclusive. - assert(Subtarget.getTargetTriple().getArchName() != "arm64e"); - BuildMI(MBB, MBBI, DL, TII->get(AArch64::SEH_Nop)) - .setMIFlags(MachineInstr::FrameSetup); - HasWinCFI = true; - } - } - - if (HomPrologEpilog) { - auto Prolog = MBBI; - --Prolog; - assert(Prolog->getOpcode() == AArch64::HOM_Prolog); - Prolog->addOperand(MachineOperand::CreateImm(FPOffset)); - } else { - // Issue sub fp, sp, FPOffset or - // mov fp,sp when FPOffset is zero. - // Note: All stores of callee-saved registers are marked as "FrameSetup". - // This code marks the instruction(s) that set the FP also. - emitFrameOffset(MBB, MBBI, DL, AArch64::FP, AArch64::SP, - StackOffset::getFixed(FPOffset), TII, - MachineInstr::FrameSetup, false, NeedsWinCFI, &HasWinCFI); - if (NeedsWinCFI && HasWinCFI) { - BuildMI(MBB, MBBI, DL, TII->get(AArch64::SEH_PrologEnd)) - .setMIFlag(MachineInstr::FrameSetup); - // After setting up the FP, the rest of the prolog doesn't need to be - // included in the SEH unwind info. - NeedsWinCFI = false; - } - } - if (EmitAsyncCFI) - emitDefineCFAWithFP(MF, MBB, MBBI, FixedObject); - } - - // Now emit the moves for whatever callee saved regs we have (including FP, - // LR if those are saved). Frame instructions for SVE register are emitted - // later, after the instruction which actually save SVE regs. - if (EmitAsyncCFI) - emitCalleeSavedGPRLocations(MBB, MBBI); - - // Alignment is required for the parent frame, not the funclet - const bool NeedsRealignment = - NumBytes && !IsFunclet && RegInfo->hasStackRealignment(MF); - const int64_t RealignmentPadding = - (NeedsRealignment && MFI.getMaxAlign() > Align(16)) - ? MFI.getMaxAlign().value() - 16 - : 0; - - if (windowsRequiresStackProbe(MF, NumBytes + RealignmentPadding)) { - if (AFI->getSVECalleeSavedStackSize()) - report_fatal_error( - "SVE callee saves not yet supported with stack probing"); - - // Find an available register to spill the value of X15 to, if X15 is being - // used already for nest. - unsigned X15Scratch = AArch64::NoRegister; - const AArch64Subtarget &STI = MF.getSubtarget<AArch64Subtarget>(); - if (llvm::any_of(MBB.liveins(), - [&STI](const MachineBasicBlock::RegisterMaskPair &LiveIn) { - return STI.getRegisterInfo()->isSuperOrSubRegisterEq( - AArch64::X15, LiveIn.PhysReg); - })) { - X15Scratch = findScratchNonCalleeSaveRegister(&MBB, true); - assert(X15Scratch != AArch64::NoRegister && - (X15Scratch < AArch64::X15 || X15Scratch > AArch64::X17)); -#ifndef NDEBUG - LiveRegs.removeReg(AArch64::X15); // ignore X15 since we restore it -#endif - BuildMI(MBB, MBBI, DL, TII->get(AArch64::ORRXrr), X15Scratch) - .addReg(AArch64::XZR) - .addReg(AArch64::X15, RegState::Undef) - .addReg(AArch64::X15, RegState::Implicit) - .setMIFlag(MachineInstr::FrameSetup); - } - - uint64_t NumWords = (NumBytes + RealignmentPadding) >> 4; - if (NeedsWinCFI) { - HasWinCFI = true; - // alloc_l can hold at most 256MB, so assume that NumBytes doesn't - // exceed this amount. We need to move at most 2^24 - 1 into x15. - // This is at most two instructions, MOVZ followed by MOVK. - // TODO: Fix to use multiple stack alloc unwind codes for stacks - // exceeding 256MB in size. - if (NumBytes >= (1 << 28)) - report_fatal_error("Stack size cannot exceed 256MB for stack " - "unwinding purposes"); - - uint32_t LowNumWords = NumWords & 0xFFFF; - BuildMI(MBB, MBBI, DL, TII->get(AArch64::MOVZXi), AArch64::X15) - .addImm(LowNumWords) - .addImm(AArch64_AM::getShifterImm(AArch64_AM::LSL, 0)) - .setMIFlag(MachineInstr::FrameSetup); - BuildMI(MBB, MBBI, DL, TII->get(AArch64::SEH_Nop)) - .setMIFlag(MachineInstr::FrameSetup); - if ((NumWords & 0xFFFF0000) != 0) { - BuildMI(MBB, MBBI, DL, TII->get(AArch64::MOVKXi), AArch64::X15) - .addReg(AArch64::X15) - .addImm((NumWords & 0xFFFF0000) >> 16) // High half - .addImm(AArch64_AM::getShifterImm(AArch64_AM::LSL, 16)) - .setMIFlag(MachineInstr::FrameSetup); - BuildMI(MBB, MBBI, DL, TII->get(AArch64::SEH_Nop)) - .setMIFlag(MachineInstr::FrameSetup); - } - } else { - BuildMI(MBB, MBBI, DL, TII->get(AArch64::MOVi64imm), AArch64::X15) - .addImm(NumWords) - .setMIFlags(MachineInstr::FrameSetup); - } - - const char *ChkStk = Subtarget.getChkStkName(); - switch (MF.getTarget().getCodeModel()) { - case CodeModel::Tiny: - case CodeModel::Small: - case CodeModel::Medium: - case CodeModel::Kernel: - BuildMI(MBB, MBBI, DL, TII->get(AArch64::BL)) - .addExternalSymbol(ChkStk) - .addReg(AArch64::X15, RegState::Implicit) - .addReg(AArch64::X16, RegState::Implicit | RegState::Define | RegState::Dead) - .addReg(AArch64::X17, RegState::Implicit | RegState::Define | RegState::Dead) - .addReg(AArch64::NZCV, RegState::Implicit | RegState::Define | RegState::Dead) - .setMIFlags(MachineInstr::FrameSetup); - if (NeedsWinCFI) { - HasWinCFI = true; - BuildMI(MBB, MBBI, DL, TII->get(AArch64::SEH_Nop)) - .setMIFlag(MachineInstr::FrameSetup); - } - break; - case CodeModel::Large: - BuildMI(MBB, MBBI, DL, TII->get(AArch64::MOVaddrEXT)) - .addReg(AArch64::X16, RegState::Define) - .addExternalSymbol(ChkStk) - .addExternalSymbol(ChkStk) - .setMIFlags(MachineInstr::FrameSetup); - if (NeedsWinCFI) { - HasWinCFI = true; - BuildMI(MBB, MBBI, DL, TII->get(AArch64::SEH_Nop)) - .setMIFlag(MachineInstr::FrameSetup); - } - - BuildMI(MBB, MBBI, DL, TII->get(getBLRCallOpcode(MF))) - .addReg(AArch64::X16, RegState::Kill) - .addReg(AArch64::X15, RegState::Implicit | RegState::Define) - .addReg(AArch64::X16, RegState::Implicit | RegState::Define | RegState::Dead) - .addReg(AArch64::X17, RegState::Implicit | RegState::Define | RegState::Dead) - .addReg(AArch64::NZCV, RegState::Implicit | RegState::Define | RegState::Dead) - .setMIFlags(MachineInstr::FrameSetup); - if (NeedsWinCFI) { - HasWinCFI = true; - BuildMI(MBB, MBBI, DL, TII->get(AArch64::SEH_Nop)) - .setMIFlag(MachineInstr::FrameSetup); - } - break; - } - - BuildMI(MBB, MBBI, DL, TII->get(AArch64::SUBXrx64), AArch64::SP) - .addReg(AArch64::SP, RegState::Kill) - .addReg(AArch64::X15, RegState::Kill) - .addImm(AArch64_AM::getArithExtendImm(AArch64_AM::UXTX, 4)) - .setMIFlags(MachineInstr::FrameSetup); - if (NeedsWinCFI) { - HasWinCFI = true; - BuildMI(MBB, MBBI, DL, TII->get(AArch64::SEH_StackAlloc)) - .addImm(NumBytes) - .setMIFlag(MachineInstr::FrameSetup); - } - NumBytes = 0; - - if (RealignmentPadding > 0) { - if (RealignmentPadding >= 4096) { - BuildMI(MBB, MBBI, DL, TII->get(AArch64::MOVi64imm)) - .addReg(AArch64::X16, RegState::Define) - .addImm(RealignmentPadding) - .setMIFlags(MachineInstr::FrameSetup); - BuildMI(MBB, MBBI, DL, TII->get(AArch64::ADDXrx64), AArch64::X15) - .addReg(AArch64::SP) - .addReg(AArch64::X16, RegState::Kill) - .addImm(AArch64_AM::getArithExtendImm(AArch64_AM::UXTX, 0)) - .setMIFlag(MachineInstr::FrameSetup); - } else { - BuildMI(MBB, MBBI, DL, TII->get(AArch64::ADDXri), AArch64::X15) - .addReg(AArch64::SP) - .addImm(RealignmentPadding) - .addImm(0) - .setMIFlag(MachineInstr::FrameSetup); - } - - uint64_t AndMask = ~(MFI.getMaxAlign().value() - 1); - BuildMI(MBB, MBBI, DL, TII->get(AArch64::ANDXri), AArch64::SP) - .addReg(AArch64::X15, RegState::Kill) - .addImm(AArch64_AM::encodeLogicalImmediate(AndMask, 64)); - AFI->setStackRealigned(true); - - // No need for SEH instructions here; if we're realigning the stack, - // we've set a frame pointer and already finished the SEH prologue. - assert(!NeedsWinCFI); - } - if (X15Scratch != AArch64::NoRegister) { - BuildMI(MBB, MBBI, DL, TII->get(AArch64::ORRXrr), AArch64::X15) - .addReg(AArch64::XZR) - .addReg(X15Scratch, RegState::Undef) - .addReg(X15Scratch, RegState::Implicit) - .setMIFlag(MachineInstr::FrameSetup); - } - } - - StackOffset SVECalleeSavesSize = {}, SVELocalsSize = SVEStackSize; - MachineBasicBlock::iterator CalleeSavesEnd = MBBI; - - StackOffset CFAOffset = - StackOffset::getFixed((int64_t)MFI.getStackSize() - NumBytes); - - // Process the SVE callee-saves to determine what space needs to be - // allocated. - if (int64_t CalleeSavedSize = AFI->getSVECalleeSavedStackSize()) { - LLVM_DEBUG(dbgs() << "SVECalleeSavedStackSize = " << CalleeSavedSize - << "\n"); - SVECalleeSavesSize = StackOffset::getScalable(CalleeSavedSize); - SVELocalsSize = SVEStackSize - SVECalleeSavesSize; - // Find callee save instructions in frame. - // Note: With FPAfterSVECalleeSaves the callee saves have already been - // allocated. - if (!FPAfterSVECalleeSaves) { - MachineBasicBlock::iterator CalleeSavesBegin = MBBI; - assert(IsSVECalleeSave(CalleeSavesBegin) && "Unexpected instruction"); - while (IsSVECalleeSave(MBBI) && MBBI != MBB.getFirstTerminator()) - ++MBBI; - CalleeSavesEnd = MBBI; - - StackOffset LocalsSize = SVELocalsSize + StackOffset::getFixed(NumBytes); - // Allocate space for the callee saves (if any). - allocateStackSpace(MBB, CalleeSavesBegin, 0, SVECalleeSavesSize, false, - nullptr, EmitAsyncCFI && !HasFP, CFAOffset, - MFI.hasVarSizedObjects() || LocalsSize); - } - } - CFAOffset += SVECalleeSavesSize; - - if (EmitAsyncCFI) - emitCalleeSavedSVELocations(MBB, CalleeSavesEnd); - - // Allocate space for the rest of the frame including SVE locals. Align the - // stack as necessary. - assert(!(canUseRedZone(MF) && NeedsRealignment) && - "Cannot use redzone with stack realignment"); - if (!canUseRedZone(MF)) { - // FIXME: in the case of dynamic re-alignment, NumBytes doesn't have - // the correct value here, as NumBytes also includes padding bytes, - // which shouldn't be counted here. - allocateStackSpace(MBB, CalleeSavesEnd, RealignmentPadding, - SVELocalsSize + StackOffset::getFixed(NumBytes), - NeedsWinCFI, &HasWinCFI, EmitAsyncCFI && !HasFP, - CFAOffset, MFI.hasVarSizedObjects()); - } - - // If we need a base pointer, set it up here. It's whatever the value of the - // stack pointer is at this point. Any variable size objects will be allocated - // after this, so we can still use the base pointer to reference locals. - // - // FIXME: Clarify FrameSetup flags here. - // Note: Use emitFrameOffset() like above for FP if the FrameSetup flag is - // needed. - // For funclets the BP belongs to the containing function. - if (!IsFunclet && RegInfo->hasBasePointer(MF)) { - TII->copyPhysReg(MBB, MBBI, DL, RegInfo->getBaseRegister(), AArch64::SP, - false); - if (NeedsWinCFI) { - HasWinCFI = true; - BuildMI(MBB, MBBI, DL, TII->get(AArch64::SEH_Nop)) - .setMIFlag(MachineInstr::FrameSetup); - } - } - - // The very last FrameSetup instruction indicates the end of prologue. Emit a - // SEH opcode indicating the prologue end. - if (NeedsWinCFI && HasWinCFI) { - BuildMI(MBB, MBBI, DL, TII->get(AArch64::SEH_PrologEnd)) - .setMIFlag(MachineInstr::FrameSetup); - } - - // SEH funclets are passed the frame pointer in X1. If the parent - // function uses the base register, then the base register is used - // directly, and is not retrieved from X1. - if (IsFunclet && F.hasPersonalityFn()) { - EHPersonality Per = classifyEHPersonality(F.getPersonalityFn()); - if (isAsynchronousEHPersonality(Per)) { - BuildMI(MBB, MBBI, DL, TII->get(TargetOpcode::COPY), AArch64::FP) - .addReg(AArch64::X1) - .setMIFlag(MachineInstr::FrameSetup); - MBB.addLiveIn(AArch64::X1); - } - } - - if (EmitCFI && !EmitAsyncCFI) { - if (HasFP) { - emitDefineCFAWithFP(MF, MBB, MBBI, FixedObject); - } else { - StackOffset TotalSize = - SVEStackSize + StackOffset::getFixed((int64_t)MFI.getStackSize()); - CFIInstBuilder CFIBuilder(MBB, MBBI, MachineInstr::FrameSetup); - CFIBuilder.insertCFIInst( - createDefCFA(*RegInfo, /*FrameReg=*/AArch64::SP, /*Reg=*/AArch64::SP, - TotalSize, /*LastAdjustmentWasScalable=*/false)); - } - emitCalleeSavedGPRLocations(MBB, MBBI); - emitCalleeSavedSVELocations(MBB, MBBI); - } + AArch64PrologueEmitter PrologueEmitter(MF, MBB, *this); + PrologueEmitter.emitPrologue(); } static bool isFuncletReturnInstr(const MachineInstr &MI) { @@ -2548,15 +1791,15 @@ void AArch64FrameLowering::emitEpilogue(MachineFunction &MF, AFI->setLocalStackSize(NumBytes - PrologueSaveSize); if (homogeneousPrologEpilog(MF, &MBB)) { assert(!NeedsWinCFI); - auto LastPopI = MBB.getFirstTerminator(); - if (LastPopI != MBB.begin()) { - auto HomogeneousEpilog = std::prev(LastPopI); + auto FirstHomogenousEpilogI = MBB.getFirstTerminator(); + if (FirstHomogenousEpilogI != MBB.begin()) { + auto HomogeneousEpilog = std::prev(FirstHomogenousEpilogI); if (HomogeneousEpilog->getOpcode() == AArch64::HOM_Epilog) - LastPopI = HomogeneousEpilog; + FirstHomogenousEpilogI = HomogeneousEpilog; } // Adjust local stack - emitFrameOffset(MBB, LastPopI, DL, AArch64::SP, AArch64::SP, + emitFrameOffset(MBB, FirstHomogenousEpilogI, DL, AArch64::SP, AArch64::SP, StackOffset::getFixed(AFI->getLocalStackSize()), TII, MachineInstr::FrameDestroy, false, NeedsWinCFI, &HasWinCFI); @@ -2602,17 +1845,17 @@ void AArch64FrameLowering::emitEpilogue(MachineFunction &MF, // Move past the restores of the callee-saved registers. // If we plan on combining the sp bump of the local stack size and the callee // save stack size, we might need to adjust the CSR save and restore offsets. - MachineBasicBlock::iterator LastPopI = MBB.getFirstTerminator(); + MachineBasicBlock::iterator FirstGPRRestoreI = MBB.getFirstTerminator(); MachineBasicBlock::iterator Begin = MBB.begin(); - while (LastPopI != Begin) { - --LastPopI; - if (!LastPopI->getFlag(MachineInstr::FrameDestroy) || - (!FPAfterSVECalleeSaves && IsSVECalleeSave(LastPopI))) { - ++LastPopI; + while (FirstGPRRestoreI != Begin) { + --FirstGPRRestoreI; + if (!FirstGPRRestoreI->getFlag(MachineInstr::FrameDestroy) || + (!FPAfterSVECalleeSaves && isSVECalleeSave(FirstGPRRestoreI))) { + ++FirstGPRRestoreI; break; } else if (CombineSPBump) - fixupCalleeSaveRestoreStackOffset(*LastPopI, AFI->getLocalStackSize(), - NeedsWinCFI, &HasWinCFI); + fixupCalleeSaveRestoreStackOffset( + *FirstGPRRestoreI, AFI->getLocalStackSize(), NeedsWinCFI, &HasWinCFI); } if (NeedsWinCFI) { @@ -2622,9 +1865,9 @@ void AArch64FrameLowering::emitEpilogue(MachineFunction &MF, // arguments. Insert the SEH_EpilogStart and remove it later if it // we didn't emit any SEH opcodes to avoid generating WinCFI for // functions that don't need it. - BuildMI(MBB, LastPopI, DL, TII->get(AArch64::SEH_EpilogStart)) + BuildMI(MBB, FirstGPRRestoreI, DL, TII->get(AArch64::SEH_EpilogStart)) .setMIFlag(MachineInstr::FrameDestroy); - EpilogStartI = LastPopI; + EpilogStartI = FirstGPRRestoreI; --EpilogStartI; } @@ -2665,7 +1908,7 @@ void AArch64FrameLowering::emitEpilogue(MachineFunction &MF, // When we are about to restore the CSRs, the CFA register is SP again. if (EmitCFI && hasFP(MF)) - CFIInstBuilder(MBB, LastPopI, MachineInstr::FrameDestroy) + CFIInstBuilder(MBB, FirstGPRRestoreI, MachineInstr::FrameDestroy) .buildDefCFA(AArch64::SP, NumBytes); emitFrameOffset(MBB, MBB.getFirstTerminator(), DL, AArch64::SP, AArch64::SP, @@ -2681,18 +1924,19 @@ void AArch64FrameLowering::emitEpilogue(MachineFunction &MF, // Process the SVE callee-saves to determine what space needs to be // deallocated. StackOffset DeallocateBefore = {}, DeallocateAfter = SVEStackSize; - MachineBasicBlock::iterator RestoreBegin = LastPopI, RestoreEnd = LastPopI; + MachineBasicBlock::iterator RestoreBegin = FirstGPRRestoreI, + RestoreEnd = FirstGPRRestoreI; if (int64_t CalleeSavedSize = AFI->getSVECalleeSavedStackSize()) { if (FPAfterSVECalleeSaves) RestoreEnd = MBB.getFirstTerminator(); RestoreBegin = std::prev(RestoreEnd); while (RestoreBegin != MBB.begin() && - IsSVECalleeSave(std::prev(RestoreBegin))) + isSVECalleeSave(std::prev(RestoreBegin))) --RestoreBegin; - assert(IsSVECalleeSave(RestoreBegin) && - IsSVECalleeSave(std::prev(RestoreEnd)) && "Unexpected instruction"); + assert(isSVECalleeSave(RestoreBegin) && + isSVECalleeSave(std::prev(RestoreEnd)) && "Unexpected instruction"); StackOffset CalleeSavedSizeAsOffset = StackOffset::getScalable(CalleeSavedSize); @@ -2706,7 +1950,7 @@ void AArch64FrameLowering::emitEpilogue(MachineFunction &MF, // deallocates non-callee-save SVE allocations. Otherwise, deallocate // them explicitly. if (!AFI->isStackRealigned() && !MFI.hasVarSizedObjects()) { - emitFrameOffset(MBB, LastPopI, DL, AArch64::SP, AArch64::SP, + emitFrameOffset(MBB, FirstGPRRestoreI, DL, AArch64::SP, AArch64::SP, DeallocateBefore, TII, MachineInstr::FrameDestroy, false, NeedsWinCFI, &HasWinCFI); } @@ -2796,7 +2040,7 @@ void AArch64FrameLowering::emitEpilogue(MachineFunction &MF, StackRestoreBytes += AfterCSRPopSize; emitFrameOffset( - MBB, LastPopI, DL, AArch64::SP, AArch64::SP, + MBB, FirstGPRRestoreI, DL, AArch64::SP, AArch64::SP, StackOffset::getFixed(StackRestoreBytes), TII, MachineInstr::FrameDestroy, false, NeedsWinCFI, &HasWinCFI, EmitCFI, StackOffset::getFixed((RedZone ? 0 : NumBytes) + PrologueSaveSize)); @@ -2816,17 +2060,17 @@ void AArch64FrameLowering::emitEpilogue(MachineFunction &MF, // be able to save any instructions. if (!IsFunclet && (MFI.hasVarSizedObjects() || AFI->isStackRealigned())) { emitFrameOffset( - MBB, LastPopI, DL, AArch64::SP, AArch64::FP, + MBB, FirstGPRRestoreI, DL, AArch64::SP, AArch64::FP, StackOffset::getFixed(-AFI->getCalleeSaveBaseToFrameRecordOffset()), TII, MachineInstr::FrameDestroy, false, NeedsWinCFI, &HasWinCFI); } else if (NumBytes) - emitFrameOffset(MBB, LastPopI, DL, AArch64::SP, AArch64::SP, + emitFrameOffset(MBB, FirstGPRRestoreI, DL, AArch64::SP, AArch64::SP, StackOffset::getFixed(NumBytes), TII, MachineInstr::FrameDestroy, false, NeedsWinCFI, &HasWinCFI); // When we are about to restore the CSRs, the CFA register is SP again. if (EmitCFI && hasFP(MF)) - CFIInstBuilder(MBB, LastPopI, MachineInstr::FrameDestroy) + CFIInstBuilder(MBB, FirstGPRRestoreI, MachineInstr::FrameDestroy) .buildDefCFA(AArch64::SP, PrologueSaveSize); // This must be placed after the callee-save restore code because that code @@ -2926,8 +2170,8 @@ AArch64FrameLowering::getNonLocalFrameIndexReference(const MachineFunction &MF, return StackOffset::getFixed(getSEHFrameIndexOffset(MF, FI)); } -static StackOffset getFPOffset(const MachineFunction &MF, - int64_t ObjectOffset) { +StackOffset AArch64FrameLowering::getFPOffset(const MachineFunction &MF, + int64_t ObjectOffset) const { const auto *AFI = MF.getInfo<AArch64FunctionInfo>(); const auto &Subtarget = MF.getSubtarget<AArch64Subtarget>(); const Function &F = MF.getFunction(); @@ -2940,8 +2184,8 @@ static StackOffset getFPOffset(const MachineFunction &MF, return StackOffset::getFixed(ObjectOffset + FixedObject + FPAdjust); } -static StackOffset getStackOffset(const MachineFunction &MF, - int64_t ObjectOffset) { +StackOffset AArch64FrameLowering::getStackOffset(const MachineFunction &MF, + int64_t ObjectOffset) const { const auto &MFI = MF.getFrameInfo(); return StackOffset::getFixed(ObjectOffset + (int64_t)MFI.getStackSize()); } @@ -3139,7 +2383,8 @@ static unsigned getPrologueDeath(MachineFunction &MF, unsigned Reg) { return getKillRegState(!IsLiveIn); } -static bool produceCompactUnwindFrame(MachineFunction &MF) { +static bool produceCompactUnwindFrame(const AArch64FrameLowering &AFL, + MachineFunction &MF) { const AArch64Subtarget &Subtarget = MF.getSubtarget<AArch64Subtarget>(); AttributeList Attrs = MF.getFunction().getAttributes(); AArch64FunctionInfo *AFI = MF.getInfo<AArch64FunctionInfo>(); @@ -3147,7 +2392,7 @@ static bool produceCompactUnwindFrame(MachineFunction &MF) { !(Subtarget.getTargetLowering()->supportSwiftError() && Attrs.hasAttrSomewhere(Attribute::SwiftError)) && MF.getFunction().getCallingConv() != CallingConv::SwiftTail && - !requiresSaveVG(MF) && !AFI->isSVECC(); + !AFL.requiresSaveVG(MF) && !AFI->isSVECC(); } static bool invalidateWindowsRegisterPairing(unsigned Reg1, unsigned Reg2, @@ -3244,16 +2489,18 @@ bool enableMultiVectorSpillFill(const AArch64Subtarget &Subtarget, (!IsLocallyStreaming && Subtarget.isStreaming())); } -static void computeCalleeSaveRegisterPairs( - MachineFunction &MF, ArrayRef<CalleeSavedInfo> CSI, - const TargetRegisterInfo *TRI, SmallVectorImpl<RegPairInfo> &RegPairs, - bool NeedsFrameRecord) { +void computeCalleeSaveRegisterPairs(const AArch64FrameLowering &AFL, + MachineFunction &MF, + ArrayRef<CalleeSavedInfo> CSI, + const TargetRegisterInfo *TRI, + SmallVectorImpl<RegPairInfo> &RegPairs, + bool NeedsFrameRecord) { if (CSI.empty()) return; bool IsWindows = isTargetWindows(MF); - bool NeedsWinCFI = needsWinCFI(MF); + bool NeedsWinCFI = AFL.needsWinCFI(MF); AArch64FunctionInfo *AFI = MF.getInfo<AArch64FunctionInfo>(); unsigned StackHazardSize = getStackHazardSize(MF); MachineFrameInfo &MFI = MF.getFrameInfo(); @@ -3262,9 +2509,10 @@ static void computeCalleeSaveRegisterPairs( (void)CC; // MachO's compact unwind format relies on all registers being stored in // pairs. - assert((!produceCompactUnwindFrame(MF) || CC == CallingConv::PreserveMost || - CC == CallingConv::PreserveAll || CC == CallingConv::CXX_FAST_TLS || - CC == CallingConv::Win64 || (Count & 1) == 0) && + assert((!produceCompactUnwindFrame(AFL, MF) || + CC == CallingConv::PreserveMost || CC == CallingConv::PreserveAll || + CC == CallingConv::CXX_FAST_TLS || CC == CallingConv::Win64 || + (Count & 1) == 0) && "Odd number of callee-saved regs to spill!"); int ByteOffset = AFI->getCalleeSavedStackSize(); int StackFillDir = -1; @@ -3380,9 +2628,9 @@ static void computeCalleeSaveRegisterPairs( // MachO's compact unwind format relies on all registers being stored in // adjacent register pairs. - assert((!produceCompactUnwindFrame(MF) || CC == CallingConv::PreserveMost || - CC == CallingConv::PreserveAll || CC == CallingConv::CXX_FAST_TLS || - CC == CallingConv::Win64 || + assert((!produceCompactUnwindFrame(AFL, MF) || + CC == CallingConv::PreserveMost || CC == CallingConv::PreserveAll || + CC == CallingConv::CXX_FAST_TLS || CC == CallingConv::Win64 || (RPI.isPaired() && ((RPI.Reg1 == AArch64::LR && RPI.Reg2 == AArch64::FP) || RPI.Reg1 + 1 == RPI.Reg2))) && @@ -3495,7 +2743,7 @@ bool AArch64FrameLowering::spillCalleeSavedRegisters( DebugLoc DL; SmallVector<RegPairInfo, 8> RegPairs; - computeCalleeSaveRegisterPairs(MF, CSI, TRI, RegPairs, hasFP(MF)); + computeCalleeSaveRegisterPairs(*this, MF, CSI, TRI, RegPairs, hasFP(MF)); MachineRegisterInfo &MRI = MF.getRegInfo(); // Refresh the reserved regs in case there are any potential changes since the @@ -3707,7 +2955,7 @@ bool AArch64FrameLowering::restoreCalleeSavedRegisters( if (MBBI != MBB.end()) DL = MBBI->getDebugLoc(); - computeCalleeSaveRegisterPairs(MF, CSI, TRI, RegPairs, hasFP(MF)); + computeCalleeSaveRegisterPairs(*this, MF, CSI, TRI, RegPairs, hasFP(MF)); if (homogeneousPrologEpilog(MF, &MBB)) { auto MIB = BuildMI(MBB, MBBI, DL, TII.get(AArch64::HOM_Epilog)) .setMIFlag(MachineInstr::FrameDestroy); @@ -4141,7 +3389,7 @@ void AArch64FrameLowering::determineCalleeSaves(MachineFunction &MF, if (producePairRegisters(MF)) { if (UnspilledCSGPRPaired == AArch64::NoRegister) { // Failed to make a pair for compact unwind format, revert spilling. - if (produceCompactUnwindFrame(MF)) { + if (produceCompactUnwindFrame(*this, MF)) { SavedRegs.reset(UnspilledCSGPR); ExtraCSSpill = AArch64::NoRegister; } diff --git a/llvm/lib/Target/AArch64/AArch64FrameLowering.h b/llvm/lib/Target/AArch64/AArch64FrameLowering.h index 555a93359c27..a9d65441a4e3 100644 --- a/llvm/lib/Target/AArch64/AArch64FrameLowering.h +++ b/llvm/lib/Target/AArch64/AArch64FrameLowering.h @@ -19,6 +19,10 @@ namespace llvm { +class TargetLowering; +class AArch64FunctionInfo; +class AArch64PrologueEmitter; + class AArch64FrameLowering : public TargetFrameLowering { public: explicit AArch64FrameLowering() @@ -130,12 +134,19 @@ public: return StackId != TargetStackID::ScalableVector; } + friend class AArch64PrologueEmitter; void orderFrameObjects(const MachineFunction &MF, SmallVectorImpl<int> &ObjectsToAllocate) const override; bool isFPReserved(const MachineFunction &MF) const; + bool needsWinCFI(const MachineFunction &MF) const; + + bool requiresSaveVG(const MachineFunction &MF) const; + + StackOffset getSVEStackSize(const MachineFunction &MF) const; + protected: bool hasFPImpl(const MachineFunction &MF) const override; @@ -159,10 +170,6 @@ private: int &MaxCSFrameIndex) const; bool shouldCombineCSRLocalStackBumpInEpilogue(MachineBasicBlock &MBB, uint64_t StackBumpBytes) const; - void emitCalleeSavedGPRLocations(MachineBasicBlock &MBB, - MachineBasicBlock::iterator MBBI) const; - void emitCalleeSavedSVELocations(MachineBasicBlock &MBB, - MachineBasicBlock::iterator MBBI) const; void emitCalleeSavedGPRRestores(MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI) const; void emitCalleeSavedSVERestores(MachineBasicBlock &MBB, @@ -196,6 +203,61 @@ private: void emitRemarks(const MachineFunction &MF, MachineOptimizationRemarkEmitter *ORE) const override; + + bool windowsRequiresStackProbe(const MachineFunction &MF, + uint64_t StackSizeInBytes) const; + + bool shouldSignReturnAddressEverywhere(const MachineFunction &MF) const; + + StackOffset getFPOffset(const MachineFunction &MF, + int64_t ObjectOffset) const; + + StackOffset getStackOffset(const MachineFunction &MF, + int64_t ObjectOffset) const; + + // Find a scratch register that we can use at the start of the prologue to + // re-align the stack pointer. We avoid using callee-save registers since + // they may appear to be free when this is called from canUseAsPrologue + // (during shrink wrapping), but then no longer be free when this is called + // from emitPrologue. + // + // FIXME: This is a bit conservative, since in the above case we could use one + // of the callee-save registers as a scratch temp to re-align the stack + // pointer, but we would then have to make sure that we were in fact saving at + // least one callee-save register in the prologue, which is additional + // complexity that doesn't seem worth the benefit. + Register findScratchNonCalleeSaveRegister(MachineBasicBlock *MBB, + bool HasCall = false) const; + + // Convert callee-save register save/restore instruction to do stack pointer + // decrement/increment to allocate/deallocate the callee-save stack area by + // converting store/load to use pre/post increment version. + MachineBasicBlock::iterator convertCalleeSaveRestoreToSPPrePostIncDec( + MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI, + const DebugLoc &DL, const TargetInstrInfo *TII, int CSStackSizeInc, + bool NeedsWinCFI, bool *HasWinCFI, bool EmitCFI, + MachineInstr::MIFlag FrameFlag = MachineInstr::FrameSetup, + int CFAOffset = 0) const; + + // Fixup callee-save register save/restore instructions to take into account + // combined SP bump by adding the local stack size to the stack offsets. + void fixupCalleeSaveRestoreStackOffset(MachineInstr &MI, + uint64_t LocalStackSize, + bool NeedsWinCFI, + bool *HasWinCFI) const; + + bool isSVECalleeSave(MachineBasicBlock::iterator I) const; + + /// Returns the size of the fixed object area (allocated next to sp on entry) + /// On Win64 this may include a var args area and an UnwindHelp object for EH. + unsigned getFixedObjectSize(const MachineFunction &MF, + const AArch64FunctionInfo *AFI, bool IsWin64, + bool IsFunclet) const; + + bool isVGInstruction(MachineBasicBlock::iterator MBBI, + const TargetLowering &TLI) const; + + bool requiresGetVGCall(const MachineFunction &MF) const; }; } // End llvm namespace diff --git a/llvm/lib/Target/AArch64/AArch64ISelDAGToDAG.cpp b/llvm/lib/Target/AArch64/AArch64ISelDAGToDAG.cpp index bc786f415b55..6fdc981fc21a 100644 --- a/llvm/lib/Target/AArch64/AArch64ISelDAGToDAG.cpp +++ b/llvm/lib/Target/AArch64/AArch64ISelDAGToDAG.cpp @@ -246,9 +246,9 @@ public: return false; } - template<MVT::SimpleValueType VT> + template <MVT::SimpleValueType VT, bool Negate> bool SelectSVEAddSubImm(SDValue N, SDValue &Imm, SDValue &Shift) { - return SelectSVEAddSubImm(N, VT, Imm, Shift); + return SelectSVEAddSubImm(N, VT, Imm, Shift, Negate); } template <MVT::SimpleValueType VT, bool Negate> @@ -489,7 +489,8 @@ private: bool SelectCMP_SWAP(SDNode *N); - bool SelectSVEAddSubImm(SDValue N, MVT VT, SDValue &Imm, SDValue &Shift); + bool SelectSVEAddSubImm(SDValue N, MVT VT, SDValue &Imm, SDValue &Shift, + bool Negate); bool SelectSVEAddSubSSatImm(SDValue N, MVT VT, SDValue &Imm, SDValue &Shift, bool Negate); bool SelectSVECpyDupImm(SDValue N, MVT VT, SDValue &Imm, SDValue &Shift); @@ -4227,35 +4228,36 @@ bool AArch64DAGToDAGISel::SelectCMP_SWAP(SDNode *N) { } bool AArch64DAGToDAGISel::SelectSVEAddSubImm(SDValue N, MVT VT, SDValue &Imm, - SDValue &Shift) { + SDValue &Shift, bool Negate) { if (!isa<ConstantSDNode>(N)) return false; SDLoc DL(N); - uint64_t Val = cast<ConstantSDNode>(N) - ->getAPIntValue() - .trunc(VT.getFixedSizeInBits()) - .getZExtValue(); + APInt Val = + cast<ConstantSDNode>(N)->getAPIntValue().trunc(VT.getFixedSizeInBits()); + + if (Negate) + Val = -Val; switch (VT.SimpleTy) { case MVT::i8: // All immediates are supported. Shift = CurDAG->getTargetConstant(0, DL, MVT::i32); - Imm = CurDAG->getTargetConstant(Val, DL, MVT::i32); + Imm = CurDAG->getTargetConstant(Val.getZExtValue(), DL, MVT::i32); return true; case MVT::i16: case MVT::i32: case MVT::i64: // Support 8bit unsigned immediates. - if (Val <= 255) { + if ((Val & ~0xff) == 0) { Shift = CurDAG->getTargetConstant(0, DL, MVT::i32); - Imm = CurDAG->getTargetConstant(Val, DL, MVT::i32); + Imm = CurDAG->getTargetConstant(Val.getZExtValue(), DL, MVT::i32); return true; } // Support 16bit unsigned immediates that are a multiple of 256. - if (Val <= 65280 && Val % 256 == 0) { + if ((Val & ~0xff00) == 0) { Shift = CurDAG->getTargetConstant(8, DL, MVT::i32); - Imm = CurDAG->getTargetConstant(Val >> 8, DL, MVT::i32); + Imm = CurDAG->getTargetConstant(Val.lshr(8).getZExtValue(), DL, MVT::i32); return true; } break; diff --git a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp index d70a46b0e893..5ffaf2c49b4c 100644 --- a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp +++ b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp @@ -1179,6 +1179,7 @@ AArch64TargetLowering::AArch64TargetLowering(const TargetMachine &TM, setTargetDAGCombine(ISD::SCALAR_TO_VECTOR); setTargetDAGCombine(ISD::SHL); + setTargetDAGCombine(ISD::VECTOR_DEINTERLEAVE); // In case of strict alignment, avoid an excessive number of byte wide stores. MaxStoresPerMemsetOptSize = 8; @@ -1918,6 +1919,20 @@ AArch64TargetLowering::AArch64TargetLowering(const TargetMachine &TM, } } + // Handle non-aliasing elements mask + if (Subtarget->hasSVE2() || + (Subtarget->hasSME() && Subtarget->isStreaming())) { + // FIXME: Support wider fixed-length types when msve-vector-bits is used. + for (auto VT : {MVT::v2i32, MVT::v4i16, MVT::v8i8, MVT::v16i8}) { + setOperationAction(ISD::LOOP_DEPENDENCE_RAW_MASK, VT, Custom); + setOperationAction(ISD::LOOP_DEPENDENCE_WAR_MASK, VT, Custom); + } + for (auto VT : {MVT::nxv2i1, MVT::nxv4i1, MVT::nxv8i1, MVT::nxv16i1}) { + setOperationAction(ISD::LOOP_DEPENDENCE_RAW_MASK, VT, Custom); + setOperationAction(ISD::LOOP_DEPENDENCE_WAR_MASK, VT, Custom); + } + } + // Handle operations that are only available in non-streaming SVE mode. if (Subtarget->isSVEAvailable()) { for (auto VT : {MVT::nxv16i8, MVT::nxv8i16, MVT::nxv4i32, MVT::nxv2i64, @@ -2585,6 +2600,30 @@ void AArch64TargetLowering::computeKnownBitsForTargetNode( Known = Known.intersectWith(Known2); break; } + case AArch64ISD::CSNEG: + case AArch64ISD::CSINC: + case AArch64ISD::CSINV: { + KnownBits KnownOp0 = DAG.computeKnownBits(Op->getOperand(0), Depth + 1); + KnownBits KnownOp1 = DAG.computeKnownBits(Op->getOperand(1), Depth + 1); + + // The result is either: + // CSINC: KnownOp0 or KnownOp1 + 1 + // CSINV: KnownOp0 or ~KnownOp1 + // CSNEG: KnownOp0 or KnownOp1 * -1 + if (Op.getOpcode() == AArch64ISD::CSINC) + KnownOp1 = KnownBits::add( + KnownOp1, + KnownBits::makeConstant(APInt(Op.getScalarValueSizeInBits(), 1))); + else if (Op.getOpcode() == AArch64ISD::CSINV) + std::swap(KnownOp1.Zero, KnownOp1.One); + else if (Op.getOpcode() == AArch64ISD::CSNEG) + KnownOp1 = + KnownBits::mul(KnownOp1, KnownBits::makeConstant(APInt::getAllOnes( + Op.getScalarValueSizeInBits()))); + + Known = KnownOp0.intersectWith(KnownOp1); + break; + } case AArch64ISD::BICi: { // Compute the bit cleared value. APInt Mask = @@ -2626,6 +2665,32 @@ void AArch64TargetLowering::computeKnownBitsForTargetNode( << Op->getConstantOperandVal(1))); break; } + case AArch64ISD::MOVImsl: { + unsigned ShiftAmt = AArch64_AM::getShiftValue(Op->getConstantOperandVal(1)); + Known = KnownBits::makeConstant(APInt( + Known.getBitWidth(), ~(~Op->getConstantOperandVal(0) << ShiftAmt))); + break; + } + case AArch64ISD::MOVIedit: { + Known = KnownBits::makeConstant(APInt( + Known.getBitWidth(), + AArch64_AM::decodeAdvSIMDModImmType10(Op->getConstantOperandVal(0)))); + break; + } + case AArch64ISD::MVNIshift: { + Known = KnownBits::makeConstant( + APInt(Known.getBitWidth(), + ~(Op->getConstantOperandVal(0) << Op->getConstantOperandVal(1)), + /*isSigned*/ false, /*implicitTrunc*/ true)); + break; + } + case AArch64ISD::MVNImsl: { + unsigned ShiftAmt = AArch64_AM::getShiftValue(Op->getConstantOperandVal(1)); + Known = KnownBits::makeConstant( + APInt(Known.getBitWidth(), (~Op->getConstantOperandVal(0) << ShiftAmt), + /*isSigned*/ false, /*implicitTrunc*/ true)); + break; + } case AArch64ISD::LOADgot: case AArch64ISD::ADDlow: { if (!Subtarget->isTargetILP32()) @@ -2984,21 +3049,20 @@ AArch64TargetLowering::EmitInitTPIDR2Object(MachineInstr &MI, AArch64FunctionInfo *FuncInfo = MF->getInfo<AArch64FunctionInfo>(); TPIDR2Object &TPIDR2 = FuncInfo->getTPIDR2Obj(); if (TPIDR2.Uses > 0) { + // Note: This case just needs to do `SVL << 48`. It is not implemented as we + // generally don't support big-endian SVE/SME. + if (!Subtarget->isLittleEndian()) + reportFatalInternalError( + "TPIDR2 block initialization is not supported on big-endian targets"); + const TargetInstrInfo *TII = Subtarget->getInstrInfo(); - // Store the buffer pointer to the TPIDR2 stack object. - BuildMI(*BB, MI, MI.getDebugLoc(), TII->get(AArch64::STRXui)) + // Store buffer pointer and num_za_save_slices. + // Bytes 10-15 are implicitly zeroed. + BuildMI(*BB, MI, MI.getDebugLoc(), TII->get(AArch64::STPXi)) .addReg(MI.getOperand(0).getReg()) + .addReg(MI.getOperand(1).getReg()) .addFrameIndex(TPIDR2.FrameIndex) .addImm(0); - // Set the reserved bytes (10-15) to zero - BuildMI(*BB, MI, MI.getDebugLoc(), TII->get(AArch64::STRHHui)) - .addReg(AArch64::WZR) - .addFrameIndex(TPIDR2.FrameIndex) - .addImm(5); - BuildMI(*BB, MI, MI.getDebugLoc(), TII->get(AArch64::STRWui)) - .addReg(AArch64::WZR) - .addFrameIndex(TPIDR2.FrameIndex) - .addImm(3); } else MFI.RemoveStackObject(TPIDR2.FrameIndex); @@ -3111,21 +3175,24 @@ MachineBasicBlock * AArch64TargetLowering::EmitEntryPStateSM(MachineInstr &MI, MachineBasicBlock *BB) const { MachineFunction *MF = BB->getParent(); - AArch64FunctionInfo *FuncInfo = MF->getInfo<AArch64FunctionInfo>(); const TargetInstrInfo *TII = Subtarget->getInstrInfo(); + const DebugLoc &DL = MI.getDebugLoc(); Register ResultReg = MI.getOperand(0).getReg(); - if (FuncInfo->isPStateSMRegUsed()) { + if (MF->getRegInfo().use_empty(ResultReg)) { + // Nothing to do. Pseudo erased below. + } else if (Subtarget->hasSME()) { + BuildMI(*BB, MI, DL, TII->get(AArch64::MRS), ResultReg) + .addImm(AArch64SysReg::SVCR) + .addReg(AArch64::VG, RegState::Implicit); + } else { RTLIB::Libcall LC = RTLIB::SMEABI_SME_STATE; const AArch64RegisterInfo *TRI = Subtarget->getRegisterInfo(); - BuildMI(*BB, MI, MI.getDebugLoc(), TII->get(AArch64::BL)) + BuildMI(*BB, MI, DL, TII->get(AArch64::BL)) .addExternalSymbol(getLibcallName(LC)) .addReg(AArch64::X0, RegState::ImplicitDefine) .addRegMask(TRI->getCallPreservedMask(*MF, getLibcallCallingConv(LC))); - BuildMI(*BB, MI, MI.getDebugLoc(), TII->get(TargetOpcode::COPY), ResultReg) + BuildMI(*BB, MI, DL, TII->get(TargetOpcode::COPY), ResultReg) .addReg(AArch64::X0); - } else { - assert(MI.getMF()->getRegInfo().use_empty(ResultReg) && - "Expected no users of the entry pstate.sm!"); } MI.eraseFromParent(); return BB; @@ -4912,6 +4979,18 @@ SDValue AArch64TargetLowering::LowerFP_TO_INT_SAT(SDValue Op, if (DstWidth < SatWidth) return SDValue(); + if (SrcVT == MVT::f16 && SatVT == MVT::i16 && DstVT == MVT::i32) { + if (Op.getOpcode() == ISD::FP_TO_SINT_SAT) { + SDValue CVTf32 = + DAG.getNode(AArch64ISD::FCVTZS_HALF, DL, MVT::f32, SrcVal); + SDValue Bitcast = DAG.getBitcast(DstVT, CVTf32); + return DAG.getNode(ISD::SIGN_EXTEND_INREG, DL, DstVT, Bitcast, + DAG.getValueType(SatVT)); + } + SDValue CVTf32 = DAG.getNode(AArch64ISD::FCVTZU_HALF, DL, MVT::f32, SrcVal); + return DAG.getBitcast(DstVT, CVTf32); + } + SDValue NativeCvt = DAG.getNode(Op.getOpcode(), DL, DstVT, SrcVal, DAG.getValueType(DstVT)); SDValue Sat; @@ -5242,6 +5321,56 @@ SDValue AArch64TargetLowering::LowerFSINCOS(SDValue Op, static MVT getSVEContainerType(EVT ContentTy); +SDValue +AArch64TargetLowering::LowerLOOP_DEPENDENCE_MASK(SDValue Op, + SelectionDAG &DAG) const { + SDLoc DL(Op); + uint64_t EltSize = Op.getConstantOperandVal(2); + EVT VT = Op.getValueType(); + switch (EltSize) { + case 1: + if (VT != MVT::v16i8 && VT != MVT::nxv16i1) + return SDValue(); + break; + case 2: + if (VT != MVT::v8i8 && VT != MVT::nxv8i1) + return SDValue(); + break; + case 4: + if (VT != MVT::v4i16 && VT != MVT::nxv4i1) + return SDValue(); + break; + case 8: + if (VT != MVT::v2i32 && VT != MVT::nxv2i1) + return SDValue(); + break; + default: + // Other element sizes are incompatible with whilewr/rw, so expand instead + return SDValue(); + } + + SDValue PtrA = Op.getOperand(0); + SDValue PtrB = Op.getOperand(1); + + if (VT.isScalableVT()) + return DAG.getNode(Op.getOpcode(), DL, VT, PtrA, PtrB, Op.getOperand(2)); + + // We can use the SVE whilewr/whilerw instruction to lower this + // intrinsic by creating the appropriate sequence of scalable vector + // operations and then extracting a fixed-width subvector from the scalable + // vector. Scalable vector variants are already legal. + EVT ContainerVT = + EVT::getVectorVT(*DAG.getContext(), VT.getVectorElementType(), + VT.getVectorNumElements(), true); + EVT WhileVT = ContainerVT.changeElementType(MVT::i1); + + SDValue Mask = + DAG.getNode(Op.getOpcode(), DL, WhileVT, PtrA, PtrB, Op.getOperand(2)); + SDValue MaskAsInt = DAG.getNode(ISD::SIGN_EXTEND, DL, ContainerVT, Mask); + return DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, VT, MaskAsInt, + DAG.getVectorIdxConstant(0, DL)); +} + SDValue AArch64TargetLowering::LowerBITCAST(SDValue Op, SelectionDAG &DAG) const { EVT OpVT = Op.getValueType(); @@ -6000,6 +6129,38 @@ SDValue AArch64TargetLowering::LowerINTRINSIC_WO_CHAIN(SDValue Op, EVT PtrVT = getPointerTy(DAG.getDataLayout()); return DAG.getNode(AArch64ISD::THREAD_POINTER, DL, PtrVT); } + case Intrinsic::aarch64_sve_whilewr_b: + return DAG.getNode(ISD::LOOP_DEPENDENCE_WAR_MASK, DL, Op.getValueType(), + Op.getOperand(1), Op.getOperand(2), + DAG.getConstant(1, DL, MVT::i64)); + case Intrinsic::aarch64_sve_whilewr_h: + return DAG.getNode(ISD::LOOP_DEPENDENCE_WAR_MASK, DL, Op.getValueType(), + Op.getOperand(1), Op.getOperand(2), + DAG.getConstant(2, DL, MVT::i64)); + case Intrinsic::aarch64_sve_whilewr_s: + return DAG.getNode(ISD::LOOP_DEPENDENCE_WAR_MASK, DL, Op.getValueType(), + Op.getOperand(1), Op.getOperand(2), + DAG.getConstant(4, DL, MVT::i64)); + case Intrinsic::aarch64_sve_whilewr_d: + return DAG.getNode(ISD::LOOP_DEPENDENCE_WAR_MASK, DL, Op.getValueType(), + Op.getOperand(1), Op.getOperand(2), + DAG.getConstant(8, DL, MVT::i64)); + case Intrinsic::aarch64_sve_whilerw_b: + return DAG.getNode(ISD::LOOP_DEPENDENCE_RAW_MASK, DL, Op.getValueType(), + Op.getOperand(1), Op.getOperand(2), + DAG.getConstant(1, DL, MVT::i64)); + case Intrinsic::aarch64_sve_whilerw_h: + return DAG.getNode(ISD::LOOP_DEPENDENCE_RAW_MASK, DL, Op.getValueType(), + Op.getOperand(1), Op.getOperand(2), + DAG.getConstant(2, DL, MVT::i64)); + case Intrinsic::aarch64_sve_whilerw_s: + return DAG.getNode(ISD::LOOP_DEPENDENCE_RAW_MASK, DL, Op.getValueType(), + Op.getOperand(1), Op.getOperand(2), + DAG.getConstant(4, DL, MVT::i64)); + case Intrinsic::aarch64_sve_whilerw_d: + return DAG.getNode(ISD::LOOP_DEPENDENCE_RAW_MASK, DL, Op.getValueType(), + Op.getOperand(1), Op.getOperand(2), + DAG.getConstant(8, DL, MVT::i64)); case Intrinsic::aarch64_neon_abs: { EVT Ty = Op.getValueType(); if (Ty == MVT::i64) { @@ -7359,6 +7520,9 @@ SDValue AArch64TargetLowering::LowerOperation(SDValue Op, default: llvm_unreachable("unimplemented operand"); return SDValue(); + case ISD::LOOP_DEPENDENCE_RAW_MASK: + case ISD::LOOP_DEPENDENCE_WAR_MASK: + return LowerLOOP_DEPENDENCE_MASK(Op, DAG); case ISD::BITCAST: return LowerBITCAST(Op, DAG); case ISD::GlobalAddress: @@ -7873,6 +8037,39 @@ static bool isPassedInFPR(EVT VT) { (VT.isFloatingPoint() && !VT.isScalableVector()); } +SDValue AArch64TargetLowering::lowerEHPadEntry(SDValue Chain, SDLoc const &DL, + SelectionDAG &DAG) const { + assert(Chain.getOpcode() == ISD::EntryToken && "Unexpected Chain value"); + SDValue Glue = Chain.getValue(1); + + MachineFunction &MF = DAG.getMachineFunction(); + SMEAttrs SMEFnAttrs = MF.getInfo<AArch64FunctionInfo>()->getSMEFnAttrs(); + + // The following conditions are true on entry to an exception handler: + // - PSTATE.SM is 0. + // - PSTATE.ZA is 0. + // - TPIDR2_EL0 is null. + // See: + // https://github.com/ARM-software/abi-aa/blob/main/aapcs64/aapcs64.rst#exceptions + // + // Therefore, if the function that contains this exception handler is a + // streaming[-compatible] function, we must re-enable streaming mode. + // + // These mode changes are usually optimized away in catch blocks as they + // occur before the __cxa_begin_catch (which is a non-streaming function), + // but are necessary in some cases (such as for cleanups). + + if (SMEFnAttrs.hasStreamingInterfaceOrBody()) + return changeStreamingMode(DAG, DL, /*Enable=*/true, Chain, + /*Glue*/ Glue, AArch64SME::Always); + + if (SMEFnAttrs.hasStreamingCompatibleInterface()) + return changeStreamingMode(DAG, DL, /*Enable=*/true, Chain, Glue, + AArch64SME::IfCallerIsStreaming); + + return Chain; +} + SDValue AArch64TargetLowering::LowerFormalArguments( SDValue Chain, CallingConv::ID CallConv, bool isVarArg, const SmallVectorImpl<ISD::InputArg> &Ins, const SDLoc &DL, @@ -8292,7 +8489,39 @@ SDValue AArch64TargetLowering::LowerFormalArguments( if (Subtarget->hasCustomCallingConv()) Subtarget->getRegisterInfo()->UpdateCustomCalleeSavedRegs(MF); - if (!getTM().useNewSMEABILowering() || Attrs.hasAgnosticZAInterface()) { + if (getTM().useNewSMEABILowering()) { + if (Subtarget->isTargetWindows() || hasInlineStackProbe(MF)) { + SDValue Size; + if (Attrs.hasZAState()) { + SDValue SVL = DAG.getNode(AArch64ISD::RDSVL, DL, MVT::i64, + DAG.getConstant(1, DL, MVT::i32)); + Size = DAG.getNode(ISD::MUL, DL, MVT::i64, SVL, SVL); + } else if (Attrs.hasAgnosticZAInterface()) { + RTLIB::Libcall LC = RTLIB::SMEABI_SME_STATE_SIZE; + SDValue Callee = DAG.getExternalSymbol( + getLibcallName(LC), getPointerTy(DAG.getDataLayout())); + auto *RetTy = EVT(MVT::i64).getTypeForEVT(*DAG.getContext()); + TargetLowering::CallLoweringInfo CLI(DAG); + CLI.setDebugLoc(DL).setChain(Chain).setLibCallee( + getLibcallCallingConv(LC), RetTy, Callee, {}); + std::tie(Size, Chain) = LowerCallTo(CLI); + } + if (Size) { + SDValue Buffer = DAG.getNode( + ISD::DYNAMIC_STACKALLOC, DL, DAG.getVTList(MVT::i64, MVT::Other), + {Chain, Size, DAG.getConstant(1, DL, MVT::i64)}); + Chain = Buffer.getValue(1); + + Register BufferPtr = + MF.getRegInfo().createVirtualRegister(&AArch64::GPR64RegClass); + Chain = DAG.getCopyToReg(Chain, DL, BufferPtr, Buffer); + Chain = DAG.getNode(AArch64ISD::SME_STATE_ALLOC, DL, + DAG.getVTList(MVT::Other), Chain); + FuncInfo->setEarlyAllocSMESaveBuffer(BufferPtr); + MFI.CreateVariableSizedObject(Align(16), nullptr); + } + } + } else { // Old SME ABI lowering (deprecated): // Create a 16 Byte TPIDR2 object. The dynamic buffer // will be expanded and stored in the static object later using a @@ -8313,9 +8542,12 @@ SDValue AArch64TargetLowering::LowerFormalArguments( {Chain, Size, DAG.getConstant(1, DL, MVT::i64)}); MFI.CreateVariableSizedObject(Align(16), nullptr); } + SDValue NumZaSaveSlices = DAG.getNode(AArch64ISD::RDSVL, DL, MVT::i64, + DAG.getConstant(1, DL, MVT::i32)); Chain = DAG.getNode( AArch64ISD::INIT_TPIDR2OBJ, DL, DAG.getVTList(MVT::Other), - {/*Chain*/ Buffer.getValue(1), /*Buffer ptr*/ Buffer.getValue(0)}); + {/*Chain*/ Buffer.getValue(1), /*Buffer ptr*/ Buffer.getValue(0), + /*Num save slices*/ NumZaSaveSlices}); } else if (Attrs.hasAgnosticZAInterface()) { // Call __arm_sme_state_size(). SDValue BufferSize = @@ -8338,7 +8570,7 @@ SDValue AArch64TargetLowering::LowerFormalArguments( Register BufferPtr = MF.getRegInfo().createVirtualRegister(&AArch64::GPR64RegClass); FuncInfo->setSMESaveBufferAddr(BufferPtr); - Chain = DAG.getCopyToReg(Chain, DL, BufferPtr, Buffer); + Chain = DAG.getCopyToReg(Buffer.getValue(1), DL, BufferPtr, Buffer); } } @@ -8905,7 +9137,6 @@ SDValue AArch64TargetLowering::changeStreamingMode(SelectionDAG &DAG, SDLoc DL, SmallVector<SDValue> Ops = {Chain, MSROp}; unsigned Opcode; if (Condition != AArch64SME::Always) { - FuncInfo->setPStateSMRegUsed(true); Register PStateReg = FuncInfo->getPStateSMReg(); assert(PStateReg.isValid() && "PStateSM Register is invalid"); SDValue PStateSM = @@ -9078,17 +9309,17 @@ AArch64TargetLowering::LowerCall(CallLoweringInfo &CLI, // Determine whether we need any streaming mode changes. SMECallAttrs CallAttrs = getSMECallAttrs(MF.getFunction(), *this, CLI); + + std::optional<unsigned> ZAMarkerNode; bool UseNewSMEABILowering = getTM().useNewSMEABILowering(); - bool IsAgnosticZAFunction = CallAttrs.caller().hasAgnosticZAInterface(); - auto ZAMarkerNode = [&]() -> std::optional<unsigned> { - // TODO: Handle agnostic ZA functions. - if (!UseNewSMEABILowering || IsAgnosticZAFunction) - return std::nullopt; - if (!CallAttrs.caller().hasZAState() && !CallAttrs.caller().hasZT0State()) - return std::nullopt; - return CallAttrs.requiresLazySave() ? AArch64ISD::REQUIRES_ZA_SAVE - : AArch64ISD::INOUT_ZA_USE; - }(); + if (UseNewSMEABILowering) { + if (CallAttrs.requiresLazySave() || + CallAttrs.requiresPreservingAllZAState()) + ZAMarkerNode = AArch64ISD::REQUIRES_ZA_SAVE; + else if (CallAttrs.caller().hasZAState() || + CallAttrs.caller().hasZT0State()) + ZAMarkerNode = AArch64ISD::INOUT_ZA_USE; + } if (IsTailCall) { // Check if it's really possible to do a tail call. @@ -9163,21 +9394,13 @@ AArch64TargetLowering::LowerCall(CallLoweringInfo &CLI, }; bool RequiresLazySave = !UseNewSMEABILowering && CallAttrs.requiresLazySave(); - bool RequiresSaveAllZA = CallAttrs.requiresPreservingAllZAState(); + bool RequiresSaveAllZA = + !UseNewSMEABILowering && CallAttrs.requiresPreservingAllZAState(); if (RequiresLazySave) { - const TPIDR2Object &TPIDR2 = FuncInfo->getTPIDR2Obj(); - MachinePointerInfo MPI = - MachinePointerInfo::getStack(MF, TPIDR2.FrameIndex); + TPIDR2Object &TPIDR2 = FuncInfo->getTPIDR2Obj(); SDValue TPIDR2ObjAddr = DAG.getFrameIndex( TPIDR2.FrameIndex, DAG.getTargetLoweringInfo().getFrameIndexTy(DAG.getDataLayout())); - SDValue NumZaSaveSlicesAddr = - DAG.getNode(ISD::ADD, DL, TPIDR2ObjAddr.getValueType(), TPIDR2ObjAddr, - DAG.getConstant(8, DL, TPIDR2ObjAddr.getValueType())); - SDValue NumZaSaveSlices = DAG.getNode(AArch64ISD::RDSVL, DL, MVT::i64, - DAG.getConstant(1, DL, MVT::i32)); - Chain = DAG.getTruncStore(Chain, DL, NumZaSaveSlices, NumZaSaveSlicesAddr, - MPI, MVT::i16); Chain = DAG.getNode( ISD::INTRINSIC_VOID, DL, MVT::Other, Chain, DAG.getConstant(Intrinsic::aarch64_sme_set_tpidr2, DL, MVT::i32), @@ -17599,14 +17822,16 @@ bool hasNearbyPairedStore(Iter It, Iter End, Value *Ptr, const DataLayout &DL) { bool AArch64TargetLowering::lowerInterleavedStore(Instruction *Store, Value *LaneMask, ShuffleVectorInst *SVI, - unsigned Factor) const { + unsigned Factor, + const APInt &GapMask) const { assert(Factor >= 2 && Factor <= getMaxSupportedInterleaveFactor() && "Invalid interleave factor"); auto *SI = dyn_cast<StoreInst>(Store); if (!SI) return false; - assert(!LaneMask && "Unexpected mask on store"); + assert(!LaneMask && GapMask.popcount() == Factor && + "Unexpected mask on store"); auto *VecTy = cast<FixedVectorType>(SVI->getType()); assert(VecTy->getNumElements() % Factor == 0 && "Invalid interleaved store"); @@ -20868,13 +21093,6 @@ static bool isNegatedInteger(SDValue Op) { return Op.getOpcode() == ISD::SUB && isNullConstant(Op.getOperand(0)); } -static SDValue getNegatedInteger(SDValue Op, SelectionDAG &DAG) { - SDLoc DL(Op); - EVT VT = Op.getValueType(); - SDValue Zero = DAG.getConstant(0, DL, VT); - return DAG.getNode(ISD::SUB, DL, VT, Zero, Op); -} - // Try to fold // // (neg (csel X, Y)) -> (csel (neg X), (neg Y)) @@ -20893,16 +21111,17 @@ static SDValue performNegCSelCombine(SDNode *N, SelectionDAG &DAG) { SDValue N0 = CSel.getOperand(0); SDValue N1 = CSel.getOperand(1); - // If both of them is not negations, it's not worth the folding as it + // If neither of them are negations, it's not worth the folding as it // introduces two additional negations while reducing one negation. if (!isNegatedInteger(N0) && !isNegatedInteger(N1)) return SDValue(); - SDValue N0N = getNegatedInteger(N0, DAG); - SDValue N1N = getNegatedInteger(N1, DAG); - SDLoc DL(N); EVT VT = CSel.getValueType(); + + SDValue N0N = DAG.getNegative(N0, DL, VT); + SDValue N1N = DAG.getNegative(N1, DL, VT); + return DAG.getNode(AArch64ISD::CSEL, DL, VT, N0N, N1N, CSel.getOperand(2), CSel.getOperand(3)); } @@ -22087,10 +22306,14 @@ static SDValue getPTest(SelectionDAG &DAG, EVT VT, SDValue Pg, SDValue Op, Op = DAG.getNode(AArch64ISD::REINTERPRET_CAST, DL, MVT::nxv16i1, Op); } + unsigned PTest = AArch64ISD::PTEST; + if (Cond == AArch64CC::ANY_ACTIVE) + PTest = AArch64ISD::PTEST_ANY; + else if (Cond == AArch64CC::FIRST_ACTIVE) + PTest = AArch64ISD::PTEST_FIRST; + // Set condition code (CC) flags. - SDValue Test = DAG.getNode( - Cond == AArch64CC::ANY_ACTIVE ? AArch64ISD::PTEST_ANY : AArch64ISD::PTEST, - DL, MVT::i32, Pg, Op); + SDValue Test = DAG.getNode(PTest, DL, MVT::i32, Pg, Op); // Convert CC to integer based on requested condition. // NOTE: Cond is inverted to promote CSEL's removal when it feeds a compare. @@ -22158,6 +22381,17 @@ static SDValue combineSVEReductionOrderedFP(SDNode *N, unsigned Opc, Zero); } +static SDValue tryCombineNeonFcvtFP16ToI16(SDNode *N, unsigned Opcode, + SelectionDAG &DAG) { + if (N->getValueType(0) != MVT::i16) + return SDValue(); + + SDLoc DL(N); + SDValue CVT = DAG.getNode(Opcode, DL, MVT::f32, N->getOperand(1)); + SDValue Bitcast = DAG.getBitcast(MVT::i32, CVT); + return DAG.getNode(ISD::TRUNCATE, DL, MVT::i16, Bitcast); +} + // If a merged operation has no inactive lanes we can relax it to a predicated // or unpredicated operation, which potentially allows better isel (perhaps // using immediate forms) or relaxing register reuse requirements. @@ -22411,6 +22645,26 @@ static SDValue performIntrinsicCombine(SDNode *N, case Intrinsic::aarch64_neon_uabd: return DAG.getNode(ISD::ABDU, SDLoc(N), N->getValueType(0), N->getOperand(1), N->getOperand(2)); + case Intrinsic::aarch64_neon_fcvtzs: + return tryCombineNeonFcvtFP16ToI16(N, AArch64ISD::FCVTZS_HALF, DAG); + case Intrinsic::aarch64_neon_fcvtzu: + return tryCombineNeonFcvtFP16ToI16(N, AArch64ISD::FCVTZU_HALF, DAG); + case Intrinsic::aarch64_neon_fcvtas: + return tryCombineNeonFcvtFP16ToI16(N, AArch64ISD::FCVTAS_HALF, DAG); + case Intrinsic::aarch64_neon_fcvtau: + return tryCombineNeonFcvtFP16ToI16(N, AArch64ISD::FCVTAU_HALF, DAG); + case Intrinsic::aarch64_neon_fcvtms: + return tryCombineNeonFcvtFP16ToI16(N, AArch64ISD::FCVTMS_HALF, DAG); + case Intrinsic::aarch64_neon_fcvtmu: + return tryCombineNeonFcvtFP16ToI16(N, AArch64ISD::FCVTMU_HALF, DAG); + case Intrinsic::aarch64_neon_fcvtns: + return tryCombineNeonFcvtFP16ToI16(N, AArch64ISD::FCVTNS_HALF, DAG); + case Intrinsic::aarch64_neon_fcvtnu: + return tryCombineNeonFcvtFP16ToI16(N, AArch64ISD::FCVTNU_HALF, DAG); + case Intrinsic::aarch64_neon_fcvtps: + return tryCombineNeonFcvtFP16ToI16(N, AArch64ISD::FCVTPS_HALF, DAG); + case Intrinsic::aarch64_neon_fcvtpu: + return tryCombineNeonFcvtFP16ToI16(N, AArch64ISD::FCVTPU_HALF, DAG); case Intrinsic::aarch64_crc32b: case Intrinsic::aarch64_crc32cb: return tryCombineCRC32(0xff, N, DAG); @@ -22419,7 +22673,7 @@ static SDValue performIntrinsicCombine(SDNode *N, return tryCombineCRC32(0xffff, N, DAG); case Intrinsic::aarch64_sve_saddv: // There is no i64 version of SADDV because the sign is irrelevant. - if (N->getOperand(2)->getValueType(0).getVectorElementType() == MVT::i64) + if (N->getOperand(2).getValueType().getVectorElementType() == MVT::i64) return combineSVEReductionInt(N, AArch64ISD::UADDV_PRED, DAG); else return combineSVEReductionInt(N, AArch64ISD::SADDV_PRED, DAG); @@ -24106,6 +24360,7 @@ static SDValue vectorToScalarBitmask(SDNode *N, SelectionDAG &DAG) { // Ensure that all elements' bits are either 0s or 1s. ComparisonResult = DAG.getSExtOrTrunc(ComparisonResult, DL, VecVT); + bool IsLE = DAG.getDataLayout().isLittleEndian(); SmallVector<SDValue, 16> MaskConstants; if (DAG.getSubtarget<AArch64Subtarget>().isNeonAvailable() && VecVT == MVT::v16i8) { @@ -24113,7 +24368,10 @@ static SDValue vectorToScalarBitmask(SDNode *N, SelectionDAG &DAG) { // per entry. We split it into two halves, apply the mask, zip the halves to // create 8x 16-bit values, and the perform the vector reduce. for (unsigned Half = 0; Half < 2; ++Half) { - for (unsigned MaskBit = 1; MaskBit <= 128; MaskBit *= 2) { + for (unsigned I = 0; I < 8; ++I) { + // On big-endian targets, the lane order in sub-byte vector elements + // gets reversed, so we need to flip the bit index. + unsigned MaskBit = IsLE ? (1u << I) : (1u << (7 - I)); MaskConstants.push_back(DAG.getConstant(MaskBit, DL, MVT::i32)); } } @@ -24131,8 +24389,9 @@ static SDValue vectorToScalarBitmask(SDNode *N, SelectionDAG &DAG) { } // All other vector sizes. - unsigned MaxBitMask = 1u << (VecVT.getVectorNumElements() - 1); - for (unsigned MaskBit = 1; MaskBit <= MaxBitMask; MaskBit *= 2) { + unsigned NumEl = VecVT.getVectorNumElements(); + for (unsigned I = 0; I < NumEl; ++I) { + unsigned MaskBit = IsLE ? (1u << I) : (1u << (NumEl - 1 - I)); MaskConstants.push_back(DAG.getConstant(MaskBit, DL, MVT::i64)); } @@ -24444,6 +24703,105 @@ static SDValue performSTORECombine(SDNode *N, return SDValue(); } +static bool +isSequentialConcatOfVectorInterleave(SDNode *N, SmallVectorImpl<SDValue> &Ops) { + if (N->getOpcode() != ISD::CONCAT_VECTORS) + return false; + + unsigned NumParts = N->getNumOperands(); + + // We should be concatenating each sequential result from a + // VECTOR_INTERLEAVE. + SDNode *InterleaveOp = N->getOperand(0).getNode(); + if (InterleaveOp->getOpcode() != ISD::VECTOR_INTERLEAVE || + InterleaveOp->getNumOperands() != NumParts) + return false; + + for (unsigned I = 0; I < NumParts; I++) + if (N->getOperand(I) != SDValue(InterleaveOp, I)) + return false; + + Ops.append(InterleaveOp->op_begin(), InterleaveOp->op_end()); + return true; +} + +static SDValue getNarrowMaskForInterleavedOps(SelectionDAG &DAG, SDLoc &DL, + SDValue WideMask, + unsigned RequiredNumParts) { + if (WideMask->getOpcode() == ISD::CONCAT_VECTORS) { + SmallVector<SDValue, 4> MaskInterleaveOps; + if (!isSequentialConcatOfVectorInterleave(WideMask.getNode(), + MaskInterleaveOps)) + return SDValue(); + + if (MaskInterleaveOps.size() != RequiredNumParts) + return SDValue(); + + // Make sure the inputs to the vector interleave are identical. + if (!llvm::all_equal(MaskInterleaveOps)) + return SDValue(); + + return MaskInterleaveOps[0]; + } + + if (WideMask->getOpcode() != ISD::SPLAT_VECTOR) + return SDValue(); + + ElementCount EC = WideMask.getValueType().getVectorElementCount(); + assert(EC.isKnownMultipleOf(RequiredNumParts) && + "Expected element count divisible by number of parts"); + EC = EC.divideCoefficientBy(RequiredNumParts); + return DAG.getNode(ISD::SPLAT_VECTOR, DL, MVT::getVectorVT(MVT::i1, EC), + WideMask->getOperand(0)); +} + +static SDValue performInterleavedMaskedStoreCombine( + SDNode *N, TargetLowering::DAGCombinerInfo &DCI, SelectionDAG &DAG) { + if (!DCI.isBeforeLegalize()) + return SDValue(); + + MaskedStoreSDNode *MST = cast<MaskedStoreSDNode>(N); + SDValue WideValue = MST->getValue(); + + // Bail out if the stored value has an unexpected number of uses, since we'll + // have to perform manual interleaving and may as well just use normal masked + // stores. Also, discard masked stores that are truncating or indexed. + if (!WideValue.hasOneUse() || !ISD::isNormalMaskedStore(MST) || + !MST->isSimple() || !MST->getOffset().isUndef()) + return SDValue(); + + SmallVector<SDValue, 4> ValueInterleaveOps; + if (!isSequentialConcatOfVectorInterleave(WideValue.getNode(), + ValueInterleaveOps)) + return SDValue(); + + unsigned NumParts = ValueInterleaveOps.size(); + if (NumParts != 2 && NumParts != 4) + return SDValue(); + + // At the moment we're unlikely to see a fixed-width vector interleave as + // we usually generate shuffles instead. + EVT SubVecTy = ValueInterleaveOps[0].getValueType(); + if (!SubVecTy.isScalableVT() || + SubVecTy.getSizeInBits().getKnownMinValue() != 128 || + !DAG.getTargetLoweringInfo().isTypeLegal(SubVecTy)) + return SDValue(); + + SDLoc DL(N); + SDValue NarrowMask = + getNarrowMaskForInterleavedOps(DAG, DL, MST->getMask(), NumParts); + if (!NarrowMask) + return SDValue(); + + const Intrinsic::ID IID = + NumParts == 2 ? Intrinsic::aarch64_sve_st2 : Intrinsic::aarch64_sve_st4; + SmallVector<SDValue, 8> NewStOps; + NewStOps.append({MST->getChain(), DAG.getConstant(IID, DL, MVT::i32)}); + NewStOps.append(ValueInterleaveOps); + NewStOps.append({NarrowMask, MST->getBasePtr()}); + return DAG.getNode(ISD::INTRINSIC_VOID, DL, MVT::Other, NewStOps); +} + static SDValue performMSTORECombine(SDNode *N, TargetLowering::DAGCombinerInfo &DCI, SelectionDAG &DAG, @@ -24453,6 +24811,9 @@ static SDValue performMSTORECombine(SDNode *N, SDValue Mask = MST->getMask(); SDLoc DL(N); + if (SDValue Res = performInterleavedMaskedStoreCombine(N, DCI, DAG)) + return Res; + // If this is a UZP1 followed by a masked store, fold this into a masked // truncating store. We can do this even if this is already a masked // truncstore. @@ -26523,6 +26884,26 @@ performSignExtendInRegCombine(SDNode *N, TargetLowering::DAGCombinerInfo &DCI, return DAG.getNode(SOpc, DL, N->getValueType(0), Ext); } + // Sign extend of CSET -> CSETM. + if (Opc == AArch64ISD::CSEL && + cast<VTSDNode>(N->getOperand(1))->getVT() == MVT::i1) { + EVT VT = N->getValueType(0); + SDValue TVal = Src.getOperand(0); + SDValue FVal = Src.getOperand(1); + + // SIGN_EXTEND_INREG (CSEL 0, 1, cc, NZCV), i1 --> CSEL 0, -1, cc, NZCV + if (isNullConstant(TVal) && isOneConstant(FVal)) + return DAG.getNode(AArch64ISD::CSEL, DL, VT, TVal, + DAG.getAllOnesConstant(DL, VT), Src.getOperand(2), + Src.getOperand(3)); + + // SIGN_EXTEND_INREG (CSEL 1, 0, cc, NZCV), i1 --> CSEL -1, 0, cc, NZCV + if (isOneConstant(TVal) && isNullConstant(FVal)) + return DAG.getNode(AArch64ISD::CSEL, DL, VT, + DAG.getAllOnesConstant(DL, VT), FVal, + Src.getOperand(2), Src.getOperand(3)); + } + if (DCI.isBeforeLegalizeOps()) return SDValue(); @@ -27020,6 +27401,83 @@ performScalarToVectorCombine(SDNode *N, TargetLowering::DAGCombinerInfo &DCI, return NVCAST; } +static SDValue performVectorDeinterleaveCombine( + SDNode *N, TargetLowering::DAGCombinerInfo &DCI, SelectionDAG &DAG) { + if (!DCI.isBeforeLegalize()) + return SDValue(); + + unsigned NumParts = N->getNumOperands(); + if (NumParts != 2 && NumParts != 4) + return SDValue(); + + EVT SubVecTy = N->getValueType(0); + + // At the moment we're unlikely to see a fixed-width vector deinterleave as + // we usually generate shuffles instead. + unsigned MinNumElements = SubVecTy.getVectorMinNumElements(); + if (!SubVecTy.isScalableVector() || + SubVecTy.getSizeInBits().getKnownMinValue() != 128 || + !DAG.getTargetLoweringInfo().isTypeLegal(SubVecTy)) + return SDValue(); + + // Make sure each input operand is the correct extract_subvector of the same + // wider vector. + SDValue Op0 = N->getOperand(0); + for (unsigned I = 0; I < NumParts; I++) { + SDValue OpI = N->getOperand(I); + if (OpI->getOpcode() != ISD::EXTRACT_SUBVECTOR || + OpI->getOperand(0) != Op0->getOperand(0)) + return SDValue(); + if (OpI->getConstantOperandVal(1) != (I * MinNumElements)) + return SDValue(); + } + + // Normal loads are currently already handled by the InterleavedAccessPass so + // we don't expect to see them here. Bail out if the masked load has an + // unexpected number of uses, since we want to avoid a situation where we have + // both deinterleaving loads and normal loads in the same block. Also, discard + // masked loads that are extending, indexed, have an unexpected offset or have + // an unsupported passthru value until we find a valid use case. + auto MaskedLoad = dyn_cast<MaskedLoadSDNode>(Op0->getOperand(0)); + if (!MaskedLoad || !MaskedLoad->hasNUsesOfValue(NumParts, 0) || + !MaskedLoad->isSimple() || !ISD::isNormalMaskedLoad(MaskedLoad) || + !MaskedLoad->getOffset().isUndef() || + (!MaskedLoad->getPassThru()->isUndef() && + !isZerosVector(MaskedLoad->getPassThru().getNode()))) + return SDValue(); + + // Now prove that the mask is an interleave of identical masks. + SDLoc DL(N); + SDValue NarrowMask = + getNarrowMaskForInterleavedOps(DAG, DL, MaskedLoad->getMask(), NumParts); + if (!NarrowMask) + return SDValue(); + + const Intrinsic::ID IID = NumParts == 2 ? Intrinsic::aarch64_sve_ld2_sret + : Intrinsic::aarch64_sve_ld4_sret; + SDValue NewLdOps[] = {MaskedLoad->getChain(), + DAG.getConstant(IID, DL, MVT::i32), NarrowMask, + MaskedLoad->getBasePtr()}; + SDValue Res; + if (NumParts == 2) + Res = DAG.getNode(ISD::INTRINSIC_W_CHAIN, DL, + {SubVecTy, SubVecTy, MVT::Other}, NewLdOps); + else + Res = DAG.getNode(ISD::INTRINSIC_W_CHAIN, DL, + {SubVecTy, SubVecTy, SubVecTy, SubVecTy, MVT::Other}, + NewLdOps); + + // We can now generate a structured load! + SmallVector<SDValue, 4> ResOps(NumParts); + for (unsigned Idx = 0; Idx < NumParts; Idx++) + ResOps[Idx] = SDValue(Res.getNode(), Idx); + + // Replace uses of the original chain result with the new chain result. + DAG.ReplaceAllUsesOfValueWith(SDValue(MaskedLoad, 1), + SDValue(Res.getNode(), NumParts)); + return DCI.CombineTo(N, ResOps, false); +} + /// If the operand is a bitwise AND with a constant RHS, and the shift has a /// constant RHS and is the only use, we can pull it out of the shift, i.e. /// @@ -27088,6 +27546,8 @@ SDValue AArch64TargetLowering::PerformDAGCombine(SDNode *N, default: LLVM_DEBUG(dbgs() << "Custom combining: skipping\n"); break; + case ISD::VECTOR_DEINTERLEAVE: + return performVectorDeinterleaveCombine(N, DCI, DAG); case ISD::VECREDUCE_AND: case ISD::VECREDUCE_OR: case ISD::VECREDUCE_XOR: @@ -30640,10 +31100,41 @@ bool AArch64TargetLowering::SimplifyDemandedBitsForTargetNode( Op, OriginalDemandedBits, OriginalDemandedElts, Known, TLO, Depth); } +bool AArch64TargetLowering::canCreateUndefOrPoisonForTargetNode( + SDValue Op, const APInt &DemandedElts, const SelectionDAG &DAG, + bool PoisonOnly, bool ConsiderFlags, unsigned Depth) const { + + // TODO: Add more target nodes. + switch (Op.getOpcode()) { + case AArch64ISD::MOVI: + case AArch64ISD::MOVIedit: + case AArch64ISD::MOVImsl: + case AArch64ISD::MOVIshift: + case AArch64ISD::MVNImsl: + case AArch64ISD::MVNIshift: + case AArch64ISD::VASHR: + case AArch64ISD::VLSHR: + case AArch64ISD::VSHL: + return false; + } + return TargetLowering::canCreateUndefOrPoisonForTargetNode( + Op, DemandedElts, DAG, PoisonOnly, ConsiderFlags, Depth); +} + bool AArch64TargetLowering::isTargetCanonicalConstantNode(SDValue Op) const { return Op.getOpcode() == AArch64ISD::DUP || Op.getOpcode() == AArch64ISD::MOVI || Op.getOpcode() == AArch64ISD::MOVIshift || + Op.getOpcode() == AArch64ISD::MOVImsl || + Op.getOpcode() == AArch64ISD::MOVIedit || + Op.getOpcode() == AArch64ISD::MVNIshift || + Op.getOpcode() == AArch64ISD::MVNImsl || + // Ignoring fneg(movi(0)), because if it is folded to FPConstant(-0.0), + // ISel will select fmov(mov i64 0x8000000000000000), resulting in a + // fmov from fpr to gpr, which is more expensive than fneg(movi(0)) + (Op.getOpcode() == ISD::FNEG && + Op.getOperand(0).getOpcode() == AArch64ISD::MOVIedit && + Op.getOperand(0).getConstantOperandVal(0) == 0) || (Op.getOpcode() == ISD::EXTRACT_SUBVECTOR && Op.getOperand(0).getOpcode() == AArch64ISD::DUP) || TargetLowering::isTargetCanonicalConstantNode(Op); diff --git a/llvm/lib/Target/AArch64/AArch64ISelLowering.h b/llvm/lib/Target/AArch64/AArch64ISelLowering.h index 6c6ae782f779..f5d14905cac6 100644 --- a/llvm/lib/Target/AArch64/AArch64ISelLowering.h +++ b/llvm/lib/Target/AArch64/AArch64ISelLowering.h @@ -233,8 +233,8 @@ public: ArrayRef<unsigned> Indices, unsigned Factor, const APInt &GapMask) const override; bool lowerInterleavedStore(Instruction *Store, Value *Mask, - ShuffleVectorInst *SVI, - unsigned Factor) const override; + ShuffleVectorInst *SVI, unsigned Factor, + const APInt &GapMask) const override; bool lowerDeinterleaveIntrinsicToLoad(Instruction *Load, Value *Mask, IntrinsicInst *DI) const override; @@ -575,6 +575,9 @@ private: bool shouldExpandBuildVectorWithShuffles(EVT, unsigned) const override; + SDValue lowerEHPadEntry(SDValue Chain, SDLoc const &DL, + SelectionDAG &DAG) const override; + SDValue LowerFormalArguments(SDValue Chain, CallingConv::ID CallConv, bool isVarArg, const SmallVectorImpl<ISD::InputArg> &Ins, @@ -735,6 +738,7 @@ private: SDValue LowerXOR(SDValue Op, SelectionDAG &DAG) const; SDValue LowerCONCAT_VECTORS(SDValue Op, SelectionDAG &DAG) const; SDValue LowerFSINCOS(SDValue Op, SelectionDAG &DAG) const; + SDValue LowerLOOP_DEPENDENCE_MASK(SDValue Op, SelectionDAG &DAG) const; SDValue LowerBITCAST(SDValue Op, SelectionDAG &DAG) const; SDValue LowerVSCALE(SDValue Op, SelectionDAG &DAG) const; SDValue LowerTRUNCATE(SDValue Op, SelectionDAG &DAG) const; @@ -868,6 +872,12 @@ private: TargetLoweringOpt &TLO, unsigned Depth) const override; + bool canCreateUndefOrPoisonForTargetNode(SDValue Op, + const APInt &DemandedElts, + const SelectionDAG &DAG, + bool PoisonOnly, bool ConsiderFlags, + unsigned Depth) const override; + bool isTargetCanonicalConstantNode(SDValue Op) const override; // With the exception of data-predicate transitions, no instructions are diff --git a/llvm/lib/Target/AArch64/AArch64InstrFormats.td b/llvm/lib/Target/AArch64/AArch64InstrFormats.td index 178dab689739..8958ad129269 100644 --- a/llvm/lib/Target/AArch64/AArch64InstrFormats.td +++ b/llvm/lib/Target/AArch64/AArch64InstrFormats.td @@ -1327,6 +1327,8 @@ def move_vec_shift : Operand<i32> { let PrintMethod = "printShifter"; let EncoderMethod = "getMoveVecShifterOpValue"; let ParserMatchClass = MoveVecShifterOperand; + let OperandType = "OPERAND_SHIFT_MSL"; + let OperandNamespace = "AArch64"; } let DiagnosticType = "AddSubSecondSource" in { @@ -3032,8 +3034,12 @@ class BaseAddSubEReg64<bit isSub, bit setFlags, RegisterClass dstRegtype, // Aliases for register+register add/subtract. class AddSubRegAlias<string asm, Instruction inst, RegisterClass dstRegtype, - RegisterClass src1Regtype, RegisterClass src2Regtype, - int shiftExt> + RegisterClass src1Regtype, dag src2> + : InstAlias<asm#"\t$dst, $src1, $src2", + (inst dstRegtype:$dst, src1Regtype:$src1, src2)>; +class AddSubRegAlias64<string asm, Instruction inst, RegisterClass dstRegtype, + RegisterClass src1Regtype, RegisterClass src2Regtype, + int shiftExt> : InstAlias<asm#"\t$dst, $src1, $src2", (inst dstRegtype:$dst, src1Regtype:$src1, src2Regtype:$src2, shiftExt)>; @@ -3101,22 +3107,22 @@ multiclass AddSub<bit isSub, string mnemonic, string alias, // Register/register aliases with no shift when SP is not used. def : AddSubRegAlias<mnemonic, !cast<Instruction>(NAME#"Wrs"), - GPR32, GPR32, GPR32, 0>; + GPR32, GPR32, (arith_shifted_reg32 GPR32:$src2, 0)>; def : AddSubRegAlias<mnemonic, !cast<Instruction>(NAME#"Xrs"), - GPR64, GPR64, GPR64, 0>; + GPR64, GPR64, (arith_shifted_reg64 GPR64:$src2, 0)>; // Register/register aliases with no shift when either the destination or // first source register is SP. def : AddSubRegAlias<mnemonic, !cast<Instruction>(NAME#"Wrx"), - GPR32sponly, GPR32sp, GPR32, 16>; // UXTW #0 + GPR32sponly, GPR32sp, + (arith_extended_reg32_i32 GPR32:$src2, 16)>; // UXTW #0 def : AddSubRegAlias<mnemonic, !cast<Instruction>(NAME#"Wrx"), - GPR32sp, GPR32sponly, GPR32, 16>; // UXTW #0 - def : AddSubRegAlias<mnemonic, - !cast<Instruction>(NAME#"Xrx64"), - GPR64sponly, GPR64sp, GPR64, 24>; // UXTX #0 - def : AddSubRegAlias<mnemonic, - !cast<Instruction>(NAME#"Xrx64"), - GPR64sp, GPR64sponly, GPR64, 24>; // UXTX #0 + GPR32sp, GPR32sponly, + (arith_extended_reg32_i32 GPR32:$src2, 16)>; // UXTW #0 + def : AddSubRegAlias64<mnemonic, !cast<Instruction>(NAME#"Xrx64"), + GPR64sponly, GPR64sp, GPR64, 24>; // UXTX #0 + def : AddSubRegAlias64<mnemonic, !cast<Instruction>(NAME#"Xrx64"), + GPR64sp, GPR64sponly, GPR64, 24>; // UXTX #0 } multiclass AddSubS<bit isSub, string mnemonic, SDNode OpNode, string cmp, @@ -3180,15 +3186,19 @@ multiclass AddSubS<bit isSub, string mnemonic, SDNode OpNode, string cmp, def : InstAlias<cmp#"\t$src, $imm", (!cast<Instruction>(NAME#"Xri") XZR, GPR64sp:$src, addsub_shifted_imm64:$imm), 5>; def : InstAlias<cmp#"\t$src1, $src2$sh", (!cast<Instruction>(NAME#"Wrx") - WZR, GPR32sp:$src1, GPR32:$src2, arith_extend:$sh), 4>; + WZR, GPR32sp:$src1, + (arith_extended_reg32_i32 GPR32:$src2, arith_extend:$sh)), 4>; def : InstAlias<cmp#"\t$src1, $src2$sh", (!cast<Instruction>(NAME#"Xrx") - XZR, GPR64sp:$src1, GPR32:$src2, arith_extend:$sh), 4>; + XZR, GPR64sp:$src1, + (arith_extended_reg32_i64 GPR32:$src2, arith_extend:$sh)), 4>; def : InstAlias<cmp#"\t$src1, $src2$sh", (!cast<Instruction>(NAME#"Xrx64") XZR, GPR64sp:$src1, GPR64:$src2, arith_extendlsl64:$sh), 4>; def : InstAlias<cmp#"\t$src1, $src2$sh", (!cast<Instruction>(NAME#"Wrs") - WZR, GPR32:$src1, GPR32:$src2, arith_shift32:$sh), 4>; + WZR, GPR32:$src1, + (arith_shifted_reg32 GPR32:$src2, arith_shift32:$sh)), 4>; def : InstAlias<cmp#"\t$src1, $src2$sh", (!cast<Instruction>(NAME#"Xrs") - XZR, GPR64:$src1, GPR64:$src2, arith_shift64:$sh), 4>; + XZR, GPR64:$src1, + (arith_shifted_reg64 GPR64:$src2, arith_shift64:$sh)), 4>; // Support negative immediates, e.g. cmp Rn, -imm -> cmn Rn, imm def : InstSubst<cmpAlias#"\t$src, $imm", (!cast<Instruction>(NAME#"Wri") @@ -3198,27 +3208,28 @@ multiclass AddSubS<bit isSub, string mnemonic, SDNode OpNode, string cmp, // Compare shorthands def : InstAlias<cmp#"\t$src1, $src2", (!cast<Instruction>(NAME#"Wrs") - WZR, GPR32:$src1, GPR32:$src2, 0), 5>; + WZR, GPR32:$src1, (arith_shifted_reg32 GPR32:$src2, 0)), 5>; def : InstAlias<cmp#"\t$src1, $src2", (!cast<Instruction>(NAME#"Xrs") - XZR, GPR64:$src1, GPR64:$src2, 0), 5>; + XZR, GPR64:$src1, (arith_shifted_reg64 GPR64:$src2, 0)), 5>; def : InstAlias<cmp#"\t$src1, $src2", (!cast<Instruction>(NAME#"Wrx") - WZR, GPR32sponly:$src1, GPR32:$src2, 16), 5>; + WZR, GPR32sponly:$src1, + (arith_extended_reg32_i32 GPR32:$src2, 16)), 5>; def : InstAlias<cmp#"\t$src1, $src2", (!cast<Instruction>(NAME#"Xrx64") XZR, GPR64sponly:$src1, GPR64:$src2, 24), 5>; // Register/register aliases with no shift when SP is not used. def : AddSubRegAlias<mnemonic, !cast<Instruction>(NAME#"Wrs"), - GPR32, GPR32, GPR32, 0>; + GPR32, GPR32, (arith_shifted_reg32 GPR32:$src2, 0)>; def : AddSubRegAlias<mnemonic, !cast<Instruction>(NAME#"Xrs"), - GPR64, GPR64, GPR64, 0>; + GPR64, GPR64, (arith_shifted_reg64 GPR64:$src2, 0)>; // Register/register aliases with no shift when the first source register // is SP. def : AddSubRegAlias<mnemonic, !cast<Instruction>(NAME#"Wrx"), - GPR32, GPR32sponly, GPR32, 16>; // UXTW #0 - def : AddSubRegAlias<mnemonic, - !cast<Instruction>(NAME#"Xrx64"), - GPR64, GPR64sponly, GPR64, 24>; // UXTX #0 + GPR32, GPR32sponly, + (arith_extended_reg32_i32 GPR32:$src2, 16)>; // UXTW #0 + def : AddSubRegAlias64<mnemonic, !cast<Instruction>(NAME#"Xrx64"), + GPR64, GPR64sponly, GPR64, 24>; // UXTX #0 } class AddSubG<bit isSub, string asm_inst, SDPatternOperator OpNode> @@ -3403,9 +3414,10 @@ class BaseLogicalSReg<bits<2> opc, bit N, RegisterClass regtype, } // Aliases for register+register logical instructions. -class LogicalRegAlias<string asm, Instruction inst, RegisterClass regtype> +class LogicalRegAlias<string asm, Instruction inst, RegisterClass regtype, + dag op2> : InstAlias<asm#"\t$dst, $src1, $src2", - (inst regtype:$dst, regtype:$src1, regtype:$src2, 0)>; + (inst regtype:$dst, regtype:$src1, op2)>; multiclass LogicalImm<bits<2> opc, string mnemonic, SDNode OpNode, string Alias> { @@ -3477,10 +3489,10 @@ multiclass LogicalReg<bits<2> opc, bit N, string mnemonic, let Inst{31} = 1; } - def : LogicalRegAlias<mnemonic, - !cast<Instruction>(NAME#"Wrs"), GPR32>; - def : LogicalRegAlias<mnemonic, - !cast<Instruction>(NAME#"Xrs"), GPR64>; + def : LogicalRegAlias<mnemonic, !cast<Instruction>(NAME#"Wrs"), + GPR32, (logical_shifted_reg32 GPR32:$src2, 0)>; + def : LogicalRegAlias<mnemonic, !cast<Instruction>(NAME#"Xrs"), + GPR64, (logical_shifted_reg64 GPR64:$src2, 0)>; } // Split from LogicalReg to allow setting NZCV Defs @@ -3500,10 +3512,10 @@ multiclass LogicalRegS<bits<2> opc, bit N, string mnemonic, } } // Defs = [NZCV] - def : LogicalRegAlias<mnemonic, - !cast<Instruction>(NAME#"Wrs"), GPR32>; - def : LogicalRegAlias<mnemonic, - !cast<Instruction>(NAME#"Xrs"), GPR64>; + def : LogicalRegAlias<mnemonic, !cast<Instruction>(NAME#"Wrs"), + GPR32, (logical_shifted_reg32 GPR32:$src2, 0)>; + def : LogicalRegAlias<mnemonic, !cast<Instruction>(NAME#"Xrs"), + GPR64, (logical_shifted_reg64 GPR64:$src2, 0)>; } //--- @@ -3991,9 +4003,10 @@ class LoadStore8RO<bits<2> sz, bit V, bits<2> opc, string asm, dag ins, let Inst{4-0} = Rt; } -class ROInstAlias<string asm, DAGOperand regtype, Instruction INST> +class ROInstAlias<string asm, DAGOperand regtype, Instruction INST, + ro_extend ext> : InstAlias<asm # "\t$Rt, [$Rn, $Rm]", - (INST regtype:$Rt, GPR64sp:$Rn, GPR64:$Rm, 0, 0)>; + (INST regtype:$Rt, GPR64sp:$Rn, GPR64:$Rm, (ext 0, 0))>; multiclass Load8RO<bits<2> sz, bit V, bits<2> opc, DAGOperand regtype, string asm, ValueType Ty, SDPatternOperator loadop> { @@ -4019,7 +4032,7 @@ multiclass Load8RO<bits<2> sz, bit V, bits<2> opc, DAGOperand regtype, let Inst{13} = 0b1; } - def : ROInstAlias<asm, regtype, !cast<Instruction>(NAME # "roX")>; + def : ROInstAlias<asm, regtype, !cast<Instruction>(NAME # "roX"), ro_Xextend8>; } multiclass Store8RO<bits<2> sz, bit V, bits<2> opc, DAGOperand regtype, @@ -4044,7 +4057,7 @@ multiclass Store8RO<bits<2> sz, bit V, bits<2> opc, DAGOperand regtype, let Inst{13} = 0b1; } - def : ROInstAlias<asm, regtype, !cast<Instruction>(NAME # "roX")>; + def : ROInstAlias<asm, regtype, !cast<Instruction>(NAME # "roX"), ro_Xextend8>; } class LoadStore16RO<bits<2> sz, bit V, bits<2> opc, string asm, dag ins, @@ -4091,7 +4104,7 @@ multiclass Load16RO<bits<2> sz, bit V, bits<2> opc, DAGOperand regtype, let Inst{13} = 0b1; } - def : ROInstAlias<asm, regtype, !cast<Instruction>(NAME # "roX")>; + def : ROInstAlias<asm, regtype, !cast<Instruction>(NAME # "roX"), ro_Xextend16>; } multiclass Store16RO<bits<2> sz, bit V, bits<2> opc, DAGOperand regtype, @@ -4116,7 +4129,7 @@ multiclass Store16RO<bits<2> sz, bit V, bits<2> opc, DAGOperand regtype, let Inst{13} = 0b1; } - def : ROInstAlias<asm, regtype, !cast<Instruction>(NAME # "roX")>; + def : ROInstAlias<asm, regtype, !cast<Instruction>(NAME # "roX"), ro_Xextend16>; } class LoadStore32RO<bits<2> sz, bit V, bits<2> opc, string asm, dag ins, @@ -4163,7 +4176,7 @@ multiclass Load32RO<bits<2> sz, bit V, bits<2> opc, DAGOperand regtype, let Inst{13} = 0b1; } - def : ROInstAlias<asm, regtype, !cast<Instruction>(NAME # "roX")>; + def : ROInstAlias<asm, regtype, !cast<Instruction>(NAME # "roX"), ro_Xextend32>; } multiclass Store32RO<bits<2> sz, bit V, bits<2> opc, DAGOperand regtype, @@ -4188,7 +4201,7 @@ multiclass Store32RO<bits<2> sz, bit V, bits<2> opc, DAGOperand regtype, let Inst{13} = 0b1; } - def : ROInstAlias<asm, regtype, !cast<Instruction>(NAME # "roX")>; + def : ROInstAlias<asm, regtype, !cast<Instruction>(NAME # "roX"), ro_Xextend32>; } class LoadStore64RO<bits<2> sz, bit V, bits<2> opc, string asm, dag ins, @@ -4235,7 +4248,7 @@ multiclass Load64RO<bits<2> sz, bit V, bits<2> opc, DAGOperand regtype, let Inst{13} = 0b1; } - def : ROInstAlias<asm, regtype, !cast<Instruction>(NAME # "roX")>; + def : ROInstAlias<asm, regtype, !cast<Instruction>(NAME # "roX"), ro_Xextend64>; } multiclass Store64RO<bits<2> sz, bit V, bits<2> opc, DAGOperand regtype, @@ -4260,7 +4273,7 @@ multiclass Store64RO<bits<2> sz, bit V, bits<2> opc, DAGOperand regtype, let Inst{13} = 0b1; } - def : ROInstAlias<asm, regtype, !cast<Instruction>(NAME # "roX")>; + def : ROInstAlias<asm, regtype, !cast<Instruction>(NAME # "roX"), ro_Xextend64>; } class LoadStore128RO<bits<2> sz, bit V, bits<2> opc, string asm, dag ins, @@ -4307,7 +4320,7 @@ multiclass Load128RO<bits<2> sz, bit V, bits<2> opc, DAGOperand regtype, let Inst{13} = 0b1; } - def : ROInstAlias<asm, regtype, !cast<Instruction>(NAME # "roX")>; + def : ROInstAlias<asm, regtype, !cast<Instruction>(NAME # "roX"), ro_Xextend128>; } multiclass Store128RO<bits<2> sz, bit V, bits<2> opc, DAGOperand regtype, @@ -4328,7 +4341,7 @@ multiclass Store128RO<bits<2> sz, bit V, bits<2> opc, DAGOperand regtype, let Inst{13} = 0b1; } - def : ROInstAlias<asm, regtype, !cast<Instruction>(NAME # "roX")>; + def : ROInstAlias<asm, regtype, !cast<Instruction>(NAME # "roX"), ro_Xextend128>; } let mayLoad = 0, mayStore = 0, hasSideEffects = 1 in @@ -4377,9 +4390,7 @@ multiclass PrefetchRO<bits<2> sz, bit V, bits<2> opc, string asm> { let Inst{13} = 0b1; } - def : InstAlias<"prfm $Rt, [$Rn, $Rm]", - (!cast<Instruction>(NAME # "roX") prfop:$Rt, - GPR64sp:$Rn, GPR64:$Rm, 0, 0)>; + def : ROInstAlias<"prfm", prfop, !cast<Instruction>(NAME # "roX"), ro_Xextend64>; } //--- diff --git a/llvm/lib/Target/AArch64/AArch64InstrInfo.cpp b/llvm/lib/Target/AArch64/AArch64InstrInfo.cpp index db028b4b7677..e56fe90259d5 100644 --- a/llvm/lib/Target/AArch64/AArch64InstrInfo.cpp +++ b/llvm/lib/Target/AArch64/AArch64InstrInfo.cpp @@ -91,8 +91,8 @@ static cl::opt<unsigned> GatherOptSearchLimit( "machine-combiner gather pattern optimization")); AArch64InstrInfo::AArch64InstrInfo(const AArch64Subtarget &STI) - : AArch64GenInstrInfo(AArch64::ADJCALLSTACKDOWN, AArch64::ADJCALLSTACKUP, - AArch64::CATCHRET), + : AArch64GenInstrInfo(STI, AArch64::ADJCALLSTACKDOWN, + AArch64::ADJCALLSTACKUP, AArch64::CATCHRET), RI(STI.getTargetTriple(), STI.getHwMode()), Subtarget(STI) {} /// GetInstSize - Return the number of bytes of code the specified @@ -1299,6 +1299,7 @@ bool AArch64InstrInfo::analyzeCompare(const MachineInstr &MI, Register &SrcReg, break; case AArch64::PTEST_PP: case AArch64::PTEST_PP_ANY: + case AArch64::PTEST_PP_FIRST: SrcReg = MI.getOperand(0).getReg(); SrcReg2 = MI.getOperand(1).getReg(); if (MI.getOperand(2).getSubReg()) @@ -1691,7 +1692,8 @@ bool AArch64InstrInfo::optimizeCompareInstr( } if (CmpInstr.getOpcode() == AArch64::PTEST_PP || - CmpInstr.getOpcode() == AArch64::PTEST_PP_ANY) + CmpInstr.getOpcode() == AArch64::PTEST_PP_ANY || + CmpInstr.getOpcode() == AArch64::PTEST_PP_FIRST) return optimizePTestInstr(&CmpInstr, SrcReg, SrcReg2, MRI); if (SrcReg2 != 0) @@ -5075,7 +5077,7 @@ void AArch64InstrInfo::copyPhysReg(MachineBasicBlock &MBB, .addImm(0) .addImm(AArch64_AM::getShifterImm(AArch64_AM::LSL, 0)); } - } else if (SrcReg == AArch64::WZR && Subtarget.hasZeroCycleZeroingGP()) { + } else if (SrcReg == AArch64::WZR && Subtarget.hasZeroCycleZeroingGPR32()) { BuildMI(MBB, I, DL, get(AArch64::MOVZWi), DestReg) .addImm(0) .addImm(AArch64_AM::getShifterImm(AArch64_AM::LSL, 0)); @@ -5202,7 +5204,7 @@ void AArch64InstrInfo::copyPhysReg(MachineBasicBlock &MBB, .addReg(SrcReg, getKillRegState(KillSrc)) .addImm(0) .addImm(AArch64_AM::getShifterImm(AArch64_AM::LSL, 0)); - } else if (SrcReg == AArch64::XZR && Subtarget.hasZeroCycleZeroingGP()) { + } else if (SrcReg == AArch64::XZR && Subtarget.hasZeroCycleZeroingGPR64()) { BuildMI(MBB, I, DL, get(AArch64::MOVZXi), DestReg) .addImm(0) .addImm(AArch64_AM::getShifterImm(AArch64_AM::LSL, 0)); @@ -5318,15 +5320,49 @@ void AArch64InstrInfo::copyPhysReg(MachineBasicBlock &MBB, if (AArch64::FPR64RegClass.contains(DestReg) && AArch64::FPR64RegClass.contains(SrcReg)) { - BuildMI(MBB, I, DL, get(AArch64::FMOVDr), DestReg) - .addReg(SrcReg, getKillRegState(KillSrc)); + if (Subtarget.hasZeroCycleRegMoveFPR128() && + !Subtarget.hasZeroCycleRegMoveFPR64() && + !Subtarget.hasZeroCycleRegMoveFPR32() && Subtarget.isNeonAvailable()) { + const TargetRegisterInfo *TRI = &getRegisterInfo(); + MCRegister DestRegQ = TRI->getMatchingSuperReg(DestReg, AArch64::dsub, + &AArch64::FPR128RegClass); + MCRegister SrcRegQ = TRI->getMatchingSuperReg(SrcReg, AArch64::dsub, + &AArch64::FPR128RegClass); + // This instruction is reading and writing Q registers. This may upset + // the register scavenger and machine verifier, so we need to indicate + // that we are reading an undefined value from SrcRegQ, but a proper + // value from SrcReg. + BuildMI(MBB, I, DL, get(AArch64::ORRv16i8), DestRegQ) + .addReg(SrcRegQ, RegState::Undef) + .addReg(SrcRegQ, RegState::Undef) + .addReg(SrcReg, RegState::Implicit | getKillRegState(KillSrc)); + } else { + BuildMI(MBB, I, DL, get(AArch64::FMOVDr), DestReg) + .addReg(SrcReg, getKillRegState(KillSrc)); + } return; } if (AArch64::FPR32RegClass.contains(DestReg) && AArch64::FPR32RegClass.contains(SrcReg)) { - if (Subtarget.hasZeroCycleRegMoveFPR64() && - !Subtarget.hasZeroCycleRegMoveFPR32()) { + if (Subtarget.hasZeroCycleRegMoveFPR128() && + !Subtarget.hasZeroCycleRegMoveFPR64() && + !Subtarget.hasZeroCycleRegMoveFPR32() && Subtarget.isNeonAvailable()) { + const TargetRegisterInfo *TRI = &getRegisterInfo(); + MCRegister DestRegQ = TRI->getMatchingSuperReg(DestReg, AArch64::ssub, + &AArch64::FPR128RegClass); + MCRegister SrcRegQ = TRI->getMatchingSuperReg(SrcReg, AArch64::ssub, + &AArch64::FPR128RegClass); + // This instruction is reading and writing Q registers. This may upset + // the register scavenger and machine verifier, so we need to indicate + // that we are reading an undefined value from SrcRegQ, but a proper + // value from SrcReg. + BuildMI(MBB, I, DL, get(AArch64::ORRv16i8), DestRegQ) + .addReg(SrcRegQ, RegState::Undef) + .addReg(SrcRegQ, RegState::Undef) + .addReg(SrcReg, RegState::Implicit | getKillRegState(KillSrc)); + } else if (Subtarget.hasZeroCycleRegMoveFPR64() && + !Subtarget.hasZeroCycleRegMoveFPR32()) { const TargetRegisterInfo *TRI = &getRegisterInfo(); MCRegister DestRegD = TRI->getMatchingSuperReg(DestReg, AArch64::ssub, &AArch64::FPR64RegClass); @@ -5348,8 +5384,24 @@ void AArch64InstrInfo::copyPhysReg(MachineBasicBlock &MBB, if (AArch64::FPR16RegClass.contains(DestReg) && AArch64::FPR16RegClass.contains(SrcReg)) { - if (Subtarget.hasZeroCycleRegMoveFPR64() && - !Subtarget.hasZeroCycleRegMoveFPR32()) { + if (Subtarget.hasZeroCycleRegMoveFPR128() && + !Subtarget.hasZeroCycleRegMoveFPR64() && + !Subtarget.hasZeroCycleRegMoveFPR32() && Subtarget.isNeonAvailable()) { + const TargetRegisterInfo *TRI = &getRegisterInfo(); + MCRegister DestRegQ = TRI->getMatchingSuperReg(DestReg, AArch64::hsub, + &AArch64::FPR128RegClass); + MCRegister SrcRegQ = TRI->getMatchingSuperReg(SrcReg, AArch64::hsub, + &AArch64::FPR128RegClass); + // This instruction is reading and writing Q registers. This may upset + // the register scavenger and machine verifier, so we need to indicate + // that we are reading an undefined value from SrcRegQ, but a proper + // value from SrcReg. + BuildMI(MBB, I, DL, get(AArch64::ORRv16i8), DestRegQ) + .addReg(SrcRegQ, RegState::Undef) + .addReg(SrcRegQ, RegState::Undef) + .addReg(SrcReg, RegState::Implicit | getKillRegState(KillSrc)); + } else if (Subtarget.hasZeroCycleRegMoveFPR64() && + !Subtarget.hasZeroCycleRegMoveFPR32()) { const TargetRegisterInfo *TRI = &getRegisterInfo(); MCRegister DestRegD = TRI->getMatchingSuperReg(DestReg, AArch64::hsub, &AArch64::FPR64RegClass); @@ -5375,8 +5427,24 @@ void AArch64InstrInfo::copyPhysReg(MachineBasicBlock &MBB, if (AArch64::FPR8RegClass.contains(DestReg) && AArch64::FPR8RegClass.contains(SrcReg)) { - if (Subtarget.hasZeroCycleRegMoveFPR64() && - !Subtarget.hasZeroCycleRegMoveFPR32()) { + if (Subtarget.hasZeroCycleRegMoveFPR128() && + !Subtarget.hasZeroCycleRegMoveFPR64() && + !Subtarget.hasZeroCycleRegMoveFPR64() && Subtarget.isNeonAvailable()) { + const TargetRegisterInfo *TRI = &getRegisterInfo(); + MCRegister DestRegQ = TRI->getMatchingSuperReg(DestReg, AArch64::bsub, + &AArch64::FPR128RegClass); + MCRegister SrcRegQ = TRI->getMatchingSuperReg(SrcReg, AArch64::bsub, + &AArch64::FPR128RegClass); + // This instruction is reading and writing Q registers. This may upset + // the register scavenger and machine verifier, so we need to indicate + // that we are reading an undefined value from SrcRegQ, but a proper + // value from SrcReg. + BuildMI(MBB, I, DL, get(AArch64::ORRv16i8), DestRegQ) + .addReg(SrcRegQ, RegState::Undef) + .addReg(SrcRegQ, RegState::Undef) + .addReg(SrcReg, RegState::Implicit | getKillRegState(KillSrc)); + } else if (Subtarget.hasZeroCycleRegMoveFPR64() && + !Subtarget.hasZeroCycleRegMoveFPR32()) { const TargetRegisterInfo *TRI = &getRegisterInfo(); MCRegister DestRegD = TRI->getMatchingSuperReg(DestReg, AArch64::bsub, &AArch64::FPR64RegClass); @@ -5403,8 +5471,12 @@ void AArch64InstrInfo::copyPhysReg(MachineBasicBlock &MBB, // Copies between GPR64 and FPR64. if (AArch64::FPR64RegClass.contains(DestReg) && AArch64::GPR64RegClass.contains(SrcReg)) { - BuildMI(MBB, I, DL, get(AArch64::FMOVXDr), DestReg) - .addReg(SrcReg, getKillRegState(KillSrc)); + if (AArch64::XZR == SrcReg) { + BuildMI(MBB, I, DL, get(AArch64::FMOVD0), DestReg); + } else { + BuildMI(MBB, I, DL, get(AArch64::FMOVXDr), DestReg) + .addReg(SrcReg, getKillRegState(KillSrc)); + } return; } if (AArch64::GPR64RegClass.contains(DestReg) && @@ -5416,8 +5488,12 @@ void AArch64InstrInfo::copyPhysReg(MachineBasicBlock &MBB, // Copies between GPR32 and FPR32. if (AArch64::FPR32RegClass.contains(DestReg) && AArch64::GPR32RegClass.contains(SrcReg)) { - BuildMI(MBB, I, DL, get(AArch64::FMOVWSr), DestReg) - .addReg(SrcReg, getKillRegState(KillSrc)); + if (AArch64::WZR == SrcReg) { + BuildMI(MBB, I, DL, get(AArch64::FMOVS0), DestReg); + } else { + BuildMI(MBB, I, DL, get(AArch64::FMOVWSr), DestReg) + .addReg(SrcReg, getKillRegState(KillSrc)); + } return; } if (AArch64::GPR32RegClass.contains(DestReg) && @@ -6652,7 +6728,7 @@ static bool canCombine(MachineBasicBlock &MBB, MachineOperand &MO, if (MO.isReg() && MO.getReg().isVirtual()) MI = MRI.getUniqueVRegDef(MO.getReg()); // And it needs to be in the trace (otherwise, it won't have a depth). - if (!MI || MI->getParent() != &MBB || (unsigned)MI->getOpcode() != CombineOpc) + if (!MI || MI->getParent() != &MBB || MI->getOpcode() != CombineOpc) return false; // Must only used by the user we combine with. if (!MRI.hasOneNonDBGUse(MI->getOperand(0).getReg())) diff --git a/llvm/lib/Target/AArch64/AArch64InstrInfo.td b/llvm/lib/Target/AArch64/AArch64InstrInfo.td index 89f88776d832..f0020a9a3c91 100644 --- a/llvm/lib/Target/AArch64/AArch64InstrInfo.td +++ b/llvm/lib/Target/AArch64/AArch64InstrInfo.td @@ -989,6 +989,17 @@ def AArch64fcvtxnv: PatFrags<(ops node:$Rn), [(int_aarch64_neon_fcvtxn node:$Rn), (AArch64fcvtxn_n node:$Rn)]>; +def AArch64fcvtzs_half : SDNode<"AArch64ISD::FCVTZS_HALF", SDTFPExtendOp>; +def AArch64fcvtzu_half : SDNode<"AArch64ISD::FCVTZU_HALF", SDTFPExtendOp>; +def AArch64fcvtas_half : SDNode<"AArch64ISD::FCVTAS_HALF", SDTFPExtendOp>; +def AArch64fcvtau_half : SDNode<"AArch64ISD::FCVTAU_HALF", SDTFPExtendOp>; +def AArch64fcvtms_half : SDNode<"AArch64ISD::FCVTMS_HALF", SDTFPExtendOp>; +def AArch64fcvtmu_half : SDNode<"AArch64ISD::FCVTMU_HALF", SDTFPExtendOp>; +def AArch64fcvtns_half : SDNode<"AArch64ISD::FCVTNS_HALF", SDTFPExtendOp>; +def AArch64fcvtnu_half : SDNode<"AArch64ISD::FCVTNU_HALF", SDTFPExtendOp>; +def AArch64fcvtps_half : SDNode<"AArch64ISD::FCVTPS_HALF", SDTFPExtendOp>; +def AArch64fcvtpu_half : SDNode<"AArch64ISD::FCVTPU_HALF", SDTFPExtendOp>; + //def Aarch64softf32tobf16v8: SDNode<"AArch64ISD::", SDTFPRoundOp>; // Vector immediate ops @@ -2155,7 +2166,7 @@ let Predicates = [HasPAuth] in { i64imm:$Disc, GPR64:$AddrDisc), [], "$AuthVal = $Val">, Sched<[WriteI, ReadI]> { let isCodeGenOnly = 1; - let hasSideEffects = 0; + let hasSideEffects = 1; let mayStore = 0; let mayLoad = 0; let Size = 32; @@ -2660,13 +2671,17 @@ defm ADD : AddSub<0, "add", "sub", add>; defm SUB : AddSub<1, "sub", "add">; def : InstAlias<"mov $dst, $src", - (ADDWri GPR32sponly:$dst, GPR32sp:$src, 0, 0)>; + (ADDWri GPR32sponly:$dst, GPR32sp:$src, + (addsub_shifted_imm32 0, 0))>; def : InstAlias<"mov $dst, $src", - (ADDWri GPR32sp:$dst, GPR32sponly:$src, 0, 0)>; + (ADDWri GPR32sp:$dst, GPR32sponly:$src, + (addsub_shifted_imm32 0, 0))>; def : InstAlias<"mov $dst, $src", - (ADDXri GPR64sponly:$dst, GPR64sp:$src, 0, 0)>; + (ADDXri GPR64sponly:$dst, GPR64sp:$src, + (addsub_shifted_imm64 0, 0))>; def : InstAlias<"mov $dst, $src", - (ADDXri GPR64sp:$dst, GPR64sponly:$src, 0, 0)>; + (ADDXri GPR64sp:$dst, GPR64sponly:$src, + (addsub_shifted_imm64 0, 0))>; defm ADDS : AddSubS<0, "adds", AArch64add_flag, "cmn", "subs", "cmp">; defm SUBS : AddSubS<1, "subs", AArch64sub_flag, "cmp", "adds", "cmn">; @@ -2726,19 +2741,31 @@ def : Pat<(AArch64sub_flag GPR64:$Rn, neg_addsub_shifted_imm64:$imm), (ADDSXri GPR64:$Rn, neg_addsub_shifted_imm64:$imm)>; } -def : InstAlias<"neg $dst, $src", (SUBWrs GPR32:$dst, WZR, GPR32:$src, 0), 3>; -def : InstAlias<"neg $dst, $src", (SUBXrs GPR64:$dst, XZR, GPR64:$src, 0), 3>; +def : InstAlias<"neg $dst, $src", + (SUBWrs GPR32:$dst, WZR, + (arith_shifted_reg32 GPR32:$src, 0)), 3>; +def : InstAlias<"neg $dst, $src", + (SUBXrs GPR64:$dst, XZR, + (arith_shifted_reg64 GPR64:$src, 0)), 3>; def : InstAlias<"neg $dst, $src$shift", - (SUBWrs GPR32:$dst, WZR, GPR32:$src, arith_shift32:$shift), 2>; + (SUBWrs GPR32:$dst, WZR, + (arith_shifted_reg32 GPR32:$src, arith_shift32:$shift)), 2>; def : InstAlias<"neg $dst, $src$shift", - (SUBXrs GPR64:$dst, XZR, GPR64:$src, arith_shift64:$shift), 2>; - -def : InstAlias<"negs $dst, $src", (SUBSWrs GPR32:$dst, WZR, GPR32:$src, 0), 3>; -def : InstAlias<"negs $dst, $src", (SUBSXrs GPR64:$dst, XZR, GPR64:$src, 0), 3>; + (SUBXrs GPR64:$dst, XZR, + (arith_shifted_reg64 GPR64:$src, arith_shift64:$shift)), 2>; + +def : InstAlias<"negs $dst, $src", + (SUBSWrs GPR32:$dst, WZR, + (arith_shifted_reg32 GPR32:$src, 0)), 3>; +def : InstAlias<"negs $dst, $src", + (SUBSXrs GPR64:$dst, XZR, + (arith_shifted_reg64 GPR64:$src, 0)), 3>; def : InstAlias<"negs $dst, $src$shift", - (SUBSWrs GPR32:$dst, WZR, GPR32:$src, arith_shift32:$shift), 2>; + (SUBSWrs GPR32:$dst, WZR, + (arith_shifted_reg32 GPR32:$src, arith_shift32:$shift)), 2>; def : InstAlias<"negs $dst, $src$shift", - (SUBSXrs GPR64:$dst, XZR, GPR64:$src, arith_shift64:$shift), 2>; + (SUBSXrs GPR64:$dst, XZR, + (arith_shifted_reg64 GPR64:$src, arith_shift64:$shift)), 2>; // Unsigned/Signed divide @@ -3165,16 +3192,26 @@ defm ORN : LogicalReg<0b01, 1, "orn", BinOpFrag<(or node:$LHS, (not node:$RHS))>>; defm ORR : LogicalReg<0b01, 0, "orr", or>; -def : InstAlias<"mov $dst, $src", (ORRWrs GPR32:$dst, WZR, GPR32:$src, 0), 2>; -def : InstAlias<"mov $dst, $src", (ORRXrs GPR64:$dst, XZR, GPR64:$src, 0), 2>; - -def : InstAlias<"mvn $Wd, $Wm", (ORNWrs GPR32:$Wd, WZR, GPR32:$Wm, 0), 3>; -def : InstAlias<"mvn $Xd, $Xm", (ORNXrs GPR64:$Xd, XZR, GPR64:$Xm, 0), 3>; +def : InstAlias<"mov $dst, $src", + (ORRWrs GPR32:$dst, WZR, + (logical_shifted_reg32 GPR32:$src, 0)), 2>; +def : InstAlias<"mov $dst, $src", + (ORRXrs GPR64:$dst, XZR, + (logical_shifted_reg64 GPR64:$src, 0)), 2>; + +def : InstAlias<"mvn $Wd, $Wm", + (ORNWrs GPR32:$Wd, WZR, + (logical_shifted_reg32 GPR32:$Wm, 0)), 3>; +def : InstAlias<"mvn $Xd, $Xm", + (ORNXrs GPR64:$Xd, XZR, + (logical_shifted_reg64 GPR64:$Xm, 0)), 3>; def : InstAlias<"mvn $Wd, $Wm$sh", - (ORNWrs GPR32:$Wd, WZR, GPR32:$Wm, logical_shift32:$sh), 2>; + (ORNWrs GPR32:$Wd, WZR, + (logical_shifted_reg32 GPR32:$Wm, logical_shift32:$sh)), 2>; def : InstAlias<"mvn $Xd, $Xm$sh", - (ORNXrs GPR64:$Xd, XZR, GPR64:$Xm, logical_shift64:$sh), 2>; + (ORNXrs GPR64:$Xd, XZR, + (logical_shifted_reg64 GPR64:$Xm, logical_shift64:$sh)), 2>; def : InstAlias<"tst $src1, $src2", (ANDSWri WZR, GPR32:$src1, logical_imm32:$src2), 2>; @@ -3182,14 +3219,18 @@ def : InstAlias<"tst $src1, $src2", (ANDSXri XZR, GPR64:$src1, logical_imm64:$src2), 2>; def : InstAlias<"tst $src1, $src2", - (ANDSWrs WZR, GPR32:$src1, GPR32:$src2, 0), 3>; + (ANDSWrs WZR, GPR32:$src1, + (logical_shifted_reg32 GPR32:$src2, 0)), 3>; def : InstAlias<"tst $src1, $src2", - (ANDSXrs XZR, GPR64:$src1, GPR64:$src2, 0), 3>; + (ANDSXrs XZR, GPR64:$src1, + (logical_shifted_reg64 GPR64:$src2, 0)), 3>; def : InstAlias<"tst $src1, $src2$sh", - (ANDSWrs WZR, GPR32:$src1, GPR32:$src2, logical_shift32:$sh), 2>; + (ANDSWrs WZR, GPR32:$src1, + (logical_shifted_reg32 GPR32:$src2, logical_shift32:$sh)), 2>; def : InstAlias<"tst $src1, $src2$sh", - (ANDSXrs XZR, GPR64:$src1, GPR64:$src2, logical_shift64:$sh), 2>; + (ANDSXrs XZR, GPR64:$src1, + (logical_shifted_reg64 GPR64:$src2, logical_shift64:$sh)), 2>; def : Pat<(not GPR32:$Wm), (ORNWrr WZR, GPR32:$Wm)>; @@ -4710,6 +4751,26 @@ let Predicates = [IsLE] in { (STRQui FPR128:$Rt, GPR64sp:$Rn, uimm12s16:$offset)>; } +// truncstorei32 of f64 bitcasted to i64 +def : Pat<(truncstorei32 (i64 (bitconvert (f64 FPR64:$Rt))), (am_indexed32 GPR64sp:$Rn, uimm12s4:$offset)), + (STRSui (EXTRACT_SUBREG FPR64:$Rt, ssub), GPR64sp:$Rn, uimm12s4:$offset)>; + +// truncstorei16 of f64 bitcasted to i64 +def : Pat<(truncstorei16 (i64 (bitconvert (f64 FPR64:$Rt))), (am_indexed16 GPR64sp:$Rn, uimm12s2:$offset)), + (STRHui (f16 (EXTRACT_SUBREG FPR64:$Rt, hsub)), GPR64sp:$Rn, uimm12s2:$offset)>; + + // truncstorei16 of f32 bitcasted to i32 +def : Pat<(truncstorei16 (i32 (bitconvert (f32 FPR32:$Rt))), (am_indexed16 GPR64sp:$Rn, uimm12s2:$off)), + (STRHui (f16 (EXTRACT_SUBREG FPR32:$Rt, hsub)), GPR64sp:$Rn, uimm12s2:$off)>; + + // truncstorei8 of f64 bitcasted to i64 +def : Pat<(truncstorei8 (i64 (bitconvert (f64 FPR64:$Rt))), (am_indexed8 GPR64sp:$Rn, uimm12s1:$off)), + (STRBui (aarch64mfp8 (EXTRACT_SUBREG FPR64:$Rt, bsub)), GPR64sp:$Rn, uimm12s1:$off)>; + + // truncstorei8 of f32 bitcasted to i32 +def : Pat<(truncstorei8 (i32 (bitconvert (f32 FPR32:$Rt))), (am_indexed8 GPR64sp:$Rn, uimm12s1:$off)), + (STRBui (aarch64mfp8 (EXTRACT_SUBREG FPR32:$Rt, bsub)), GPR64sp:$Rn, uimm12s1:$off)>; + // truncstore i64 def : Pat<(truncstorei32 GPR64:$Rt, (am_indexed32 GPR64sp:$Rn, uimm12s4:$offset)), @@ -6536,9 +6597,33 @@ defm UQXTN : SIMDTwoScalarMixedBHS<1, 0b10100, "uqxtn", int_aarch64_neon_scalar defm USQADD : SIMDTwoScalarBHSDTied< 1, 0b00011, "usqadd", int_aarch64_neon_usqadd>; +// f16 -> s16 conversions +let Predicates = [HasFullFP16] in { + def : Pat<(i16(fp_to_sint_sat_gi f16:$Rn)), (FCVTZSv1f16 f16:$Rn)>; + def : Pat<(i16(fp_to_uint_sat_gi f16:$Rn)), (FCVTZUv1f16 f16:$Rn)>; +} + def : Pat<(v1i64 (AArch64vashr (v1i64 V64:$Rn), (i32 63))), (CMLTv1i64rz V64:$Rn)>; +// f16 -> i16 conversions leave the bit pattern in a f32 +class F16ToI16ScalarPat<SDNode cvt_isd, BaseSIMDTwoScalar instr> + : Pat<(f32 (cvt_isd (f16 FPR16:$Rn))), + (f32 (SUBREG_TO_REG (i64 0), (instr FPR16:$Rn), hsub))>; + +let Predicates = [HasFullFP16] in { +def : F16ToI16ScalarPat<AArch64fcvtzs_half, FCVTZSv1f16>; +def : F16ToI16ScalarPat<AArch64fcvtzu_half, FCVTZUv1f16>; +def : F16ToI16ScalarPat<AArch64fcvtas_half, FCVTASv1f16>; +def : F16ToI16ScalarPat<AArch64fcvtau_half, FCVTAUv1f16>; +def : F16ToI16ScalarPat<AArch64fcvtms_half, FCVTMSv1f16>; +def : F16ToI16ScalarPat<AArch64fcvtmu_half, FCVTMUv1f16>; +def : F16ToI16ScalarPat<AArch64fcvtns_half, FCVTNSv1f16>; +def : F16ToI16ScalarPat<AArch64fcvtnu_half, FCVTNUv1f16>; +def : F16ToI16ScalarPat<AArch64fcvtps_half, FCVTPSv1f16>; +def : F16ToI16ScalarPat<AArch64fcvtpu_half, FCVTPUv1f16>; +} + // Round FP64 to BF16. let Predicates = [HasNEONandIsStreamingSafe, HasBF16] in def : Pat<(bf16 (any_fpround (f64 FPR64:$Rn))), @@ -6641,20 +6726,24 @@ def : Pat<(v2f64 (AArch64frsqrts (v2f64 FPR128:$Rn), (v2f64 FPR128:$Rm))), // Some float -> int -> float conversion patterns for which we want to keep the // int values in FP registers using the corresponding NEON instructions to // avoid more costly int <-> fp register transfers. +let HasOneUse = 1 in { +def any_fp_to_sint_oneuse: PatFrag<(ops node:$src0), (any_fp_to_sint $src0)>; +def any_fp_to_uint_oneuse: PatFrag<(ops node:$src0), (any_fp_to_uint $src0)>; +} let Predicates = [HasNEONandIsSME2p2StreamingSafe] in { -def : Pat<(f64 (any_sint_to_fp (i64 (any_fp_to_sint f64:$Rn)))), +def : Pat<(f64 (any_sint_to_fp (i64 (any_fp_to_sint_oneuse f64:$Rn)))), (SCVTFv1i64 (i64 (FCVTZSv1i64 f64:$Rn)))>; -def : Pat<(f32 (any_sint_to_fp (i32 (any_fp_to_sint f32:$Rn)))), +def : Pat<(f32 (any_sint_to_fp (i32 (any_fp_to_sint_oneuse f32:$Rn)))), (SCVTFv1i32 (i32 (FCVTZSv1i32 f32:$Rn)))>; -def : Pat<(f64 (any_uint_to_fp (i64 (any_fp_to_uint f64:$Rn)))), +def : Pat<(f64 (any_uint_to_fp (i64 (any_fp_to_uint_oneuse f64:$Rn)))), (UCVTFv1i64 (i64 (FCVTZUv1i64 f64:$Rn)))>; -def : Pat<(f32 (any_uint_to_fp (i32 (any_fp_to_uint f32:$Rn)))), +def : Pat<(f32 (any_uint_to_fp (i32 (any_fp_to_uint_oneuse f32:$Rn)))), (UCVTFv1i32 (i32 (FCVTZUv1i32 f32:$Rn)))>; let Predicates = [HasNEONandIsSME2p2StreamingSafe, HasFullFP16] in { -def : Pat<(f16 (any_sint_to_fp (i32 (any_fp_to_sint f16:$Rn)))), +def : Pat<(f16 (any_sint_to_fp (i32 (any_fp_to_sint_oneuse f16:$Rn)))), (SCVTFv1i16 (f16 (FCVTZSv1f16 f16:$Rn)))>; -def : Pat<(f16 (any_uint_to_fp (i32 (any_fp_to_uint f16:$Rn)))), +def : Pat<(f16 (any_uint_to_fp (i32 (any_fp_to_uint_oneuse f16:$Rn)))), (UCVTFv1i16 (f16 (FCVTZUv1f16 f16:$Rn)))>; } @@ -8234,6 +8323,29 @@ def MVNIv4s_msl : SIMDModifiedImmMoveMSL<1, 1, {1,1,0,?}, V128, "mvni", ".4s", (AArch64mvni_msl imm0_255:$imm8, (i32 imm:$shift)))]>; } +// SABA patterns for add(x, abs(y)) -> saba(x, y, 0) +def : Pat<(v8i8 (add V64:$Vn, (abs V64:$Vm))), + (SABAv8i8 V64:$Vn, V64:$Vm, (EXTRACT_SUBREG (MOVIv2d_ns (i32 0)), dsub))>; +def : Pat<(v4i16 (add V64:$Vn, (abs V64:$Vm))), + (SABAv4i16 V64:$Vn, V64:$Vm, (EXTRACT_SUBREG (MOVIv2d_ns (i32 0)), dsub))>; +def : Pat<(v2i32 (add V64:$Vn, (abs V64:$Vm))), + (SABAv2i32 V64:$Vn, V64:$Vm, (EXTRACT_SUBREG (MOVIv2d_ns (i32 0)), dsub))>; +def : Pat<(v16i8 (add V128:$Vn, (abs V128:$Vm))), + (SABAv16i8 V128:$Vn, V128:$Vm, (MOVIv2d_ns (i32 0)))>; +def : Pat<(v8i16 (add V128:$Vn, (abs V128:$Vm))), + (SABAv8i16 V128:$Vn, V128:$Vm, (MOVIv2d_ns (i32 0)))>; +def : Pat<(v4i32 (add V128:$Vn, (abs V128:$Vm))), + (SABAv4i32 V128:$Vn, V128:$Vm, (MOVIv2d_ns (i32 0)))>; + +// SABAL patterns for add(x, zext(abs(y))) -> sabal(x, y, 0) +def : Pat<(v8i16 (add V128:$Vn, (zext (abs (v8i8 V64:$Vm))))), + (SABALv8i8_v8i16 V128:$Vn, V64:$Vm, (EXTRACT_SUBREG (MOVIv2d_ns (i32 0)), dsub))>; +def : Pat<(v4i32 (add V128:$Vn, (zext (abs (v4i16 V64:$Vm))))), + (SABALv4i16_v4i32 V128:$Vn, V64:$Vm, (EXTRACT_SUBREG (MOVIv2d_ns (i32 0)), dsub))>; +def : Pat<(v2i64 (add V128:$Vn, (zext (abs (v2i32 V64:$Vm))))), + (SABALv2i32_v2i64 V128:$Vn, V64:$Vm, (EXTRACT_SUBREG (MOVIv2d_ns (i32 0)), dsub))>; + + //---------------------------------------------------------------------------- // AdvSIMD indexed element //---------------------------------------------------------------------------- diff --git a/llvm/lib/Target/AArch64/AArch64MachineFunctionInfo.h b/llvm/lib/Target/AArch64/AArch64MachineFunctionInfo.h index 1fde87e65a34..993cff112ba8 100644 --- a/llvm/lib/Target/AArch64/AArch64MachineFunctionInfo.h +++ b/llvm/lib/Target/AArch64/AArch64MachineFunctionInfo.h @@ -228,9 +228,6 @@ class AArch64FunctionInfo final : public MachineFunctionInfo { // on function entry to record the initial pstate of a function. Register PStateSMReg = MCRegister::NoRegister; - // true if PStateSMReg is used. - bool PStateSMRegUsed = false; - // Has the PNReg used to build PTRUE instruction. // The PTRUE is used for the LD/ST of ZReg pairs in save and restore. unsigned PredicateRegForFillSpill = 0; @@ -238,6 +235,10 @@ class AArch64FunctionInfo final : public MachineFunctionInfo { // Holds the SME function attributes (streaming mode, ZA/ZT0 state). SMEAttrs SMEFnAttrs; + // Holds the TPIDR2 block if allocated early (for Windows/stack probes + // support). + Register EarlyAllocSMESaveBuffer = AArch64::NoRegister; + // Note: The following properties are only used for the old SME ABI lowering: /// The frame-index for the TPIDR2 object used for lazy saves. TPIDR2Object TPIDR2; @@ -256,6 +257,14 @@ public: const DenseMap<MachineBasicBlock *, MachineBasicBlock *> &Src2DstMBB) const override; + void setEarlyAllocSMESaveBuffer(Register Ptr) { + EarlyAllocSMESaveBuffer = Ptr; + } + + Register getEarlyAllocSMESaveBuffer() const { + return EarlyAllocSMESaveBuffer; + } + // Old SME ABI lowering state getters/setters: Register getSMESaveBufferAddr() const { return SMESaveBufferAddr; }; void setSMESaveBufferAddr(Register Reg) { SMESaveBufferAddr = Reg; }; @@ -273,9 +282,6 @@ public: Register getPStateSMReg() const { return PStateSMReg; }; void setPStateSMReg(Register Reg) { PStateSMReg = Reg; }; - unsigned isPStateSMRegUsed() const { return PStateSMRegUsed; }; - void setPStateSMRegUsed(bool Used = true) { PStateSMRegUsed = Used; }; - bool isSVECC() const { return IsSVECC; }; void setIsSVECC(bool s) { IsSVECC = s; }; diff --git a/llvm/lib/Target/AArch64/AArch64MacroFusion.cpp b/llvm/lib/Target/AArch64/AArch64MacroFusion.cpp index ff7a0d1faedf..f4a7f774d477 100644 --- a/llvm/lib/Target/AArch64/AArch64MacroFusion.cpp +++ b/llvm/lib/Target/AArch64/AArch64MacroFusion.cpp @@ -237,8 +237,8 @@ static bool isAddressLdStPair(const MachineInstr *FirstMI, } /// Compare and conditional select. -static bool isCCSelectPair(const MachineInstr *FirstMI, - const MachineInstr &SecondMI) { +static bool isCmpCSelPair(const MachineInstr *FirstMI, + const MachineInstr &SecondMI) { // 32 bits if (SecondMI.getOpcode() == AArch64::CSELWr) { // Assume the 1st instr to be a wildcard if it is unspecified. @@ -279,6 +279,40 @@ static bool isCCSelectPair(const MachineInstr *FirstMI, return false; } +/// Compare and cset. +static bool isCmpCSetPair(const MachineInstr *FirstMI, + const MachineInstr &SecondMI) { + if ((SecondMI.getOpcode() == AArch64::CSINCWr && + SecondMI.getOperand(1).getReg() == AArch64::WZR && + SecondMI.getOperand(2).getReg() == AArch64::WZR) || + (SecondMI.getOpcode() == AArch64::CSINCXr && + SecondMI.getOperand(1).getReg() == AArch64::XZR && + SecondMI.getOperand(2).getReg() == AArch64::XZR)) { + // Assume the 1st instr to be a wildcard if it is unspecified. + if (FirstMI == nullptr) + return true; + + if (FirstMI->definesRegister(AArch64::WZR, /*TRI=*/nullptr) || + FirstMI->definesRegister(AArch64::XZR, /*TRI=*/nullptr)) + switch (FirstMI->getOpcode()) { + case AArch64::SUBSWrs: + case AArch64::SUBSXrs: + return !AArch64InstrInfo::hasShiftedReg(*FirstMI); + case AArch64::SUBSWrx: + case AArch64::SUBSXrx: + case AArch64::SUBSXrx64: + return !AArch64InstrInfo::hasExtendedReg(*FirstMI); + case AArch64::SUBSWri: + case AArch64::SUBSWrr: + case AArch64::SUBSXri: + case AArch64::SUBSXrr: + return true; + } + } + + return false; +} + // Arithmetic and logic. static bool isArithmeticLogicPair(const MachineInstr *FirstMI, const MachineInstr &SecondMI) { @@ -465,7 +499,9 @@ static bool shouldScheduleAdjacent(const TargetInstrInfo &TII, return true; if (ST.hasFuseAddress() && isAddressLdStPair(FirstMI, SecondMI)) return true; - if (ST.hasFuseCCSelect() && isCCSelectPair(FirstMI, SecondMI)) + if (ST.hasFuseCmpCSel() && isCmpCSelPair(FirstMI, SecondMI)) + return true; + if (ST.hasFuseCmpCSet() && isCmpCSetPair(FirstMI, SecondMI)) return true; if (ST.hasFuseArithmeticLogic() && isArithmeticLogicPair(FirstMI, SecondMI)) return true; diff --git a/llvm/lib/Target/AArch64/AArch64Processors.td b/llvm/lib/Target/AArch64/AArch64Processors.td index 42eaeca906e6..81f5d075729d 100644 --- a/llvm/lib/Target/AArch64/AArch64Processors.td +++ b/llvm/lib/Target/AArch64/AArch64Processors.td @@ -134,6 +134,8 @@ def TuneA78 : SubtargetFeature<"a78", "ARMProcFamily", "CortexA78", FeatureCmpBccFusion, FeatureFuseAES, FeatureFuseAdrpAdd, + FeatureFuseCmpCSel, + FeatureFuseCmpCSet, FeatureAddrLSLSlow14, FeatureALULSLFast, FeaturePostRAScheduler, @@ -146,6 +148,8 @@ def TuneA78AE : SubtargetFeature<"a78ae", "ARMProcFamily", FeatureCmpBccFusion, FeatureFuseAES, FeatureFuseAdrpAdd, + FeatureFuseCmpCSel, + FeatureFuseCmpCSet, FeatureAddrLSLSlow14, FeatureALULSLFast, FeaturePostRAScheduler, @@ -158,6 +162,8 @@ def TuneA78C : SubtargetFeature<"a78c", "ARMProcFamily", FeatureCmpBccFusion, FeatureFuseAES, FeatureFuseAdrpAdd, + FeatureFuseCmpCSel, + FeatureFuseCmpCSet, FeatureAddrLSLSlow14, FeatureALULSLFast, FeaturePostRAScheduler, @@ -169,6 +175,8 @@ def TuneA710 : SubtargetFeature<"a710", "ARMProcFamily", "CortexA710", FeatureCmpBccFusion, FeatureFuseAES, FeatureFuseAdrpAdd, + FeatureFuseCmpCSel, + FeatureFuseCmpCSet, FeatureALULSLFast, FeaturePostRAScheduler, FeatureEnableSelectOptimize, @@ -181,6 +189,8 @@ def TuneA715 : SubtargetFeature<"a715", "ARMProcFamily", "CortexA715", FeatureCmpBccFusion, FeatureALULSLFast, FeatureFuseAdrpAdd, + FeatureFuseCmpCSel, + FeatureFuseCmpCSet, FeatureEnableSelectOptimize, FeaturePredictableSelectIsExpensive]>; @@ -191,6 +201,8 @@ def TuneA720 : SubtargetFeature<"a720", "ARMProcFamily", "CortexA720", FeatureCmpBccFusion, FeatureALULSLFast, FeatureFuseAdrpAdd, + FeatureFuseCmpCSel, + FeatureFuseCmpCSet, FeatureEnableSelectOptimize, FeaturePredictableSelectIsExpensive]>; @@ -201,6 +213,8 @@ def TuneA720AE : SubtargetFeature<"a720ae", "ARMProcFamily", "CortexA720", FeatureCmpBccFusion, FeatureALULSLFast, FeatureFuseAdrpAdd, + FeatureFuseCmpCSel, + FeatureFuseCmpCSet, FeatureEnableSelectOptimize, FeaturePredictableSelectIsExpensive]>; @@ -212,6 +226,8 @@ def TuneA725 : SubtargetFeature<"cortex-a725", "ARMProcFamily", FeatureCmpBccFusion, FeatureALULSLFast, FeatureFuseAdrpAdd, + FeatureFuseCmpCSel, + FeatureFuseCmpCSet, FeatureEnableSelectOptimize, FeaturePredictableSelectIsExpensive]>; @@ -262,6 +278,8 @@ def TuneX4 : SubtargetFeature<"cortex-x4", "ARMProcFamily", "CortexX4", "Cortex-X4 ARM processors", [ FeatureALULSLFast, FeatureFuseAdrpAdd, + FeatureFuseCmpCSel, + FeatureFuseCmpCSet, FeatureFuseAES, FeaturePostRAScheduler, FeatureEnableSelectOptimize, @@ -273,6 +291,8 @@ def TuneX925 : SubtargetFeature<"cortex-x925", "ARMProcFamily", "CortexX925", "Cortex-X925 ARM processors",[ FeatureALULSLFast, FeatureFuseAdrpAdd, + FeatureFuseCmpCSel, + FeatureFuseCmpCSet, FeatureFuseAES, FeaturePostRAScheduler, FeatureEnableSelectOptimize, @@ -321,7 +341,11 @@ def TuneAppleA7 : SubtargetFeature<"apple-a7", "ARMProcFamily", "AppleA7", FeatureFuseAES, FeatureFuseCryptoEOR, FeatureStorePairSuppress, FeatureZCRegMoveGPR64, - FeatureZCZeroing, + FeatureZCRegMoveFPR128, + FeatureZCZeroingGPR32, + FeatureZCZeroingGPR64, + FeatureNoZCZeroingFPR64, + FeatureZCZeroingFPR128, FeatureZCZeroingFPWorkaround]>; def TuneAppleA10 : SubtargetFeature<"apple-a10", "ARMProcFamily", "AppleA10", @@ -334,7 +358,11 @@ def TuneAppleA10 : SubtargetFeature<"apple-a10", "ARMProcFamily", "AppleA10", FeatureFuseCryptoEOR, FeatureStorePairSuppress, FeatureZCRegMoveGPR64, - FeatureZCZeroing]>; + FeatureZCRegMoveFPR128, + FeatureZCZeroingGPR32, + FeatureZCZeroingGPR64, + FeatureNoZCZeroingFPR64, + FeatureZCZeroingFPR128]>; def TuneAppleA11 : SubtargetFeature<"apple-a11", "ARMProcFamily", "AppleA11", "Apple A11", [ @@ -346,7 +374,11 @@ def TuneAppleA11 : SubtargetFeature<"apple-a11", "ARMProcFamily", "AppleA11", FeatureFuseCryptoEOR, FeatureStorePairSuppress, FeatureZCRegMoveGPR64, - FeatureZCZeroing]>; + FeatureZCRegMoveFPR128, + FeatureZCZeroingGPR32, + FeatureZCZeroingGPR64, + FeatureNoZCZeroingFPR64, + FeatureZCZeroingFPR128]>; def TuneAppleA12 : SubtargetFeature<"apple-a12", "ARMProcFamily", "AppleA12", "Apple A12", [ @@ -358,7 +390,11 @@ def TuneAppleA12 : SubtargetFeature<"apple-a12", "ARMProcFamily", "AppleA12", FeatureFuseCryptoEOR, FeatureStorePairSuppress, FeatureZCRegMoveGPR64, - FeatureZCZeroing]>; + FeatureZCRegMoveFPR128, + FeatureZCZeroingGPR32, + FeatureZCZeroingGPR64, + FeatureNoZCZeroingFPR64, + FeatureZCZeroingFPR128]>; def TuneAppleA13 : SubtargetFeature<"apple-a13", "ARMProcFamily", "AppleA13", "Apple A13", [ @@ -370,7 +406,11 @@ def TuneAppleA13 : SubtargetFeature<"apple-a13", "ARMProcFamily", "AppleA13", FeatureFuseCryptoEOR, FeatureStorePairSuppress, FeatureZCRegMoveGPR64, - FeatureZCZeroing]>; + FeatureZCRegMoveFPR128, + FeatureZCZeroingGPR32, + FeatureZCZeroingGPR64, + FeatureNoZCZeroingFPR64, + FeatureZCZeroingFPR128]>; def TuneAppleA14 : SubtargetFeature<"apple-a14", "ARMProcFamily", "AppleA14", "Apple A14", [ @@ -382,12 +422,16 @@ def TuneAppleA14 : SubtargetFeature<"apple-a14", "ARMProcFamily", "AppleA14", FeatureFuseAddress, FeatureFuseAES, FeatureFuseArithmeticLogic, - FeatureFuseCCSelect, + FeatureFuseCmpCSel, FeatureFuseCryptoEOR, FeatureFuseLiterals, FeatureStorePairSuppress, FeatureZCRegMoveGPR64, - FeatureZCZeroing]>; + FeatureZCRegMoveFPR128, + FeatureZCZeroingGPR32, + FeatureZCZeroingGPR64, + FeatureNoZCZeroingFPR64, + FeatureZCZeroingFPR128]>; def TuneAppleA15 : SubtargetFeature<"apple-a15", "ARMProcFamily", "AppleA15", "Apple A15", [ @@ -399,12 +443,16 @@ def TuneAppleA15 : SubtargetFeature<"apple-a15", "ARMProcFamily", "AppleA15", FeatureFuseAdrpAdd, FeatureFuseAES, FeatureFuseArithmeticLogic, - FeatureFuseCCSelect, + FeatureFuseCmpCSel, FeatureFuseCryptoEOR, FeatureFuseLiterals, FeatureStorePairSuppress, FeatureZCRegMoveGPR64, - FeatureZCZeroing]>; + FeatureZCRegMoveFPR128, + FeatureZCZeroingGPR32, + FeatureZCZeroingGPR64, + FeatureNoZCZeroingFPR64, + FeatureZCZeroingFPR128]>; def TuneAppleA16 : SubtargetFeature<"apple-a16", "ARMProcFamily", "AppleA16", "Apple A16", [ @@ -416,12 +464,16 @@ def TuneAppleA16 : SubtargetFeature<"apple-a16", "ARMProcFamily", "AppleA16", FeatureFuseAdrpAdd, FeatureFuseAES, FeatureFuseArithmeticLogic, - FeatureFuseCCSelect, + FeatureFuseCmpCSel, FeatureFuseCryptoEOR, FeatureFuseLiterals, FeatureStorePairSuppress, FeatureZCRegMoveGPR64, - FeatureZCZeroing]>; + FeatureZCRegMoveFPR128, + FeatureZCZeroingGPR32, + FeatureZCZeroingGPR64, + FeatureNoZCZeroingFPR64, + FeatureZCZeroingFPR128]>; def TuneAppleA17 : SubtargetFeature<"apple-a17", "ARMProcFamily", "AppleA17", "Apple A17", [ @@ -433,12 +485,16 @@ def TuneAppleA17 : SubtargetFeature<"apple-a17", "ARMProcFamily", "AppleA17", FeatureFuseAdrpAdd, FeatureFuseAES, FeatureFuseArithmeticLogic, - FeatureFuseCCSelect, + FeatureFuseCmpCSel, FeatureFuseCryptoEOR, FeatureFuseLiterals, FeatureStorePairSuppress, FeatureZCRegMoveGPR64, - FeatureZCZeroing]>; + FeatureZCRegMoveFPR128, + FeatureZCZeroingGPR32, + FeatureZCZeroingGPR64, + FeatureNoZCZeroingFPR64, + FeatureZCZeroingFPR128]>; def TuneAppleM4 : SubtargetFeature<"apple-m4", "ARMProcFamily", "AppleM4", "Apple M4", [ @@ -450,12 +506,15 @@ def TuneAppleM4 : SubtargetFeature<"apple-m4", "ARMProcFamily", "AppleM4", FeatureFuseAdrpAdd, FeatureFuseAES, FeatureFuseArithmeticLogic, - FeatureFuseCCSelect, + FeatureFuseCmpCSel, FeatureFuseCryptoEOR, FeatureFuseLiterals, FeatureZCRegMoveGPR64, - FeatureZCZeroing - ]>; + FeatureZCRegMoveFPR128, + FeatureZCZeroingGPR32, + FeatureZCZeroingGPR64, + FeatureNoZCZeroingFPR64, + FeatureZCZeroingFPR128]>; def TuneExynosM3 : SubtargetFeature<"exynosm3", "ARMProcFamily", "ExynosM3", "Samsung Exynos-M3 processors", @@ -463,7 +522,7 @@ def TuneExynosM3 : SubtargetFeature<"exynosm3", "ARMProcFamily", "ExynosM3", FeatureForce32BitJumpTables, FeatureFuseAddress, FeatureFuseAES, - FeatureFuseCCSelect, + FeatureFuseCmpCSel, FeatureFuseAdrpAdd, FeatureFuseLiterals, FeatureStorePairSuppress, @@ -481,19 +540,21 @@ def TuneExynosM4 : SubtargetFeature<"exynosm4", "ARMProcFamily", "ExynosM3", FeatureFuseAddress, FeatureFuseAES, FeatureFuseArithmeticLogic, - FeatureFuseCCSelect, + FeatureFuseCmpCSel, FeatureFuseAdrpAdd, FeatureFuseLiterals, FeatureStorePairSuppress, FeatureALULSLFast, FeaturePostRAScheduler, - FeatureZCZeroing]>; + FeatureZCZeroingGPR32, + FeatureZCZeroingGPR64]>; def TuneKryo : SubtargetFeature<"kryo", "ARMProcFamily", "Kryo", "Qualcomm Kryo processors", [ FeaturePostRAScheduler, FeaturePredictableSelectIsExpensive, - FeatureZCZeroing, + FeatureZCZeroingGPR32, + FeatureZCZeroingGPR64, FeatureALULSLFast, FeatureStorePairSuppress]>; @@ -501,7 +562,8 @@ def TuneFalkor : SubtargetFeature<"falkor", "ARMProcFamily", "Falkor", "Qualcomm Falkor processors", [ FeaturePostRAScheduler, FeaturePredictableSelectIsExpensive, - FeatureZCZeroing, + FeatureZCZeroingGPR32, + FeatureZCZeroingGPR64, FeatureStorePairSuppress, FeatureALULSLFast, FeatureSlowSTRQro]>; @@ -526,6 +588,8 @@ def TuneNeoverseN2 : SubtargetFeature<"neoversen2", "ARMProcFamily", "NeoverseN2 "Neoverse N2 ARM processors", [ FeatureFuseAES, FeatureFuseAdrpAdd, + FeatureFuseCmpCSel, + FeatureFuseCmpCSet, FeatureALULSLFast, FeaturePostRAScheduler, FeatureEnableSelectOptimize, @@ -537,6 +601,8 @@ def TuneNeoverseN3 : SubtargetFeature<"neoversen3", "ARMProcFamily", "NeoverseN3 FeaturePostRAScheduler, FeatureALULSLFast, FeatureFuseAdrpAdd, + FeatureFuseCmpCSel, + FeatureFuseCmpCSet, FeatureEnableSelectOptimize, FeaturePredictableSelectIsExpensive]>; @@ -553,6 +619,8 @@ def TuneNeoverseV1 : SubtargetFeature<"neoversev1", "ARMProcFamily", "NeoverseV1 "Neoverse V1 ARM processors", [ FeatureFuseAES, FeatureFuseAdrpAdd, + FeatureFuseCmpCSel, + FeatureFuseCmpCSet, FeatureAddrLSLSlow14, FeatureALULSLFast, FeaturePostRAScheduler, @@ -565,6 +633,8 @@ def TuneNeoverseV2 : SubtargetFeature<"neoversev2", "ARMProcFamily", "NeoverseV2 FeatureFuseAES, FeatureCmpBccFusion, FeatureFuseAdrpAdd, + FeatureFuseCmpCSel, + FeatureFuseCmpCSet, FeatureALULSLFast, FeaturePostRAScheduler, FeatureEnableSelectOptimize, @@ -578,6 +648,8 @@ def TuneNeoverseV3 : SubtargetFeature<"neoversev3", "ARMProcFamily", "NeoverseV3 FeatureFuseAES, FeatureALULSLFast, FeatureFuseAdrpAdd, + FeatureFuseCmpCSel, + FeatureFuseCmpCSet, FeaturePostRAScheduler, FeatureEnableSelectOptimize, FeatureAvoidLDAPUR, @@ -588,6 +660,8 @@ def TuneNeoverseV3AE : SubtargetFeature<"neoversev3AE", "ARMProcFamily", "Neover FeatureFuseAES, FeatureALULSLFast, FeatureFuseAdrpAdd, + FeatureFuseCmpCSel, + FeatureFuseCmpCSet, FeaturePostRAScheduler, FeatureEnableSelectOptimize, FeatureAvoidLDAPUR, @@ -597,7 +671,8 @@ def TuneSaphira : SubtargetFeature<"saphira", "ARMProcFamily", "Saphira", "Qualcomm Saphira processors", [ FeaturePostRAScheduler, FeaturePredictableSelectIsExpensive, - FeatureZCZeroing, + FeatureZCZeroingGPR32, + FeatureZCZeroingGPR64, FeatureStorePairSuppress, FeatureALULSLFast]>; diff --git a/llvm/lib/Target/AArch64/AArch64PrologueEpilogue.cpp b/llvm/lib/Target/AArch64/AArch64PrologueEpilogue.cpp new file mode 100644 index 000000000000..af424987b8dd --- /dev/null +++ b/llvm/lib/Target/AArch64/AArch64PrologueEpilogue.cpp @@ -0,0 +1,794 @@ +//===----------------------------------------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#include "AArch64PrologueEpilogue.h" +#include "AArch64FrameLowering.h" +#include "AArch64MachineFunctionInfo.h" +#include "AArch64Subtarget.h" +#include "MCTargetDesc/AArch64AddressingModes.h" +#include "llvm/ADT/Statistic.h" +#include "llvm/BinaryFormat/Dwarf.h" +#include "llvm/CodeGen/CFIInstBuilder.h" +#include "llvm/MC/MCContext.h" + +#define DEBUG_TYPE "frame-info" + +STATISTIC(NumRedZoneFunctions, "Number of functions using red zone"); + +namespace llvm { + +AArch64PrologueEmitter::AArch64PrologueEmitter(MachineFunction &MF, + MachineBasicBlock &MBB, + const AArch64FrameLowering &AFL) + : MF(MF), MBB(MBB), F(MF.getFunction()), MFI(MF.getFrameInfo()), + Subtarget(MF.getSubtarget<AArch64Subtarget>()), AFL(AFL), + RegInfo(*Subtarget.getRegisterInfo()) { + TII = Subtarget.getInstrInfo(); + AFI = MF.getInfo<AArch64FunctionInfo>(); + + EmitCFI = AFI->needsDwarfUnwindInfo(MF); + EmitAsyncCFI = AFI->needsAsyncDwarfUnwindInfo(MF); + HasFP = AFL.hasFP(MF); + NeedsWinCFI = AFL.needsWinCFI(MF); + IsFunclet = MBB.isEHFuncletEntry(); + HomPrologEpilog = AFL.homogeneousPrologEpilog(MF); + +#ifndef NDEBUG + collectBlockLiveins(); +#endif +} + +#ifndef NDEBUG +/// Collect live registers from the end of \p MI's parent up to (including) \p +/// MI in \p LiveRegs. +static void getLivePhysRegsUpTo(MachineInstr &MI, const TargetRegisterInfo &TRI, + LivePhysRegs &LiveRegs) { + + MachineBasicBlock &MBB = *MI.getParent(); + LiveRegs.addLiveOuts(MBB); + for (const MachineInstr &MI : + reverse(make_range(MI.getIterator(), MBB.instr_end()))) + LiveRegs.stepBackward(MI); +} + +void AArch64PrologueEmitter::collectBlockLiveins() { + // Collect live register from the end of MBB up to the start of the existing + // frame setup instructions. + PrologueEndI = MBB.begin(); + while (PrologueEndI != MBB.end() && + PrologueEndI->getFlag(MachineInstr::FrameSetup)) + ++PrologueEndI; + + if (PrologueEndI != MBB.end()) { + getLivePhysRegsUpTo(*PrologueEndI, RegInfo, LiveRegs); + // Ignore registers used for stack management for now. + LiveRegs.removeReg(AArch64::SP); + LiveRegs.removeReg(AArch64::X19); + LiveRegs.removeReg(AArch64::FP); + LiveRegs.removeReg(AArch64::LR); + + // X0 will be clobbered by a call to __arm_get_current_vg in the prologue. + // This is necessary to spill VG if required where SVE is unavailable, but + // X0 is preserved around this call. + if (AFL.requiresGetVGCall(MF)) + LiveRegs.removeReg(AArch64::X0); + } +} + +void AArch64PrologueEmitter::verifyPrologueClobbers() const { + if (PrologueEndI == MBB.end()) + return; + // Check if any of the newly instructions clobber any of the live registers. + for (MachineInstr &MI : + make_range(MBB.instr_begin(), PrologueEndI->getIterator())) { + for (auto &Op : MI.operands()) + if (Op.isReg() && Op.isDef()) + assert(!LiveRegs.contains(Op.getReg()) && + "live register clobbered by inserted prologue instructions"); + } +} +#endif + +void AArch64PrologueEmitter::determineLocalsStackSize( + uint64_t StackSize, uint64_t PrologueSaveSize) { + AFI->setLocalStackSize(StackSize - PrologueSaveSize); + CombineSPBump = AFL.shouldCombineCSRLocalStackBump(MF, StackSize); +} + +void AArch64PrologueEmitter::emitPrologue() { + const MachineBasicBlock::iterator PrologueBeginI = MBB.begin(); + const MachineBasicBlock::iterator EndI = MBB.end(); + + // At this point, we're going to decide whether or not the function uses a + // redzone. In most cases, the function doesn't have a redzone so let's + // assume that's false and set it to true in the case that there's a redzone. + AFI->setHasRedZone(false); + + // Debug location must be unknown since the first debug location is used + // to determine the end of the prologue. + DebugLoc DL; + + if (AFI->shouldSignReturnAddress(MF)) { + // If pac-ret+leaf is in effect, PAUTH_PROLOGUE pseudo instructions + // are inserted by emitPacRetPlusLeafHardening(). + if (!AFL.shouldSignReturnAddressEverywhere(MF)) { + BuildMI(MBB, PrologueBeginI, DL, TII->get(AArch64::PAUTH_PROLOGUE)) + .setMIFlag(MachineInstr::FrameSetup); + } + // AArch64PointerAuth pass will insert SEH_PACSignLR + HasWinCFI |= NeedsWinCFI; + } + + if (AFI->needsShadowCallStackPrologueEpilogue(MF)) { + emitShadowCallStackPrologue(PrologueBeginI, DL); + HasWinCFI |= NeedsWinCFI; + } + + if (EmitCFI && AFI->isMTETagged()) + BuildMI(MBB, PrologueBeginI, DL, TII->get(AArch64::EMITMTETAGGED)) + .setMIFlag(MachineInstr::FrameSetup); + + // We signal the presence of a Swift extended frame to external tools by + // storing FP with 0b0001 in bits 63:60. In normal userland operation a simple + // ORR is sufficient, it is assumed a Swift kernel would initialize the TBI + // bits so that is still true. + if (HasFP && AFI->hasSwiftAsyncContext()) + emitSwiftAsyncContextFramePointer(PrologueBeginI, DL); + + // All calls are tail calls in GHC calling conv, and functions have no + // prologue/epilogue. + if (MF.getFunction().getCallingConv() == CallingConv::GHC) + return; + + // Set tagged base pointer to the requested stack slot. Ideally it should + // match SP value after prologue. + if (std::optional<int> TBPI = AFI->getTaggedBasePointerIndex()) + AFI->setTaggedBasePointerOffset(-MFI.getObjectOffset(*TBPI)); + else + AFI->setTaggedBasePointerOffset(MFI.getStackSize()); + + // getStackSize() includes all the locals in its size calculation. We don't + // include these locals when computing the stack size of a funclet, as they + // are allocated in the parent's stack frame and accessed via the frame + // pointer from the funclet. We only save the callee saved registers in the + // funclet, which are really the callee saved registers of the parent + // function, including the funclet. + int64_t NumBytes = + IsFunclet ? AFL.getWinEHFuncletFrameSize(MF) : MFI.getStackSize(); + if (!AFI->hasStackFrame() && !AFL.windowsRequiresStackProbe(MF, NumBytes)) + return emitEmptyStackFramePrologue(NumBytes, PrologueBeginI, DL); + + bool IsWin64 = Subtarget.isCallingConvWin64(F.getCallingConv(), F.isVarArg()); + unsigned FixedObject = AFL.getFixedObjectSize(MF, AFI, IsWin64, IsFunclet); + + // Windows unwind can't represent the required stack adjustments if we have + // both SVE callee-saves and dynamic stack allocations, and the frame + // pointer is before the SVE spills. The allocation of the frame pointer + // must be the last instruction in the prologue so the unwinder can restore + // the stack pointer correctly. (And there isn't any unwind opcode for + // `addvl sp, x29, -17`.) + // + // Because of this, we do spills in the opposite order on Windows: first SVE, + // then GPRs. The main side-effect of this is that it makes accessing + // parameters passed on the stack more expensive. + // + // We could consider rearranging the spills for simpler cases. + bool FPAfterSVECalleeSaves = + Subtarget.isTargetWindows() && AFI->getSVECalleeSavedStackSize(); + + if (FPAfterSVECalleeSaves && AFI->hasStackHazardSlotIndex()) + reportFatalUsageError("SME hazard padding is not supported on Windows"); + + auto PrologueSaveSize = AFI->getCalleeSavedStackSize() + FixedObject; + // All of the remaining stack allocations are for locals. + determineLocalsStackSize(NumBytes, PrologueSaveSize); + + MachineBasicBlock::iterator FirstGPRSaveI = PrologueBeginI; + if (FPAfterSVECalleeSaves) { + // If we're doing SVE saves first, we need to immediately allocate space + // for fixed objects, then space for the SVE callee saves. + // + // Windows unwind requires that the scalable size is a multiple of 16; + // that's handled when the callee-saved size is computed. + auto SaveSize = + StackOffset::getScalable(AFI->getSVECalleeSavedStackSize()) + + StackOffset::getFixed(FixedObject); + AFL.allocateStackSpace(MBB, PrologueBeginI, 0, SaveSize, NeedsWinCFI, + &HasWinCFI, + /*EmitCFI=*/false, StackOffset{}, + /*FollowupAllocs=*/true); + NumBytes -= FixedObject; + + // Now allocate space for the GPR callee saves. + MachineBasicBlock::iterator MBBI = PrologueBeginI; + while (MBBI != EndI && AFL.isSVECalleeSave(MBBI)) + ++MBBI; + FirstGPRSaveI = AFL.convertCalleeSaveRestoreToSPPrePostIncDec( + MBB, MBBI, DL, TII, -AFI->getCalleeSavedStackSize(), NeedsWinCFI, + &HasWinCFI, EmitAsyncCFI); + NumBytes -= AFI->getCalleeSavedStackSize(); + } else if (CombineSPBump) { + assert(!AFL.getSVEStackSize(MF) && "Cannot combine SP bump with SVE"); + emitFrameOffset(MBB, PrologueBeginI, DL, AArch64::SP, AArch64::SP, + StackOffset::getFixed(-NumBytes), TII, + MachineInstr::FrameSetup, false, NeedsWinCFI, &HasWinCFI, + EmitAsyncCFI); + NumBytes = 0; + } else if (HomPrologEpilog) { + // Stack has been already adjusted. + NumBytes -= PrologueSaveSize; + } else if (PrologueSaveSize != 0) { + FirstGPRSaveI = AFL.convertCalleeSaveRestoreToSPPrePostIncDec( + MBB, PrologueBeginI, DL, TII, -PrologueSaveSize, NeedsWinCFI, + &HasWinCFI, EmitAsyncCFI); + NumBytes -= PrologueSaveSize; + } + assert(NumBytes >= 0 && "Negative stack allocation size!?"); + + // Move past the saves of the callee-saved registers, fixing up the offsets + // and pre-inc if we decided to combine the callee-save and local stack + // pointer bump above. + auto &TLI = *MF.getSubtarget().getTargetLowering(); + + MachineBasicBlock::iterator AfterGPRSavesI = FirstGPRSaveI; + while (AfterGPRSavesI != EndI && + AfterGPRSavesI->getFlag(MachineInstr::FrameSetup) && + !AFL.isSVECalleeSave(AfterGPRSavesI)) { + if (CombineSPBump && + // Only fix-up frame-setup load/store instructions. + (!AFL.requiresSaveVG(MF) || !AFL.isVGInstruction(AfterGPRSavesI, TLI))) + AFL.fixupCalleeSaveRestoreStackOffset( + *AfterGPRSavesI, AFI->getLocalStackSize(), NeedsWinCFI, &HasWinCFI); + ++AfterGPRSavesI; + } + + // For funclets the FP belongs to the containing function. Only set up FP if + // we actually need to. + if (!IsFunclet && HasFP) + emitFramePointerSetup(AfterGPRSavesI, DL, FixedObject); + + // Now emit the moves for whatever callee saved regs we have (including FP, + // LR if those are saved). Frame instructions for SVE register are emitted + // later, after the instruction which actually save SVE regs. + if (EmitAsyncCFI) + emitCalleeSavedGPRLocations(AfterGPRSavesI); + + // Alignment is required for the parent frame, not the funclet + const bool NeedsRealignment = + NumBytes && !IsFunclet && RegInfo.hasStackRealignment(MF); + const int64_t RealignmentPadding = + (NeedsRealignment && MFI.getMaxAlign() > Align(16)) + ? MFI.getMaxAlign().value() - 16 + : 0; + + if (AFL.windowsRequiresStackProbe(MF, NumBytes + RealignmentPadding)) + emitWindowsStackProbe(AfterGPRSavesI, DL, NumBytes, RealignmentPadding); + + StackOffset SVEStackSize = AFL.getSVEStackSize(MF); + StackOffset SVECalleeSavesSize = {}, SVELocalsSize = SVEStackSize; + MachineBasicBlock::iterator CalleeSavesEnd = AfterGPRSavesI; + + StackOffset CFAOffset = + StackOffset::getFixed((int64_t)MFI.getStackSize() - NumBytes); + + // Process the SVE callee-saves to determine what space needs to be + // allocated. + MachineBasicBlock::iterator AfterSVESavesI = AfterGPRSavesI; + if (int64_t CalleeSavedSize = AFI->getSVECalleeSavedStackSize()) { + LLVM_DEBUG(dbgs() << "SVECalleeSavedStackSize = " << CalleeSavedSize + << "\n"); + SVECalleeSavesSize = StackOffset::getScalable(CalleeSavedSize); + SVELocalsSize = SVEStackSize - SVECalleeSavesSize; + // Find callee save instructions in frame. + // Note: With FPAfterSVECalleeSaves the callee saves have already been + // allocated. + if (!FPAfterSVECalleeSaves) { + MachineBasicBlock::iterator CalleeSavesBegin = AfterGPRSavesI; + assert(AFL.isSVECalleeSave(CalleeSavesBegin) && "Unexpected instruction"); + while (AFL.isSVECalleeSave(AfterSVESavesI) && + AfterSVESavesI != MBB.getFirstTerminator()) + ++AfterSVESavesI; + CalleeSavesEnd = AfterSVESavesI; + + StackOffset LocalsSize = SVELocalsSize + StackOffset::getFixed(NumBytes); + // Allocate space for the callee saves (if any). + AFL.allocateStackSpace(MBB, CalleeSavesBegin, 0, SVECalleeSavesSize, + false, nullptr, EmitAsyncCFI && !HasFP, CFAOffset, + MFI.hasVarSizedObjects() || LocalsSize); + } + } + CFAOffset += SVECalleeSavesSize; + + if (EmitAsyncCFI) + emitCalleeSavedSVELocations(CalleeSavesEnd); + + // Allocate space for the rest of the frame including SVE locals. Align the + // stack as necessary. + assert(!(AFL.canUseRedZone(MF) && NeedsRealignment) && + "Cannot use redzone with stack realignment"); + if (!AFL.canUseRedZone(MF)) { + // FIXME: in the case of dynamic re-alignment, NumBytes doesn't have + // the correct value here, as NumBytes also includes padding bytes, + // which shouldn't be counted here. + AFL.allocateStackSpace(MBB, CalleeSavesEnd, RealignmentPadding, + SVELocalsSize + StackOffset::getFixed(NumBytes), + NeedsWinCFI, &HasWinCFI, EmitAsyncCFI && !HasFP, + CFAOffset, MFI.hasVarSizedObjects()); + } + + // If we need a base pointer, set it up here. It's whatever the value of the + // stack pointer is at this point. Any variable size objects will be allocated + // after this, so we can still use the base pointer to reference locals. + // + // FIXME: Clarify FrameSetup flags here. + // Note: Use emitFrameOffset() like above for FP if the FrameSetup flag is + // needed. + // For funclets the BP belongs to the containing function. + if (!IsFunclet && RegInfo.hasBasePointer(MF)) { + TII->copyPhysReg(MBB, AfterSVESavesI, DL, RegInfo.getBaseRegister(), + AArch64::SP, false); + if (NeedsWinCFI) { + HasWinCFI = true; + BuildMI(MBB, AfterSVESavesI, DL, TII->get(AArch64::SEH_Nop)) + .setMIFlag(MachineInstr::FrameSetup); + } + } + + // The very last FrameSetup instruction indicates the end of prologue. Emit a + // SEH opcode indicating the prologue end. + if (NeedsWinCFI && HasWinCFI) { + BuildMI(MBB, AfterSVESavesI, DL, TII->get(AArch64::SEH_PrologEnd)) + .setMIFlag(MachineInstr::FrameSetup); + } + + // SEH funclets are passed the frame pointer in X1. If the parent + // function uses the base register, then the base register is used + // directly, and is not retrieved from X1. + if (IsFunclet && F.hasPersonalityFn()) { + EHPersonality Per = classifyEHPersonality(F.getPersonalityFn()); + if (isAsynchronousEHPersonality(Per)) { + BuildMI(MBB, AfterSVESavesI, DL, TII->get(TargetOpcode::COPY), + AArch64::FP) + .addReg(AArch64::X1) + .setMIFlag(MachineInstr::FrameSetup); + MBB.addLiveIn(AArch64::X1); + } + } + + if (EmitCFI && !EmitAsyncCFI) { + if (HasFP) { + emitDefineCFAWithFP(AfterSVESavesI, FixedObject); + } else { + StackOffset TotalSize = + SVEStackSize + StackOffset::getFixed((int64_t)MFI.getStackSize()); + CFIInstBuilder CFIBuilder(MBB, AfterSVESavesI, MachineInstr::FrameSetup); + CFIBuilder.insertCFIInst( + createDefCFA(RegInfo, /*FrameReg=*/AArch64::SP, /*Reg=*/AArch64::SP, + TotalSize, /*LastAdjustmentWasScalable=*/false)); + } + emitCalleeSavedGPRLocations(AfterSVESavesI); + emitCalleeSavedSVELocations(AfterSVESavesI); + } +} + +void AArch64PrologueEmitter::emitShadowCallStackPrologue( + MachineBasicBlock::iterator MBBI, const DebugLoc &DL) const { + // Shadow call stack prolog: str x30, [x18], #8 + BuildMI(MBB, MBBI, DL, TII->get(AArch64::STRXpost)) + .addReg(AArch64::X18, RegState::Define) + .addReg(AArch64::LR) + .addReg(AArch64::X18) + .addImm(8) + .setMIFlag(MachineInstr::FrameSetup); + + // This instruction also makes x18 live-in to the entry block. + MBB.addLiveIn(AArch64::X18); + + if (NeedsWinCFI) + BuildMI(MBB, MBBI, DL, TII->get(AArch64::SEH_Nop)) + .setMIFlag(MachineInstr::FrameSetup); + + if (EmitCFI) { + // Emit a CFI instruction that causes 8 to be subtracted from the value of + // x18 when unwinding past this frame. + static const char CFIInst[] = { + dwarf::DW_CFA_val_expression, + 18, // register + 2, // length + static_cast<char>(unsigned(dwarf::DW_OP_breg18)), + static_cast<char>(-8) & 0x7f, // addend (sleb128) + }; + CFIInstBuilder(MBB, MBBI, MachineInstr::FrameSetup) + .buildEscape(StringRef(CFIInst, sizeof(CFIInst))); + } +} + +void AArch64PrologueEmitter::emitSwiftAsyncContextFramePointer( + MachineBasicBlock::iterator MBBI, const DebugLoc &DL) const { + switch (MF.getTarget().Options.SwiftAsyncFramePointer) { + case SwiftAsyncFramePointerMode::DeploymentBased: + if (Subtarget.swiftAsyncContextIsDynamicallySet()) { + // The special symbol below is absolute and has a *value* that can be + // combined with the frame pointer to signal an extended frame. + BuildMI(MBB, MBBI, DL, TII->get(AArch64::LOADgot), AArch64::X16) + .addExternalSymbol("swift_async_extendedFramePointerFlags", + AArch64II::MO_GOT); + if (NeedsWinCFI) { + BuildMI(MBB, MBBI, DL, TII->get(AArch64::SEH_Nop)) + .setMIFlags(MachineInstr::FrameSetup); + HasWinCFI = true; + } + BuildMI(MBB, MBBI, DL, TII->get(AArch64::ORRXrs), AArch64::FP) + .addUse(AArch64::FP) + .addUse(AArch64::X16) + .addImm(Subtarget.isTargetILP32() ? 32 : 0); + if (NeedsWinCFI) { + BuildMI(MBB, MBBI, DL, TII->get(AArch64::SEH_Nop)) + .setMIFlags(MachineInstr::FrameSetup); + HasWinCFI = true; + } + break; + } + [[fallthrough]]; + + case SwiftAsyncFramePointerMode::Always: + // ORR x29, x29, #0x1000_0000_0000_0000 + BuildMI(MBB, MBBI, DL, TII->get(AArch64::ORRXri), AArch64::FP) + .addUse(AArch64::FP) + .addImm(0x1100) + .setMIFlag(MachineInstr::FrameSetup); + if (NeedsWinCFI) { + BuildMI(MBB, MBBI, DL, TII->get(AArch64::SEH_Nop)) + .setMIFlags(MachineInstr::FrameSetup); + HasWinCFI = true; + } + break; + + case SwiftAsyncFramePointerMode::Never: + break; + } +} + +void AArch64PrologueEmitter::emitEmptyStackFramePrologue( + int64_t NumBytes, MachineBasicBlock::iterator MBBI, + const DebugLoc &DL) const { + assert(!HasFP && "unexpected function without stack frame but with FP"); + assert(!AFL.getSVEStackSize(MF) && + "unexpected function without stack frame but with SVE objects"); + // All of the stack allocation is for locals. + AFI->setLocalStackSize(NumBytes); + if (!NumBytes) { + if (NeedsWinCFI && HasWinCFI) { + BuildMI(MBB, MBBI, DL, TII->get(AArch64::SEH_PrologEnd)) + .setMIFlag(MachineInstr::FrameSetup); + } + return; + } + // REDZONE: If the stack size is less than 128 bytes, we don't need + // to actually allocate. + if (AFL.canUseRedZone(MF)) { + AFI->setHasRedZone(true); + ++NumRedZoneFunctions; + } else { + emitFrameOffset(MBB, MBBI, DL, AArch64::SP, AArch64::SP, + StackOffset::getFixed(-NumBytes), TII, + MachineInstr::FrameSetup, false, NeedsWinCFI, &HasWinCFI); + if (EmitCFI) { + // Label used to tie together the PROLOG_LABEL and the MachineMoves. + MCSymbol *FrameLabel = MF.getContext().createTempSymbol(); + // Encode the stack size of the leaf function. + CFIInstBuilder(MBB, MBBI, MachineInstr::FrameSetup) + .buildDefCFAOffset(NumBytes, FrameLabel); + } + } + + if (NeedsWinCFI) { + HasWinCFI = true; + BuildMI(MBB, MBBI, DL, TII->get(AArch64::SEH_PrologEnd)) + .setMIFlag(MachineInstr::FrameSetup); + } +} + +void AArch64PrologueEmitter::emitFramePointerSetup( + MachineBasicBlock::iterator MBBI, const DebugLoc &DL, + unsigned FixedObject) { + int64_t FPOffset = AFI->getCalleeSaveBaseToFrameRecordOffset(); + if (CombineSPBump) + FPOffset += AFI->getLocalStackSize(); + + if (AFI->hasSwiftAsyncContext()) { + // Before we update the live FP we have to ensure there's a valid (or + // null) asynchronous context in its slot just before FP in the frame + // record, so store it now. + const auto &Attrs = MF.getFunction().getAttributes(); + bool HaveInitialContext = Attrs.hasAttrSomewhere(Attribute::SwiftAsync); + if (HaveInitialContext) + MBB.addLiveIn(AArch64::X22); + Register Reg = HaveInitialContext ? AArch64::X22 : AArch64::XZR; + BuildMI(MBB, MBBI, DL, TII->get(AArch64::StoreSwiftAsyncContext)) + .addUse(Reg) + .addUse(AArch64::SP) + .addImm(FPOffset - 8) + .setMIFlags(MachineInstr::FrameSetup); + if (NeedsWinCFI) { + // WinCFI and arm64e, where StoreSwiftAsyncContext is expanded + // to multiple instructions, should be mutually-exclusive. + assert(Subtarget.getTargetTriple().getArchName() != "arm64e"); + BuildMI(MBB, MBBI, DL, TII->get(AArch64::SEH_Nop)) + .setMIFlags(MachineInstr::FrameSetup); + HasWinCFI = true; + } + } + + if (HomPrologEpilog) { + auto Prolog = MBBI; + --Prolog; + assert(Prolog->getOpcode() == AArch64::HOM_Prolog); + Prolog->addOperand(MachineOperand::CreateImm(FPOffset)); + } else { + // Issue sub fp, sp, FPOffset or + // mov fp,sp when FPOffset is zero. + // Note: All stores of callee-saved registers are marked as "FrameSetup". + // This code marks the instruction(s) that set the FP also. + emitFrameOffset(MBB, MBBI, DL, AArch64::FP, AArch64::SP, + StackOffset::getFixed(FPOffset), TII, + MachineInstr::FrameSetup, false, NeedsWinCFI, &HasWinCFI); + if (NeedsWinCFI && HasWinCFI) { + BuildMI(MBB, MBBI, DL, TII->get(AArch64::SEH_PrologEnd)) + .setMIFlag(MachineInstr::FrameSetup); + // After setting up the FP, the rest of the prolog doesn't need to be + // included in the SEH unwind info. + NeedsWinCFI = false; + } + } + if (EmitAsyncCFI) + emitDefineCFAWithFP(MBBI, FixedObject); +} + +// Define the current CFA rule to use the provided FP. +void AArch64PrologueEmitter::emitDefineCFAWithFP( + MachineBasicBlock::iterator MBBI, unsigned FixedObject) const { + const AArch64RegisterInfo *TRI = Subtarget.getRegisterInfo(); + const int OffsetToFirstCalleeSaveFromFP = + AFI->getCalleeSaveBaseToFrameRecordOffset() - + AFI->getCalleeSavedStackSize(); + Register FramePtr = TRI->getFrameRegister(MF); + CFIInstBuilder(MBB, MBBI, MachineInstr::FrameSetup) + .buildDefCFA(FramePtr, FixedObject - OffsetToFirstCalleeSaveFromFP); +} + +void AArch64PrologueEmitter::emitWindowsStackProbe( + MachineBasicBlock::iterator MBBI, const DebugLoc &DL, int64_t &NumBytes, + int64_t RealignmentPadding) const { + if (AFI->getSVECalleeSavedStackSize()) + report_fatal_error("SVE callee saves not yet supported with stack probing"); + + // Find an available register to spill the value of X15 to, if X15 is being + // used already for nest. + unsigned X15Scratch = AArch64::NoRegister; + const AArch64Subtarget &STI = MF.getSubtarget<AArch64Subtarget>(); + if (llvm::any_of(MBB.liveins(), + [&STI](const MachineBasicBlock::RegisterMaskPair &LiveIn) { + return STI.getRegisterInfo()->isSuperOrSubRegisterEq( + AArch64::X15, LiveIn.PhysReg); + })) { + X15Scratch = AFL.findScratchNonCalleeSaveRegister(&MBB, /*HasCall=*/true); + assert(X15Scratch != AArch64::NoRegister && + (X15Scratch < AArch64::X15 || X15Scratch > AArch64::X17)); +#ifndef NDEBUG + LiveRegs.removeReg(AArch64::X15); // ignore X15 since we restore it +#endif + BuildMI(MBB, MBBI, DL, TII->get(AArch64::ORRXrr), X15Scratch) + .addReg(AArch64::XZR) + .addReg(AArch64::X15, RegState::Undef) + .addReg(AArch64::X15, RegState::Implicit) + .setMIFlag(MachineInstr::FrameSetup); + } + + uint64_t NumWords = (NumBytes + RealignmentPadding) >> 4; + if (NeedsWinCFI) { + HasWinCFI = true; + // alloc_l can hold at most 256MB, so assume that NumBytes doesn't + // exceed this amount. We need to move at most 2^24 - 1 into x15. + // This is at most two instructions, MOVZ followed by MOVK. + // TODO: Fix to use multiple stack alloc unwind codes for stacks + // exceeding 256MB in size. + if (NumBytes >= (1 << 28)) + report_fatal_error("Stack size cannot exceed 256MB for stack " + "unwinding purposes"); + + uint32_t LowNumWords = NumWords & 0xFFFF; + BuildMI(MBB, MBBI, DL, TII->get(AArch64::MOVZXi), AArch64::X15) + .addImm(LowNumWords) + .addImm(AArch64_AM::getShifterImm(AArch64_AM::LSL, 0)) + .setMIFlag(MachineInstr::FrameSetup); + BuildMI(MBB, MBBI, DL, TII->get(AArch64::SEH_Nop)) + .setMIFlag(MachineInstr::FrameSetup); + if ((NumWords & 0xFFFF0000) != 0) { + BuildMI(MBB, MBBI, DL, TII->get(AArch64::MOVKXi), AArch64::X15) + .addReg(AArch64::X15) + .addImm((NumWords & 0xFFFF0000) >> 16) // High half + .addImm(AArch64_AM::getShifterImm(AArch64_AM::LSL, 16)) + .setMIFlag(MachineInstr::FrameSetup); + BuildMI(MBB, MBBI, DL, TII->get(AArch64::SEH_Nop)) + .setMIFlag(MachineInstr::FrameSetup); + } + } else { + BuildMI(MBB, MBBI, DL, TII->get(AArch64::MOVi64imm), AArch64::X15) + .addImm(NumWords) + .setMIFlags(MachineInstr::FrameSetup); + } + + const char *ChkStk = Subtarget.getChkStkName(); + switch (MF.getTarget().getCodeModel()) { + case CodeModel::Tiny: + case CodeModel::Small: + case CodeModel::Medium: + case CodeModel::Kernel: + BuildMI(MBB, MBBI, DL, TII->get(AArch64::BL)) + .addExternalSymbol(ChkStk) + .addReg(AArch64::X15, RegState::Implicit) + .addReg(AArch64::X16, + RegState::Implicit | RegState::Define | RegState::Dead) + .addReg(AArch64::X17, + RegState::Implicit | RegState::Define | RegState::Dead) + .addReg(AArch64::NZCV, + RegState::Implicit | RegState::Define | RegState::Dead) + .setMIFlags(MachineInstr::FrameSetup); + if (NeedsWinCFI) { + HasWinCFI = true; + BuildMI(MBB, MBBI, DL, TII->get(AArch64::SEH_Nop)) + .setMIFlag(MachineInstr::FrameSetup); + } + break; + case CodeModel::Large: + BuildMI(MBB, MBBI, DL, TII->get(AArch64::MOVaddrEXT)) + .addReg(AArch64::X16, RegState::Define) + .addExternalSymbol(ChkStk) + .addExternalSymbol(ChkStk) + .setMIFlags(MachineInstr::FrameSetup); + if (NeedsWinCFI) { + HasWinCFI = true; + BuildMI(MBB, MBBI, DL, TII->get(AArch64::SEH_Nop)) + .setMIFlag(MachineInstr::FrameSetup); + } + + BuildMI(MBB, MBBI, DL, TII->get(getBLRCallOpcode(MF))) + .addReg(AArch64::X16, RegState::Kill) + .addReg(AArch64::X15, RegState::Implicit | RegState::Define) + .addReg(AArch64::X16, + RegState::Implicit | RegState::Define | RegState::Dead) + .addReg(AArch64::X17, + RegState::Implicit | RegState::Define | RegState::Dead) + .addReg(AArch64::NZCV, + RegState::Implicit | RegState::Define | RegState::Dead) + .setMIFlags(MachineInstr::FrameSetup); + if (NeedsWinCFI) { + HasWinCFI = true; + BuildMI(MBB, MBBI, DL, TII->get(AArch64::SEH_Nop)) + .setMIFlag(MachineInstr::FrameSetup); + } + break; + } + + BuildMI(MBB, MBBI, DL, TII->get(AArch64::SUBXrx64), AArch64::SP) + .addReg(AArch64::SP, RegState::Kill) + .addReg(AArch64::X15, RegState::Kill) + .addImm(AArch64_AM::getArithExtendImm(AArch64_AM::UXTX, 4)) + .setMIFlags(MachineInstr::FrameSetup); + if (NeedsWinCFI) { + HasWinCFI = true; + BuildMI(MBB, MBBI, DL, TII->get(AArch64::SEH_StackAlloc)) + .addImm(NumBytes) + .setMIFlag(MachineInstr::FrameSetup); + } + NumBytes = 0; + + if (RealignmentPadding > 0) { + if (RealignmentPadding >= 4096) { + BuildMI(MBB, MBBI, DL, TII->get(AArch64::MOVi64imm)) + .addReg(AArch64::X16, RegState::Define) + .addImm(RealignmentPadding) + .setMIFlags(MachineInstr::FrameSetup); + BuildMI(MBB, MBBI, DL, TII->get(AArch64::ADDXrx64), AArch64::X15) + .addReg(AArch64::SP) + .addReg(AArch64::X16, RegState::Kill) + .addImm(AArch64_AM::getArithExtendImm(AArch64_AM::UXTX, 0)) + .setMIFlag(MachineInstr::FrameSetup); + } else { + BuildMI(MBB, MBBI, DL, TII->get(AArch64::ADDXri), AArch64::X15) + .addReg(AArch64::SP) + .addImm(RealignmentPadding) + .addImm(0) + .setMIFlag(MachineInstr::FrameSetup); + } + + uint64_t AndMask = ~(MFI.getMaxAlign().value() - 1); + BuildMI(MBB, MBBI, DL, TII->get(AArch64::ANDXri), AArch64::SP) + .addReg(AArch64::X15, RegState::Kill) + .addImm(AArch64_AM::encodeLogicalImmediate(AndMask, 64)); + AFI->setStackRealigned(true); + + // No need for SEH instructions here; if we're realigning the stack, + // we've set a frame pointer and already finished the SEH prologue. + assert(!NeedsWinCFI); + } + if (X15Scratch != AArch64::NoRegister) { + BuildMI(MBB, MBBI, DL, TII->get(AArch64::ORRXrr), AArch64::X15) + .addReg(AArch64::XZR) + .addReg(X15Scratch, RegState::Undef) + .addReg(X15Scratch, RegState::Implicit) + .setMIFlag(MachineInstr::FrameSetup); + } +} + +void AArch64PrologueEmitter::emitCalleeSavedGPRLocations( + MachineBasicBlock::iterator MBBI) const { + MachineFunction &MF = *MBB.getParent(); + MachineFrameInfo &MFI = MF.getFrameInfo(); + + const std::vector<CalleeSavedInfo> &CSI = MFI.getCalleeSavedInfo(); + if (CSI.empty()) + return; + + CFIInstBuilder CFIBuilder(MBB, MBBI, MachineInstr::FrameSetup); + for (const auto &Info : CSI) { + unsigned FrameIdx = Info.getFrameIdx(); + if (MFI.getStackID(FrameIdx) == TargetStackID::ScalableVector) + continue; + + assert(!Info.isSpilledToReg() && "Spilling to registers not implemented"); + int64_t Offset = MFI.getObjectOffset(FrameIdx) - AFL.getOffsetOfLocalArea(); + CFIBuilder.buildOffset(Info.getReg(), Offset); + } +} + +void AArch64PrologueEmitter::emitCalleeSavedSVELocations( + MachineBasicBlock::iterator MBBI) const { + MachineFunction &MF = *MBB.getParent(); + MachineFrameInfo &MFI = MF.getFrameInfo(); + + // Add callee saved registers to move list. + const std::vector<CalleeSavedInfo> &CSI = MFI.getCalleeSavedInfo(); + if (CSI.empty()) + return; + + const TargetSubtargetInfo &STI = MF.getSubtarget(); + const TargetRegisterInfo &TRI = *STI.getRegisterInfo(); + AArch64FunctionInfo &AFI = *MF.getInfo<AArch64FunctionInfo>(); + CFIInstBuilder CFIBuilder(MBB, MBBI, MachineInstr::FrameSetup); + + std::optional<int64_t> IncomingVGOffsetFromDefCFA; + if (AFL.requiresSaveVG(MF)) { + auto IncomingVG = *find_if( + reverse(CSI), [](auto &Info) { return Info.getReg() == AArch64::VG; }); + IncomingVGOffsetFromDefCFA = MFI.getObjectOffset(IncomingVG.getFrameIdx()) - + AFL.getOffsetOfLocalArea(); + } + + for (const auto &Info : CSI) { + if (MFI.getStackID(Info.getFrameIdx()) != TargetStackID::ScalableVector) + continue; + + // Not all unwinders may know about SVE registers, so assume the lowest + // common denominator. + assert(!Info.isSpilledToReg() && "Spilling to registers not implemented"); + MCRegister Reg = Info.getReg(); + if (!static_cast<const AArch64RegisterInfo &>(TRI).regNeedsCFI(Reg, Reg)) + continue; + + StackOffset Offset = + StackOffset::getScalable(MFI.getObjectOffset(Info.getFrameIdx())) - + StackOffset::getFixed(AFI.getCalleeSavedStackSize(MFI)); + + CFIBuilder.insertCFIInst( + createCFAOffset(TRI, Reg, Offset, IncomingVGOffsetFromDefCFA)); + } +} + +} // namespace llvm diff --git a/llvm/lib/Target/AArch64/AArch64PrologueEpilogue.h b/llvm/lib/Target/AArch64/AArch64PrologueEpilogue.h new file mode 100644 index 000000000000..94029ede60c7 --- /dev/null +++ b/llvm/lib/Target/AArch64/AArch64PrologueEpilogue.h @@ -0,0 +1,111 @@ +//===----------------------------------------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +/// +/// \file +/// This file contains the declaration of the AArch64PrologueEmitter class, +/// which is is used to emit the prologue on AArch64. +/// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_LIB_TARGET_AARCH64_AARCH64PROLOGUEEPILOGUE_H +#define LLVM_LIB_TARGET_AARCH64_AARCH64PROLOGUEEPILOGUE_H + +#include "AArch64RegisterInfo.h" +#include "llvm/CodeGen/LivePhysRegs.h" +#include "llvm/CodeGen/MachineFunction.h" + +namespace llvm { + +class AArch64Subtarget; +class AArch64FunctionInfo; +class AArch64FrameLowering; + +/// A helper class for emitting the prologue. Substantial new functionality +/// should be factored into a new method. Where possible "emit*" methods should +/// be const, and any flags that change how the prologue is emitted should be +/// set in the constructor. +class AArch64PrologueEmitter { +public: + AArch64PrologueEmitter(MachineFunction &MF, MachineBasicBlock &MBB, + const AArch64FrameLowering &AFL); + + /// Emit the prologue. + void emitPrologue(); + + ~AArch64PrologueEmitter() { + MF.setHasWinCFI(HasWinCFI); +#ifndef NDEBUG + verifyPrologueClobbers(); +#endif + } + +private: + void emitShadowCallStackPrologue(MachineBasicBlock::iterator MBBI, + const DebugLoc &DL) const; + + void emitSwiftAsyncContextFramePointer(MachineBasicBlock::iterator MBBI, + const DebugLoc &DL) const; + + void emitEmptyStackFramePrologue(int64_t NumBytes, + MachineBasicBlock::iterator MBBI, + const DebugLoc &DL) const; + + void emitFramePointerSetup(MachineBasicBlock::iterator MBBI, + const DebugLoc &DL, unsigned FixedObject); + + void emitDefineCFAWithFP(MachineBasicBlock::iterator MBBI, + unsigned FixedObject) const; + + void emitWindowsStackProbe(MachineBasicBlock::iterator MBBI, + const DebugLoc &DL, int64_t &NumBytes, + int64_t RealignmentPadding) const; + + void emitCalleeSavedGPRLocations(MachineBasicBlock::iterator MBBI) const; + void emitCalleeSavedSVELocations(MachineBasicBlock::iterator MBBI) const; + + void determineLocalsStackSize(uint64_t StackSize, uint64_t PrologueSaveSize); + + MachineFunction &MF; + MachineBasicBlock &MBB; + + const Function &F; + const MachineFrameInfo &MFI; + const AArch64Subtarget &Subtarget; + const AArch64FrameLowering &AFL; + const AArch64RegisterInfo &RegInfo; + +#ifndef NDEBUG + mutable LivePhysRegs LiveRegs{RegInfo}; + MachineBasicBlock::iterator PrologueEndI; + + void collectBlockLiveins(); + void verifyPrologueClobbers() const; +#endif + + // Prologue flags. These generally should not change outside of the + // constructor. Two exceptions are "CombineSPBump" which is set in + // determineLocalsStackSize, and "NeedsWinCFI" which is set in + // emitFramePointerSetup. + bool EmitCFI = false; + bool EmitAsyncCFI = false; + bool HasFP = false; + bool IsFunclet = false; + bool CombineSPBump = false; + bool HomPrologEpilog = false; + bool NeedsWinCFI = false; + + // Note: "HasWinCFI" is mutable as it can change in any "emit" function. + mutable bool HasWinCFI = false; + + const TargetInstrInfo *TII = nullptr; + AArch64FunctionInfo *AFI = nullptr; +}; + +} // namespace llvm + +#endif diff --git a/llvm/lib/Target/AArch64/AArch64RegisterInfo.td b/llvm/lib/Target/AArch64/AArch64RegisterInfo.td index 1a7609bfee8a..431ed6ec34e7 100644 --- a/llvm/lib/Target/AArch64/AArch64RegisterInfo.td +++ b/llvm/lib/Target/AArch64/AArch64RegisterInfo.td @@ -983,7 +983,7 @@ class ZPRRegOp <string Suffix, AsmOperandClass C, ElementSizeEnum Size, // Note: This hardware mode is enabled in AArch64Subtarget::getHwModeSet() // (without the use of the table-gen'd predicates). -def SMEWithZPRPredicateSpills : HwMode<"", [Predicate<"false">]>; +def SMEWithZPRPredicateSpills : HwMode<[Predicate<"false">]>; def PPRSpillFillRI : RegInfoByHwMode< [DefaultMode, SMEWithZPRPredicateSpills], diff --git a/llvm/lib/Target/AArch64/AArch64SMEInstrInfo.td b/llvm/lib/Target/AArch64/AArch64SMEInstrInfo.td index 125225df1546..601dc34d74b9 100644 --- a/llvm/lib/Target/AArch64/AArch64SMEInstrInfo.td +++ b/llvm/lib/Target/AArch64/AArch64SMEInstrInfo.td @@ -61,10 +61,10 @@ let usesCustomInserter = 1, Defs = [SP], Uses = [SP] in { def : Pat<(i64 (AArch64AllocateZABuffer GPR64:$size)), (AllocateZABuffer $size)>; -def AArch64InitTPIDR2Obj : SDNode<"AArch64ISD::INIT_TPIDR2OBJ", SDTypeProfile<0, 1, - [SDTCisInt<0>]>, [SDNPHasChain, SDNPMayStore]>; +def AArch64InitTPIDR2Obj : SDNode<"AArch64ISD::INIT_TPIDR2OBJ", SDTypeProfile<0, 2, + [SDTCisInt<0>, SDTCisInt<1>]>, [SDNPHasChain, SDNPMayStore]>; let usesCustomInserter = 1 in { - def InitTPIDR2Obj : Pseudo<(outs), (ins GPR64:$buffer), [(AArch64InitTPIDR2Obj GPR64:$buffer)]>, Sched<[WriteI]> {} + def InitTPIDR2Obj : Pseudo<(outs), (ins GPR64:$buffer, GPR64:$save_slices), [(AArch64InitTPIDR2Obj GPR64:$buffer, GPR64:$save_slices)]>, Sched<[WriteI]> {} } // Nodes to allocate a save buffer for SME. @@ -93,6 +93,8 @@ let hasSideEffects = 1, isMeta = 1 in { def RequiresZASavePseudo : Pseudo<(outs), (ins), []>, Sched<[]>; } +def SMEStateAllocPseudo : Pseudo<(outs), (ins), []>, Sched<[]>; + def CommitZASavePseudo : Pseudo<(outs), (ins GPR64:$tpidr2_el0, i1imm:$zero_za, i64imm:$commit_routine, variable_ops), []>, @@ -108,6 +110,11 @@ def AArch64_requires_za_save [SDNPHasChain, SDNPInGlue]>; def : Pat<(AArch64_requires_za_save), (RequiresZASavePseudo)>; +def AArch64_sme_state_alloc + : SDNode<"AArch64ISD::SME_STATE_ALLOC", SDTypeProfile<0, 0,[]>, + [SDNPHasChain]>; +def : Pat<(AArch64_sme_state_alloc), (SMEStateAllocPseudo)>; + //===----------------------------------------------------------------------===// // Instruction naming conventions. //===----------------------------------------------------------------------===// diff --git a/llvm/lib/Target/AArch64/AArch64SVEInstrInfo.td b/llvm/lib/Target/AArch64/AArch64SVEInstrInfo.td index eeb47b4d9975..7604ffdc9f64 100644 --- a/llvm/lib/Target/AArch64/AArch64SVEInstrInfo.td +++ b/llvm/lib/Target/AArch64/AArch64SVEInstrInfo.td @@ -412,6 +412,7 @@ def SDT_AArch64PTest : SDTypeProfile<1, 2, [ ]>; def AArch64ptest : SDNode<"AArch64ISD::PTEST", SDT_AArch64PTest>; def AArch64ptest_any : SDNode<"AArch64ISD::PTEST_ANY", SDT_AArch64PTest>; +def AArch64ptest_first : SDNode<"AArch64ISD::PTEST_FIRST", SDT_AArch64PTest>; def SDT_AArch64DUP_PRED : SDTypeProfile<1, 3, [SDTCisVec<0>, SDTCisSameAs<0, 3>, SDTCisVec<1>, SDTCVecEltisVT<1,i1>, SDTCisSameNumEltsAs<0, 1>]>; @@ -650,7 +651,7 @@ let Predicates = [HasSVE_or_SME, UseExperimentalZeroingPseudos] in { let Predicates = [HasSVE_or_SME] in { defm ADD_ZI : sve_int_arith_imm0<0b000, "add", add>; - defm SUB_ZI : sve_int_arith_imm0<0b001, "sub", sub>; + defm SUB_ZI : sve_int_arith_imm0<0b001, "sub", sub, add>; defm SUBR_ZI : sve_int_arith_imm0<0b011, "subr", AArch64subr>; defm SQADD_ZI : sve_int_arith_imm0_ssat<0b100, "sqadd", saddsat, ssubsat>; defm UQADD_ZI : sve_int_arith_imm0<0b101, "uqadd", uaddsat>; @@ -1071,7 +1072,7 @@ let Predicates = [HasSVE_or_SME] in { defm BRKB_PPmP : sve_int_break_m<0b101, "brkb", int_aarch64_sve_brkb>; defm BRKBS_PPzP : sve_int_break_z<0b110, "brkbs", null_frag>; - defm PTEST_PP : sve_int_ptest<0b010000, "ptest", AArch64ptest, AArch64ptest_any>; + defm PTEST_PP : sve_int_ptest<0b010000, "ptest", AArch64ptest, AArch64ptest_any, AArch64ptest_first>; defm PFALSE : sve_int_pfalse<0b000000, "pfalse">; defm PFIRST : sve_int_pfirst<0b00000, "pfirst", int_aarch64_sve_pfirst>; defm PNEXT : sve_int_pnext<0b00110, "pnext", int_aarch64_sve_pnext>; @@ -4141,8 +4142,8 @@ let Predicates = [HasSVE2_or_SME] in { defm WHILEHI_PXX : sve_int_while8_rr<0b101, "whilehi", int_aarch64_sve_whilehi, get_active_lane_mask>; // SVE2 pointer conflict compare - defm WHILEWR_PXX : sve2_int_while_rr<0b0, "whilewr", "int_aarch64_sve_whilewr">; - defm WHILERW_PXX : sve2_int_while_rr<0b1, "whilerw", "int_aarch64_sve_whilerw">; + defm WHILEWR_PXX : sve2_int_while_rr<0b0, "whilewr", loop_dependence_war_mask>; + defm WHILERW_PXX : sve2_int_while_rr<0b1, "whilerw", loop_dependence_raw_mask>; } // End HasSVE2_or_SME let Predicates = [HasSVEAES, HasNonStreamingSVE_or_SSVE_AES] in { diff --git a/llvm/lib/Target/AArch64/AArch64Subtarget.cpp b/llvm/lib/Target/AArch64/AArch64Subtarget.cpp index 0f4f0129e9cd..98e0a1180510 100644 --- a/llvm/lib/Target/AArch64/AArch64Subtarget.cpp +++ b/llvm/lib/Target/AArch64/AArch64Subtarget.cpp @@ -91,6 +91,10 @@ static cl::opt<bool> EnableZPRPredicateSpills( cl::desc( "Enables spilling/reloading SVE predicates as data vectors (ZPRs)")); +static cl::opt<unsigned> + VScaleForTuningOpt("sve-vscale-for-tuning", cl::Hidden, + cl::desc("Force a vscale for tuning factor for SVE")); + // Subreg liveness tracking is disabled by default for now until all issues // are ironed out. This option allows the feature to be used in tests. static cl::opt<bool> @@ -364,6 +368,8 @@ void AArch64Subtarget::initializeProperties(bool HasMinSize) { if (AArch64MinimumJumpTableEntries.getNumOccurrences() > 0 || !HasMinSize) MinimumJumpTableEntries = AArch64MinimumJumpTableEntries; + if (VScaleForTuningOpt.getNumOccurrences() > 0) + VScaleForTuning = VScaleForTuningOpt; } AArch64Subtarget::AArch64Subtarget(const Triple &TT, StringRef CPU, diff --git a/llvm/lib/Target/AArch64/AArch64Subtarget.h b/llvm/lib/Target/AArch64/AArch64Subtarget.h index 01c0bcc3a6a7..671df35cd379 100644 --- a/llvm/lib/Target/AArch64/AArch64Subtarget.h +++ b/llvm/lib/Target/AArch64/AArch64Subtarget.h @@ -246,8 +246,8 @@ public: /// Return true if the CPU supports any kind of instruction fusion. bool hasFusion() const { return hasArithmeticBccFusion() || hasArithmeticCbzFusion() || - hasFuseAES() || hasFuseArithmeticLogic() || hasFuseCCSelect() || - hasFuseAdrpAdd() || hasFuseLiterals(); + hasFuseAES() || hasFuseArithmeticLogic() || hasFuseCmpCSel() || + hasFuseCmpCSet() || hasFuseAdrpAdd() || hasFuseLiterals(); } unsigned getEpilogueVectorizationMinVF() const { diff --git a/llvm/lib/Target/AArch64/AArch64SystemOperands.td b/llvm/lib/Target/AArch64/AArch64SystemOperands.td index 1b0e90b0e0dc..65b752ed40c9 100644 --- a/llvm/lib/Target/AArch64/AArch64SystemOperands.td +++ b/llvm/lib/Target/AArch64/AArch64SystemOperands.td @@ -362,7 +362,7 @@ def lookupTSBByName : SearchIndex { let Key = ["Name"]; } -def : TSB<"csync", 0>; +def : TSB<"csync", 2>; //===----------------------------------------------------------------------===// // PRFM (prefetch) instruction options. diff --git a/llvm/lib/Target/AArch64/AArch64TargetMachine.cpp b/llvm/lib/Target/AArch64/AArch64TargetMachine.cpp index e67bd5869ccd..4650b2d0c815 100644 --- a/llvm/lib/Target/AArch64/AArch64TargetMachine.cpp +++ b/llvm/lib/Target/AArch64/AArch64TargetMachine.cpp @@ -589,7 +589,8 @@ void AArch64TargetMachine::registerPassBuilderCallbacks(PassBuilder &PB) { PB.registerLateLoopOptimizationsEPCallback( [=](LoopPassManager &LPM, OptimizationLevel Level) { - LPM.addPass(LoopIdiomVectorizePass()); + if (Level != OptimizationLevel::O0) + LPM.addPass(LoopIdiomVectorizePass()); }); if (getTargetTriple().isOSWindows()) PB.registerPipelineEarlySimplificationEPCallback( diff --git a/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp b/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp index 490f6391c15a..92321a76dbd8 100644 --- a/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp +++ b/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp @@ -25,6 +25,7 @@ #include "llvm/Support/Debug.h" #include "llvm/TargetParser/AArch64TargetParser.h" #include "llvm/Transforms/InstCombine/InstCombiner.h" +#include "llvm/Transforms/Utils/UnrollLoop.h" #include "llvm/Transforms/Vectorize/LoopVectorizationLegality.h" #include <algorithm> #include <optional> @@ -4409,6 +4410,32 @@ AArch64TTIImpl::getAddressComputationCost(Type *PtrTy, ScalarEvolution *SE, return 1; } +/// Check whether Opcode1 has less throughput according to the scheduling +/// model than Opcode2. +bool AArch64TTIImpl::hasKnownLowerThroughputFromSchedulingModel( + unsigned Opcode1, unsigned Opcode2) const { + const MCSchedModel &Sched = ST->getSchedModel(); + const TargetInstrInfo *TII = ST->getInstrInfo(); + if (!Sched.hasInstrSchedModel()) + return false; + + const MCSchedClassDesc *SCD1 = + Sched.getSchedClassDesc(TII->get(Opcode1).getSchedClass()); + const MCSchedClassDesc *SCD2 = + Sched.getSchedClassDesc(TII->get(Opcode2).getSchedClass()); + // We cannot handle variant scheduling classes without an MI. If we need to + // support them for any of the instructions we query the information of we + // might need to add a way to resolve them without a MI or not use the + // scheduling info. + assert(!SCD1->isVariant() && !SCD2->isVariant() && + "Cannot handle variant scheduling classes without an MI"); + if (!SCD1->isValid() || !SCD2->isValid()) + return false; + + return MCSchedModel::getReciprocalThroughput(*ST, *SCD1) > + MCSchedModel::getReciprocalThroughput(*ST, *SCD2); +} + InstructionCost AArch64TTIImpl::getCmpSelInstrCost( unsigned Opcode, Type *ValTy, Type *CondTy, CmpInst::Predicate VecPred, TTI::TargetCostKind CostKind, TTI::OperandValueInfo Op1Info, @@ -4506,6 +4533,12 @@ InstructionCost AArch64TTIImpl::getCmpSelInstrCost( (VecPred == FCmpInst::FCMP_ONE || VecPred == FCmpInst::FCMP_UEQ)) Factor = 3; // fcmxx+fcmyy+or + if (isa<ScalableVectorType>(ValTy) && + CostKind == TTI::TCK_RecipThroughput && + hasKnownLowerThroughputFromSchedulingModel(AArch64::FCMEQ_PPzZZ_S, + AArch64::FCMEQv4f32)) + Factor *= 2; + return Factor * (CostKind == TTI::TCK_Latency ? 2 : LT.first); } @@ -4937,6 +4970,23 @@ getAppleRuntimeUnrollPreferences(Loop *L, ScalarEvolution &SE, if (!L->getExitBlock()) return; + // Check if the loop contains any reductions that could be parallelized when + // unrolling. If so, enable partial unrolling, if the trip count is know to be + // a multiple of 2. + bool HasParellelizableReductions = + L->getNumBlocks() == 1 && + any_of(L->getHeader()->phis(), + [&SE, L](PHINode &Phi) { + return canParallelizeReductionWhenUnrolling(Phi, L, &SE); + }) && + isLoopSizeWithinBudget(L, TTI, 12, nullptr); + if (HasParellelizableReductions && + SE.getSmallConstantTripMultiple(L, L->getExitingBlock()) % 2 == 0) { + UP.Partial = true; + UP.MaxCount = 4; + UP.AddAdditionalAccumulators = true; + } + const SCEV *BTC = SE.getSymbolicMaxBackedgeTakenCount(L); if (isa<SCEVConstant>(BTC) || isa<SCEVCouldNotCompute>(BTC) || (SE.getSmallConstantMaxTripCount(L) > 0 && @@ -4952,6 +5002,12 @@ getAppleRuntimeUnrollPreferences(Loop *L, ScalarEvolution &SE, // Limit to loops with trip counts that are cheap to expand. UP.SCEVExpansionBudget = 1; + if (HasParellelizableReductions) { + UP.Runtime = true; + UP.DefaultUnrollRuntimeCount = 4; + UP.AddAdditionalAccumulators = true; + } + // Try to unroll small loops, of few-blocks with low budget, if they have // load/store dependencies, to expose more parallel memory access streams, // or if they do little work inside a block (i.e. load -> X -> store pattern). @@ -5486,13 +5542,14 @@ InstructionCost AArch64TTIImpl::getExtendedReductionCost( } InstructionCost -AArch64TTIImpl::getMulAccReductionCost(bool IsUnsigned, Type *ResTy, - VectorType *VecTy, +AArch64TTIImpl::getMulAccReductionCost(bool IsUnsigned, unsigned RedOpcode, + Type *ResTy, VectorType *VecTy, TTI::TargetCostKind CostKind) const { EVT VecVT = TLI->getValueType(DL, VecTy); EVT ResVT = TLI->getValueType(DL, ResTy); - if (ST->hasDotProd() && VecVT.isSimple() && ResVT.isSimple()) { + if (ST->hasDotProd() && VecVT.isSimple() && ResVT.isSimple() && + RedOpcode == Instruction::Add) { std::pair<InstructionCost, MVT> LT = getTypeLegalizationCost(VecTy); // The legal cases with dotprod are @@ -5503,7 +5560,8 @@ AArch64TTIImpl::getMulAccReductionCost(bool IsUnsigned, Type *ResTy, return LT.first + 2; } - return BaseT::getMulAccReductionCost(IsUnsigned, ResTy, VecTy, CostKind); + return BaseT::getMulAccReductionCost(IsUnsigned, RedOpcode, ResTy, VecTy, + CostKind); } InstructionCost @@ -5750,11 +5808,14 @@ AArch64TTIImpl::getShuffleCost(TTI::ShuffleKind Kind, VectorType *DstTy, Kind = improveShuffleKindFromMask(Kind, Mask, SrcTy, Index, SubTp); bool IsExtractSubvector = Kind == TTI::SK_ExtractSubvector; - // A subvector extract can be implemented with an ext (or trivial extract, if - // from lane 0). This currently only handles low or high extracts to prevent - // SLP vectorizer regressions. + // A subvector extract can be implemented with a NEON/SVE ext (or trivial + // extract, if from lane 0) for 128-bit NEON vectors or legal SVE vectors. + // This currently only handles low or high extracts to prevent SLP vectorizer + // regressions. + // Note that SVE's ext instruction is destructive, but it can be fused with + // a movprfx to act like a constructive instruction. if (IsExtractSubvector && LT.second.isFixedLengthVector()) { - if (LT.second.is128BitVector() && + if (LT.second.getFixedSizeInBits() >= 128 && cast<FixedVectorType>(SubTp)->getNumElements() == LT.second.getVectorNumElements() / 2) { if (Index == 0) @@ -6017,9 +6078,15 @@ static bool containsDecreasingPointers(Loop *TheLoop, return false; } -bool AArch64TTIImpl::preferFixedOverScalableIfEqualCost() const { +bool AArch64TTIImpl::preferFixedOverScalableIfEqualCost(bool IsEpilogue) const { if (SVEPreferFixedOverScalableIfEqualCost.getNumOccurrences()) return SVEPreferFixedOverScalableIfEqualCost; + // For cases like post-LTO vectorization, when we eventually know the trip + // count, epilogue with fixed-width vectorization can be deleted if the trip + // count is less than the epilogue iterations. That's why we prefer + // fixed-width vectorization in epilogue in case of equal costs. + if (IsEpilogue) + return true; return ST->useFixedOverScalableIfEqualCost(); } diff --git a/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.h b/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.h index 42ae962b3b42..fe2e849258e3 100644 --- a/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.h +++ b/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.h @@ -174,6 +174,11 @@ public: bool prefersVectorizedAddressing() const override; + /// Check whether Opcode1 has less throughput according to the scheduling + /// model than Opcode2. + bool hasKnownLowerThroughputFromSchedulingModel(unsigned Opcode1, + unsigned Opcode2) const; + InstructionCost getMaskedMemoryOpCost(unsigned Opcode, Type *Src, Align Alignment, unsigned AddressSpace, @@ -424,7 +429,7 @@ public: return TailFoldingStyle::DataWithoutLaneMask; } - bool preferFixedOverScalableIfEqualCost() const override; + bool preferFixedOverScalableIfEqualCost(bool IsEpilogue) const override; unsigned getEpilogueVectorizationMinVF() const override; @@ -460,7 +465,7 @@ public: TTI::TargetCostKind CostKind) const override; InstructionCost getMulAccReductionCost( - bool IsUnsigned, Type *ResTy, VectorType *Ty, + bool IsUnsigned, unsigned RedOpcode, Type *ResTy, VectorType *Ty, TTI::TargetCostKind CostKind = TTI::TCK_RecipThroughput) const override; InstructionCost diff --git a/llvm/lib/Target/AArch64/CMakeLists.txt b/llvm/lib/Target/AArch64/CMakeLists.txt index 803943fd57c4..a8185358d6df 100644 --- a/llvm/lib/Target/AArch64/CMakeLists.txt +++ b/llvm/lib/Target/AArch64/CMakeLists.txt @@ -7,7 +7,8 @@ tablegen(LLVM AArch64GenAsmWriter.inc -gen-asm-writer) tablegen(LLVM AArch64GenAsmWriter1.inc -gen-asm-writer -asmwriternum=1) tablegen(LLVM AArch64GenCallingConv.inc -gen-callingconv) tablegen(LLVM AArch64GenDAGISel.inc -gen-dag-isel) -tablegen(LLVM AArch64GenDisassemblerTables.inc -gen-disassembler) +tablegen(LLVM AArch64GenDisassemblerTables.inc -gen-disassembler + -ignore-non-decodable-operands) tablegen(LLVM AArch64GenFastISel.inc -gen-fast-isel) tablegen(LLVM AArch64GenGlobalISel.inc -gen-global-isel) tablegen(LLVM AArch64GenO0PreLegalizeGICombiner.inc -gen-global-isel-combiner @@ -91,6 +92,7 @@ add_llvm_target(AArch64CodeGen SVEIntrinsicOpts.cpp MachineSMEABIPass.cpp AArch64SIMDInstrOpt.cpp + AArch64PrologueEpilogue.cpp DEPENDS intrinsics_gen @@ -107,6 +109,7 @@ add_llvm_target(AArch64CodeGen Core GlobalISel MC + Passes Scalar SelectionDAG Support diff --git a/llvm/lib/Target/AArch64/Disassembler/AArch64Disassembler.cpp b/llvm/lib/Target/AArch64/Disassembler/AArch64Disassembler.cpp index 323db2a0728e..aa1c1c882e22 100644 --- a/llvm/lib/Target/AArch64/Disassembler/AArch64Disassembler.cpp +++ b/llvm/lib/Target/AArch64/Disassembler/AArch64Disassembler.cpp @@ -35,308 +35,14 @@ using namespace llvm::MCD; // Pull DecodeStatus and its enum values into the global namespace. using DecodeStatus = MCDisassembler::DecodeStatus; -// Forward declare these because the autogenerated code will reference them. -// Definitions are further down. -template <unsigned RegClassID, unsigned FirstReg, unsigned NumRegsInClass> -static DecodeStatus DecodeSimpleRegisterClass(MCInst &Inst, unsigned RegNo, - uint64_t Address, - const MCDisassembler *Decoder); -static DecodeStatus -DecodeGPR64x8ClassRegisterClass(MCInst &Inst, unsigned RegNo, uint64_t Address, - const MCDisassembler *Decoder); -template <unsigned Min, unsigned Max> -static DecodeStatus DecodeZPRMul2_MinMax(MCInst &Inst, unsigned RegNo, - uint64_t Address, - const MCDisassembler *Decoder); -static DecodeStatus DecodeZK(MCInst &Inst, unsigned RegNo, uint64_t Address, - const MCDisassembler *Decoder); -template <unsigned Min, unsigned Max> -static DecodeStatus DecodeZPR2Mul2RegisterClass(MCInst &Inst, unsigned RegNo, - uint64_t Address, - const void *Decoder); -static DecodeStatus DecodeZPR4Mul4RegisterClass(MCInst &Inst, unsigned RegNo, - uint64_t Address, - const void *Decoder); -template <unsigned NumBitsForTile> -static DecodeStatus DecodeMatrixTile(MCInst &Inst, unsigned RegNo, - uint64_t Address, - const MCDisassembler *Decoder); -static DecodeStatus -DecodeMatrixTileListRegisterClass(MCInst &Inst, unsigned RegMask, - uint64_t Address, - const MCDisassembler *Decoder); -static DecodeStatus DecodePPR2Mul2RegisterClass(MCInst &Inst, unsigned RegNo, - uint64_t Address, - const void *Decoder); - -static DecodeStatus DecodeFixedPointScaleImm32(MCInst &Inst, unsigned Imm, - uint64_t Address, - const MCDisassembler *Decoder); -static DecodeStatus DecodeFixedPointScaleImm64(MCInst &Inst, unsigned Imm, - uint64_t Address, - const MCDisassembler *Decoder); -static DecodeStatus DecodePCRelLabel16(MCInst &Inst, unsigned Imm, - uint64_t Address, - const MCDisassembler *Decoder); -static DecodeStatus DecodePCRelLabel19(MCInst &Inst, unsigned Imm, - uint64_t Address, - const MCDisassembler *Decoder); -static DecodeStatus DecodePCRelLabel9(MCInst &Inst, unsigned Imm, - uint64_t Address, - const MCDisassembler *Decoder); -static DecodeStatus DecodeMemExtend(MCInst &Inst, unsigned Imm, - uint64_t Address, - const MCDisassembler *Decoder); -static DecodeStatus DecodeMRSSystemRegister(MCInst &Inst, unsigned Imm, - uint64_t Address, - const MCDisassembler *Decoder); -static DecodeStatus DecodeMSRSystemRegister(MCInst &Inst, unsigned Imm, - uint64_t Address, - const MCDisassembler *Decoder); -static DecodeStatus -DecodeThreeAddrSRegInstruction(MCInst &Inst, uint32_t insn, uint64_t Address, - const MCDisassembler *Decoder); -static DecodeStatus DecodeMoveImmInstruction(MCInst &Inst, uint32_t insn, - uint64_t Address, - const MCDisassembler *Decoder); -static DecodeStatus -DecodeUnsignedLdStInstruction(MCInst &Inst, uint32_t insn, uint64_t Address, - const MCDisassembler *Decoder); -static DecodeStatus DecodeSignedLdStInstruction(MCInst &Inst, uint32_t insn, - uint64_t Address, - const MCDisassembler *Decoder); -static DecodeStatus -DecodeExclusiveLdStInstruction(MCInst &Inst, uint32_t insn, uint64_t Address, - const MCDisassembler *Decoder); -static DecodeStatus DecodePairLdStInstruction(MCInst &Inst, uint32_t insn, - uint64_t Address, - const MCDisassembler *Decoder); -static DecodeStatus DecodeAuthLoadInstruction(MCInst &Inst, uint32_t insn, - uint64_t Address, - const MCDisassembler *Decoder); -static DecodeStatus DecodeAddSubERegInstruction(MCInst &Inst, uint32_t insn, - uint64_t Address, - const MCDisassembler *Decoder); -static DecodeStatus DecodeLogicalImmInstruction(MCInst &Inst, uint32_t insn, - uint64_t Address, - const MCDisassembler *Decoder); -static DecodeStatus DecodeModImmInstruction(MCInst &Inst, uint32_t insn, - uint64_t Address, - const MCDisassembler *Decoder); -static DecodeStatus DecodeModImmTiedInstruction(MCInst &Inst, uint32_t insn, - uint64_t Address, - const MCDisassembler *Decoder); -static DecodeStatus DecodeAdrInstruction(MCInst &Inst, uint32_t insn, - uint64_t Address, - const MCDisassembler *Decoder); -static DecodeStatus DecodeAddSubImmShift(MCInst &Inst, uint32_t insn, - uint64_t Address, - const MCDisassembler *Decoder); -static DecodeStatus DecodeUnconditionalBranch(MCInst &Inst, uint32_t insn, - uint64_t Address, - const MCDisassembler *Decoder); -static DecodeStatus -DecodeSystemPStateImm0_15Instruction(MCInst &Inst, uint32_t insn, - uint64_t Address, - const MCDisassembler *Decoder); -static DecodeStatus -DecodeSystemPStateImm0_1Instruction(MCInst &Inst, uint32_t insn, - uint64_t Address, - const MCDisassembler *Decoder); -static DecodeStatus DecodeTestAndBranch(MCInst &Inst, uint32_t insn, - uint64_t Address, - const MCDisassembler *Decoder); - -static DecodeStatus DecodeFMOVLaneInstruction(MCInst &Inst, unsigned Insn, - uint64_t Address, - const MCDisassembler *Decoder); -static DecodeStatus DecodeVecShiftR64Imm(MCInst &Inst, unsigned Imm, - uint64_t Addr, - const MCDisassembler *Decoder); -static DecodeStatus DecodeVecShiftR64ImmNarrow(MCInst &Inst, unsigned Imm, - uint64_t Addr, - const MCDisassembler *Decoder); -static DecodeStatus DecodeVecShiftR32Imm(MCInst &Inst, unsigned Imm, - uint64_t Addr, - const MCDisassembler *Decoder); -static DecodeStatus DecodeVecShiftR32ImmNarrow(MCInst &Inst, unsigned Imm, - uint64_t Addr, - const MCDisassembler *Decoder); -static DecodeStatus DecodeVecShiftR16Imm(MCInst &Inst, unsigned Imm, - uint64_t Addr, - const MCDisassembler *Decoder); -static DecodeStatus DecodeVecShiftR16ImmNarrow(MCInst &Inst, unsigned Imm, - uint64_t Addr, - const MCDisassembler *Decoder); -static DecodeStatus DecodeVecShiftR8Imm(MCInst &Inst, unsigned Imm, - uint64_t Addr, - const MCDisassembler *Decoder); -static DecodeStatus DecodeVecShiftL64Imm(MCInst &Inst, unsigned Imm, - uint64_t Addr, - const MCDisassembler *Decoder); -static DecodeStatus DecodeVecShiftL32Imm(MCInst &Inst, unsigned Imm, - uint64_t Addr, - const MCDisassembler *Decoder); -static DecodeStatus DecodeVecShiftL16Imm(MCInst &Inst, unsigned Imm, - uint64_t Addr, - const MCDisassembler *Decoder); -static DecodeStatus DecodeVecShiftL8Imm(MCInst &Inst, unsigned Imm, - uint64_t Addr, - const MCDisassembler *Decoder); -static DecodeStatus -DecodeWSeqPairsClassRegisterClass(MCInst &Inst, unsigned RegNo, uint64_t Addr, - const MCDisassembler *Decoder); -static DecodeStatus -DecodeXSeqPairsClassRegisterClass(MCInst &Inst, unsigned RegNo, uint64_t Addr, - const MCDisassembler *Decoder); -static DecodeStatus DecodeSyspXzrInstruction(MCInst &Inst, uint32_t insn, - uint64_t Addr, - const MCDisassembler *Decoder); -static DecodeStatus -DecodeSVELogicalImmInstruction(MCInst &Inst, uint32_t insn, uint64_t Address, - const MCDisassembler *Decoder); template <int Bits> static DecodeStatus DecodeSImm(MCInst &Inst, uint64_t Imm, uint64_t Address, const MCDisassembler *Decoder); -template <int ElementWidth> -static DecodeStatus DecodeImm8OptLsl(MCInst &Inst, unsigned Imm, uint64_t Addr, - const MCDisassembler *Decoder); -static DecodeStatus DecodeSVEIncDecImm(MCInst &Inst, unsigned Imm, - uint64_t Addr, - const MCDisassembler *Decoder); -static DecodeStatus DecodeSVCROp(MCInst &Inst, unsigned Imm, uint64_t Address, - const MCDisassembler *Decoder); -static DecodeStatus DecodeCPYMemOpInstruction(MCInst &Inst, uint32_t insn, - uint64_t Addr, - const MCDisassembler *Decoder); -static DecodeStatus DecodeSETMemOpInstruction(MCInst &Inst, uint32_t insn, - uint64_t Addr, - const MCDisassembler *Decoder); -static DecodeStatus DecodePRFMRegInstruction(MCInst &Inst, uint32_t insn, - uint64_t Address, - const MCDisassembler *Decoder); - -#include "AArch64GenDisassemblerTables.inc" -#include "AArch64GenInstrInfo.inc" #define Success MCDisassembler::Success #define Fail MCDisassembler::Fail #define SoftFail MCDisassembler::SoftFail -static MCDisassembler *createAArch64Disassembler(const Target &T, - const MCSubtargetInfo &STI, - MCContext &Ctx) { - - return new AArch64Disassembler(STI, Ctx, T.createMCInstrInfo()); -} - -DecodeStatus AArch64Disassembler::getInstruction(MCInst &MI, uint64_t &Size, - ArrayRef<uint8_t> Bytes, - uint64_t Address, - raw_ostream &CS) const { - CommentStream = &CS; - - Size = 0; - // We want to read exactly 4 bytes of data. - if (Bytes.size() < 4) - return Fail; - Size = 4; - - // Encoded as a small-endian 32-bit word in the stream. - uint32_t Insn = - (Bytes[3] << 24) | (Bytes[2] << 16) | (Bytes[1] << 8) | (Bytes[0] << 0); - - const uint8_t *Tables[] = {DecoderTable32, DecoderTableFallback32}; - - for (const auto *Table : Tables) { - DecodeStatus Result = - decodeInstruction(Table, MI, Insn, Address, this, STI); - - const MCInstrDesc &Desc = MCII->get(MI.getOpcode()); - - // For Scalable Matrix Extension (SME) instructions that have an implicit - // operand for the accumulator (ZA) or implicit immediate zero which isn't - // encoded, manually insert operand. - for (unsigned i = 0; i < Desc.getNumOperands(); i++) { - if (Desc.operands()[i].OperandType == MCOI::OPERAND_REGISTER) { - switch (Desc.operands()[i].RegClass) { - default: - break; - case AArch64::MPRRegClassID: - MI.insert(MI.begin() + i, MCOperand::createReg(AArch64::ZA)); - break; - case AArch64::MPR8RegClassID: - MI.insert(MI.begin() + i, MCOperand::createReg(AArch64::ZAB0)); - break; - case AArch64::ZTRRegClassID: - MI.insert(MI.begin() + i, MCOperand::createReg(AArch64::ZT0)); - break; - } - } else if (Desc.operands()[i].OperandType == - AArch64::OPERAND_IMPLICIT_IMM_0) { - MI.insert(MI.begin() + i, MCOperand::createImm(0)); - } - } - - if (MI.getOpcode() == AArch64::LDR_ZA || - MI.getOpcode() == AArch64::STR_ZA) { - // Spill and fill instructions have a single immediate used for both - // the vector select offset and optional memory offset. Replicate - // the decoded immediate. - const MCOperand &Imm4Op = MI.getOperand(2); - assert(Imm4Op.isImm() && "Unexpected operand type!"); - MI.addOperand(Imm4Op); - } - - if (Result != MCDisassembler::Fail) - return Result; - } - - return MCDisassembler::Fail; -} - -uint64_t AArch64Disassembler::suggestBytesToSkip(ArrayRef<uint8_t> Bytes, - uint64_t Address) const { - // AArch64 instructions are always 4 bytes wide, so there's no point - // in skipping any smaller number of bytes if an instruction can't - // be decoded. - return 4; -} - -static MCSymbolizer * -createAArch64ExternalSymbolizer(const Triple &TT, LLVMOpInfoCallback GetOpInfo, - LLVMSymbolLookupCallback SymbolLookUp, - void *DisInfo, MCContext *Ctx, - std::unique_ptr<MCRelocationInfo> &&RelInfo) { - return new AArch64ExternalSymbolizer(*Ctx, std::move(RelInfo), GetOpInfo, - SymbolLookUp, DisInfo); -} - -extern "C" LLVM_ABI LLVM_EXTERNAL_VISIBILITY void -LLVMInitializeAArch64Disassembler() { - TargetRegistry::RegisterMCDisassembler(getTheAArch64leTarget(), - createAArch64Disassembler); - TargetRegistry::RegisterMCDisassembler(getTheAArch64beTarget(), - createAArch64Disassembler); - TargetRegistry::RegisterMCSymbolizer(getTheAArch64leTarget(), - createAArch64ExternalSymbolizer); - TargetRegistry::RegisterMCSymbolizer(getTheAArch64beTarget(), - createAArch64ExternalSymbolizer); - TargetRegistry::RegisterMCDisassembler(getTheAArch64_32Target(), - createAArch64Disassembler); - TargetRegistry::RegisterMCSymbolizer(getTheAArch64_32Target(), - createAArch64ExternalSymbolizer); - - TargetRegistry::RegisterMCDisassembler(getTheARM64Target(), - createAArch64Disassembler); - TargetRegistry::RegisterMCSymbolizer(getTheARM64Target(), - createAArch64ExternalSymbolizer); - TargetRegistry::RegisterMCDisassembler(getTheARM64_32Target(), - createAArch64Disassembler); - TargetRegistry::RegisterMCSymbolizer(getTheARM64_32Target(), - createAArch64ExternalSymbolizer); -} - template <unsigned RegClassID, unsigned FirstReg, unsigned NumRegsInClass> static DecodeStatus DecodeSimpleRegisterClass(MCInst &Inst, unsigned RegNo, uint64_t Address, @@ -492,11 +198,7 @@ static DecodeStatus DecodePCRelLabel16(MCInst &Inst, unsigned Imm, static DecodeStatus DecodePCRelLabel19(MCInst &Inst, unsigned Imm, uint64_t Addr, const MCDisassembler *Decoder) { - int64_t ImmVal = Imm; - - // Sign-extend 19-bit immediate. - if (ImmVal & (1 << (19 - 1))) - ImmVal |= ~((1LL << 19) - 1); + int64_t ImmVal = SignExtend64<19>(Imm); if (!Decoder->tryAddingSymbolicOperand( Inst, ImmVal * 4, Addr, Inst.getOpcode() != AArch64::LDRXl, 0, 0, 4)) @@ -506,11 +208,7 @@ static DecodeStatus DecodePCRelLabel19(MCInst &Inst, unsigned Imm, static DecodeStatus DecodePCRelLabel9(MCInst &Inst, unsigned Imm, uint64_t Addr, const MCDisassembler *Decoder) { - int64_t ImmVal = Imm; - - // Sign-extend 9-bit immediate. - if (ImmVal & (1 << (9 - 1))) - ImmVal |= ~((1LL << 9) - 1); + int64_t ImmVal = SignExtend64<9>(Imm); if (!Decoder->tryAddingSymbolicOperand(Inst, (ImmVal * 4), Addr, /*IsBranch=*/true, 0, 0, 4)) @@ -827,12 +525,7 @@ static DecodeStatus DecodeSignedLdStInstruction(MCInst &Inst, uint32_t insn, const MCDisassembler *Decoder) { unsigned Rt = fieldFromInstruction(insn, 0, 5); unsigned Rn = fieldFromInstruction(insn, 5, 5); - int64_t offset = fieldFromInstruction(insn, 12, 9); - - // offset is a 9-bit signed immediate, so sign extend it to - // fill the unsigned. - if (offset & (1 << (9 - 1))) - offset |= ~((1LL << 9) - 1); + int64_t offset = SignExtend64<9>(fieldFromInstruction(insn, 12, 9)); // First operand is always the writeback to the address register, if needed. switch (Inst.getOpcode()) { @@ -1129,14 +822,9 @@ static DecodeStatus DecodePairLdStInstruction(MCInst &Inst, uint32_t insn, unsigned Rt = fieldFromInstruction(insn, 0, 5); unsigned Rn = fieldFromInstruction(insn, 5, 5); unsigned Rt2 = fieldFromInstruction(insn, 10, 5); - int64_t offset = fieldFromInstruction(insn, 15, 7); + int64_t offset = SignExtend64<7>(fieldFromInstruction(insn, 15, 7)); bool IsLoad = fieldFromInstruction(insn, 22, 1); - // offset is a 7-bit signed immediate, so sign extend it to - // fill the unsigned. - if (offset & (1 << (7 - 1))) - offset |= ~((1LL << 7) - 1); - unsigned Opcode = Inst.getOpcode(); bool NeedsDisjointWritebackTransfer = false; @@ -1505,12 +1193,8 @@ static DecodeStatus DecodeAdrInstruction(MCInst &Inst, uint32_t insn, uint64_t Addr, const MCDisassembler *Decoder) { unsigned Rd = fieldFromInstruction(insn, 0, 5); - int64_t imm = fieldFromInstruction(insn, 5, 19) << 2; - imm |= fieldFromInstruction(insn, 29, 2); - - // Sign-extend the 21-bit immediate. - if (imm & (1 << (21 - 1))) - imm |= ~((1LL << 21) - 1); + int64_t imm = SignExtend64<21>((fieldFromInstruction(insn, 5, 19) << 2) | + fieldFromInstruction(insn, 29, 2)); DecodeSimpleRegisterClass<AArch64::GPR64RegClassID, 0, 32>(Inst, Rd, Addr, Decoder); @@ -1564,11 +1248,7 @@ static DecodeStatus DecodeAddSubImmShift(MCInst &Inst, uint32_t insn, static DecodeStatus DecodeUnconditionalBranch(MCInst &Inst, uint32_t insn, uint64_t Addr, const MCDisassembler *Decoder) { - int64_t imm = fieldFromInstruction(insn, 0, 26); - - // Sign-extend the 26-bit immediate. - if (imm & (1 << (26 - 1))) - imm |= ~((1LL << 26) - 1); + int64_t imm = SignExtend64<26>(fieldFromInstruction(insn, 0, 26)); if (!Decoder->tryAddingSymbolicOperand(Inst, imm * 4, Addr, true, 0, 0, 4)) Inst.addOperand(MCOperand::createImm(imm)); @@ -1631,11 +1311,7 @@ static DecodeStatus DecodeTestAndBranch(MCInst &Inst, uint32_t insn, uint64_t Rt = fieldFromInstruction(insn, 0, 5); uint64_t bit = fieldFromInstruction(insn, 31, 1) << 5; bit |= fieldFromInstruction(insn, 19, 5); - int64_t dst = fieldFromInstruction(insn, 5, 14); - - // Sign-extend 14-bit immediate. - if (dst & (1 << (14 - 1))) - dst |= ~((1LL << 14) - 1); + int64_t dst = SignExtend64<14>(fieldFromInstruction(insn, 5, 14)); if (fieldFromInstruction(insn, 31, 1) == 0) DecodeSimpleRegisterClass<AArch64::GPR32RegClassID, 0, 32>(Inst, Rt, Addr, @@ -1856,3 +1532,129 @@ static DecodeStatus DecodePRFMRegInstruction(MCInst &Inst, uint32_t insn, return Success; } + +static DecodeStatus +DecodeSMESpillFillInstruction(MCInst &Inst, uint32_t Bits, uint64_t Addr, + const MCDisassembler *Decoder) { + unsigned RvBits = fieldFromInstruction(Bits, 13, 2); + unsigned RnBits = fieldFromInstruction(Bits, 5, 5); + unsigned Imm4Bits = fieldFromInstruction(Bits, 0, 4); + + DecodeSimpleRegisterClass<AArch64::MatrixIndexGPR32_12_15RegClassID, 0, 4>( + Inst, RvBits, Addr, Decoder); + Inst.addOperand(MCOperand::createImm(Imm4Bits)); + DecodeSimpleRegisterClass<AArch64::GPR64spRegClassID, 0, 32>(Inst, RnBits, + Addr, Decoder); + // Spill and fill instructions have a single immediate used for both + // the vector select offset and optional memory offset. Replicate + // the decoded immediate. + Inst.addOperand(MCOperand::createImm(Imm4Bits)); + return Success; +} + +#include "AArch64GenDisassemblerTables.inc" +#include "AArch64GenInstrInfo.inc" + +static MCDisassembler *createAArch64Disassembler(const Target &T, + const MCSubtargetInfo &STI, + MCContext &Ctx) { + + return new AArch64Disassembler(STI, Ctx, T.createMCInstrInfo()); +} + +DecodeStatus AArch64Disassembler::getInstruction(MCInst &MI, uint64_t &Size, + ArrayRef<uint8_t> Bytes, + uint64_t Address, + raw_ostream &CS) const { + CommentStream = &CS; + + Size = 0; + // We want to read exactly 4 bytes of data. + if (Bytes.size() < 4) + return Fail; + Size = 4; + + // Encoded as a small-endian 32-bit word in the stream. + uint32_t Insn = + (Bytes[3] << 24) | (Bytes[2] << 16) | (Bytes[1] << 8) | (Bytes[0] << 0); + + const uint8_t *Tables[] = {DecoderTable32, DecoderTableFallback32}; + + for (const auto *Table : Tables) { + DecodeStatus Result = + decodeInstruction(Table, MI, Insn, Address, this, STI); + + const MCInstrDesc &Desc = MCII->get(MI.getOpcode()); + + // For Scalable Matrix Extension (SME) instructions that have an implicit + // operand for the accumulator (ZA) or implicit immediate zero which isn't + // encoded, manually insert operand. + for (unsigned i = 0; i < Desc.getNumOperands(); i++) { + if (Desc.operands()[i].OperandType == MCOI::OPERAND_REGISTER) { + switch (Desc.operands()[i].RegClass) { + default: + break; + case AArch64::MPRRegClassID: + MI.insert(MI.begin() + i, MCOperand::createReg(AArch64::ZA)); + break; + case AArch64::MPR8RegClassID: + MI.insert(MI.begin() + i, MCOperand::createReg(AArch64::ZAB0)); + break; + case AArch64::ZTRRegClassID: + MI.insert(MI.begin() + i, MCOperand::createReg(AArch64::ZT0)); + break; + } + } else if (Desc.operands()[i].OperandType == + AArch64::OPERAND_IMPLICIT_IMM_0) { + MI.insert(MI.begin() + i, MCOperand::createImm(0)); + } + } + + if (Result != MCDisassembler::Fail) + return Result; + } + + return MCDisassembler::Fail; +} + +uint64_t AArch64Disassembler::suggestBytesToSkip(ArrayRef<uint8_t> Bytes, + uint64_t Address) const { + // AArch64 instructions are always 4 bytes wide, so there's no point + // in skipping any smaller number of bytes if an instruction can't + // be decoded. + return 4; +} + +static MCSymbolizer * +createAArch64ExternalSymbolizer(const Triple &TT, LLVMOpInfoCallback GetOpInfo, + LLVMSymbolLookupCallback SymbolLookUp, + void *DisInfo, MCContext *Ctx, + std::unique_ptr<MCRelocationInfo> &&RelInfo) { + return new AArch64ExternalSymbolizer(*Ctx, std::move(RelInfo), GetOpInfo, + SymbolLookUp, DisInfo); +} + +extern "C" LLVM_ABI LLVM_EXTERNAL_VISIBILITY void +LLVMInitializeAArch64Disassembler() { + TargetRegistry::RegisterMCDisassembler(getTheAArch64leTarget(), + createAArch64Disassembler); + TargetRegistry::RegisterMCDisassembler(getTheAArch64beTarget(), + createAArch64Disassembler); + TargetRegistry::RegisterMCSymbolizer(getTheAArch64leTarget(), + createAArch64ExternalSymbolizer); + TargetRegistry::RegisterMCSymbolizer(getTheAArch64beTarget(), + createAArch64ExternalSymbolizer); + TargetRegistry::RegisterMCDisassembler(getTheAArch64_32Target(), + createAArch64Disassembler); + TargetRegistry::RegisterMCSymbolizer(getTheAArch64_32Target(), + createAArch64ExternalSymbolizer); + + TargetRegistry::RegisterMCDisassembler(getTheARM64Target(), + createAArch64Disassembler); + TargetRegistry::RegisterMCSymbolizer(getTheARM64Target(), + createAArch64ExternalSymbolizer); + TargetRegistry::RegisterMCDisassembler(getTheARM64_32Target(), + createAArch64Disassembler); + TargetRegistry::RegisterMCSymbolizer(getTheARM64_32Target(), + createAArch64ExternalSymbolizer); +} diff --git a/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp b/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp index 0bceb322726d..5748556d0728 100644 --- a/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp +++ b/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp @@ -6608,45 +6608,6 @@ bool AArch64InstructionSelector::selectIntrinsic(MachineInstr &I, switch (IntrinID) { default: break; - case Intrinsic::aarch64_crypto_sha1h: { - Register DstReg = I.getOperand(0).getReg(); - Register SrcReg = I.getOperand(2).getReg(); - - // FIXME: Should this be an assert? - if (MRI.getType(DstReg).getSizeInBits() != 32 || - MRI.getType(SrcReg).getSizeInBits() != 32) - return false; - - // The operation has to happen on FPRs. Set up some new FPR registers for - // the source and destination if they are on GPRs. - if (RBI.getRegBank(SrcReg, MRI, TRI)->getID() != AArch64::FPRRegBankID) { - SrcReg = MRI.createVirtualRegister(&AArch64::FPR32RegClass); - MIB.buildCopy({SrcReg}, {I.getOperand(2)}); - - // Make sure the copy ends up getting constrained properly. - RBI.constrainGenericRegister(I.getOperand(2).getReg(), - AArch64::GPR32RegClass, MRI); - } - - if (RBI.getRegBank(DstReg, MRI, TRI)->getID() != AArch64::FPRRegBankID) - DstReg = MRI.createVirtualRegister(&AArch64::FPR32RegClass); - - // Actually insert the instruction. - auto SHA1Inst = MIB.buildInstr(AArch64::SHA1Hrr, {DstReg}, {SrcReg}); - constrainSelectedInstRegOperands(*SHA1Inst, TII, TRI, RBI); - - // Did we create a new register for the destination? - if (DstReg != I.getOperand(0).getReg()) { - // Yep. Copy the result of the instruction back into the original - // destination. - MIB.buildCopy({I.getOperand(0)}, {DstReg}); - RBI.constrainGenericRegister(I.getOperand(0).getReg(), - AArch64::GPR32RegClass, MRI); - } - - I.eraseFromParent(); - return true; - } case Intrinsic::ptrauth_resign: { Register DstReg = I.getOperand(0).getReg(); Register ValReg = I.getOperand(2).getReg(); diff --git a/llvm/lib/Target/AArch64/GISel/AArch64LegalizerInfo.cpp b/llvm/lib/Target/AArch64/GISel/AArch64LegalizerInfo.cpp index 210643f6f2f4..ff09b375c310 100644 --- a/llvm/lib/Target/AArch64/GISel/AArch64LegalizerInfo.cpp +++ b/llvm/lib/Target/AArch64/GISel/AArch64LegalizerInfo.cpp @@ -222,7 +222,9 @@ AArch64LegalizerInfo::AArch64LegalizerInfo(const AArch64Subtarget &ST) .clampNumElements(0, v2s64, v2s64) .moreElementsToNextPow2(0) .minScalarSameAs(1, 0) - .scalarizeIf(scalarOrEltWiderThan(0, 64), 0); + .scalarizeIf(scalarOrEltWiderThan(0, 64), 0) + .minScalarEltSameAsIf(isVector(0), 1, 0) + .maxScalarEltSameAsIf(isVector(0), 1, 0); getActionDefinitionsBuilder(G_PTR_ADD) .legalFor({{p0, s64}, {v2p0, v2s64}}) @@ -879,8 +881,9 @@ AArch64LegalizerInfo::AArch64LegalizerInfo(const AArch64Subtarget &ST) {v2s32, v2s32}, {v4s32, v4s32}, {v2s64, v2s64}}) - .legalFor(HasFP16, - {{s32, s16}, {s64, s16}, {v4s16, v4s16}, {v8s16, v8s16}}) + .legalFor( + HasFP16, + {{s16, s16}, {s32, s16}, {s64, s16}, {v4s16, v4s16}, {v8s16, v8s16}}) // Handle types larger than i64 by scalarizing/lowering. .scalarizeIf(scalarOrEltWiderThan(0, 64), 0) .scalarizeIf(scalarOrEltWiderThan(1, 64), 1) @@ -1150,7 +1153,8 @@ AArch64LegalizerInfo::AArch64LegalizerInfo(const AArch64Subtarget &ST) .clampMaxNumElements(1, s32, 4) .clampMaxNumElements(1, s16, 8) .clampMaxNumElements(1, s8, 16) - .clampMaxNumElements(1, p0, 2); + .clampMaxNumElements(1, p0, 2) + .scalarizeIf(scalarOrEltWiderThan(1, 64), 1); getActionDefinitionsBuilder(G_INSERT_VECTOR_ELT) .legalIf( @@ -1165,7 +1169,8 @@ AArch64LegalizerInfo::AArch64LegalizerInfo(const AArch64Subtarget &ST) .clampNumElements(0, v4s16, v8s16) .clampNumElements(0, v2s32, v4s32) .clampMaxNumElements(0, s64, 2) - .clampMaxNumElements(0, p0, 2); + .clampMaxNumElements(0, p0, 2) + .scalarizeIf(scalarOrEltWiderThan(0, 64), 0); getActionDefinitionsBuilder(G_BUILD_VECTOR) .legalFor({{v8s8, s8}, diff --git a/llvm/lib/Target/AArch64/GISel/AArch64RegisterBankInfo.cpp b/llvm/lib/Target/AArch64/GISel/AArch64RegisterBankInfo.cpp index 1b919abd222e..62de86bf87f5 100644 --- a/llvm/lib/Target/AArch64/GISel/AArch64RegisterBankInfo.cpp +++ b/llvm/lib/Target/AArch64/GISel/AArch64RegisterBankInfo.cpp @@ -482,6 +482,10 @@ static bool isFPIntrinsic(const MachineRegisterInfo &MRI, case Intrinsic::aarch64_neon_sqrdmulh: case Intrinsic::aarch64_neon_sqadd: case Intrinsic::aarch64_neon_sqsub: + case Intrinsic::aarch64_crypto_sha1h: + case Intrinsic::aarch64_crypto_sha1c: + case Intrinsic::aarch64_crypto_sha1p: + case Intrinsic::aarch64_crypto_sha1m: return true; case Intrinsic::aarch64_neon_saddlv: { const LLT SrcTy = MRI.getType(MI.getOperand(2).getReg()); @@ -848,10 +852,20 @@ AArch64RegisterBankInfo::getInstrMapping(const MachineInstr &MI) const { OpRegBankIdx = {PMI_FirstFPR, PMI_FirstGPR}; break; } + case TargetOpcode::G_FPTOSI_SAT: + case TargetOpcode::G_FPTOUI_SAT: { + LLT DstType = MRI.getType(MI.getOperand(0).getReg()); + if (DstType.isVector()) + break; + if (DstType == LLT::scalar(16)) { + OpRegBankIdx = {PMI_FirstFPR, PMI_FirstFPR}; + break; + } + OpRegBankIdx = {PMI_FirstGPR, PMI_FirstFPR}; + break; + } case TargetOpcode::G_FPTOSI: case TargetOpcode::G_FPTOUI: - case TargetOpcode::G_FPTOSI_SAT: - case TargetOpcode::G_FPTOUI_SAT: case TargetOpcode::G_INTRINSIC_LRINT: case TargetOpcode::G_INTRINSIC_LLRINT: if (MRI.getType(MI.getOperand(0).getReg()).isVector()) diff --git a/llvm/lib/Target/AArch64/MCTargetDesc/AArch64InstPrinter.cpp b/llvm/lib/Target/AArch64/MCTargetDesc/AArch64InstPrinter.cpp index 54b58e948daf..2552ee300933 100644 --- a/llvm/lib/Target/AArch64/MCTargetDesc/AArch64InstPrinter.cpp +++ b/llvm/lib/Target/AArch64/MCTargetDesc/AArch64InstPrinter.cpp @@ -365,13 +365,6 @@ void AArch64InstPrinter::printInst(const MCInst *MI, uint64_t Address, return; } - // Instruction TSB is specified as a one operand instruction, but 'csync' is - // not encoded, so for printing it is treated as a special case here: - if (Opcode == AArch64::TSB) { - O << "\ttsb\tcsync"; - return; - } - if (!PrintAliases || !printAliasInstr(MI, Address, STI, O)) printInstruction(MI, Address, STI, O); diff --git a/llvm/lib/Target/AArch64/MCTargetDesc/AArch64MCAsmInfo.cpp b/llvm/lib/Target/AArch64/MCTargetDesc/AArch64MCAsmInfo.cpp index 828c5c546240..2b5cf3484ffc 100644 --- a/llvm/lib/Target/AArch64/MCTargetDesc/AArch64MCAsmInfo.cpp +++ b/llvm/lib/Target/AArch64/MCTargetDesc/AArch64MCAsmInfo.cpp @@ -53,9 +53,9 @@ const MCAsmInfo::AtSpecifier MachOAtSpecifiers[] = { {AArch64::S_MACHO_TLVPPAGEOFF, "TLVPPAGEOFF"}, }; -StringRef AArch64::getSpecifierName(const MCSpecifierExpr &Expr) { +StringRef AArch64::getSpecifierName(AArch64::Specifier S) { // clang-format off - switch (static_cast<uint32_t>(Expr.getSpecifier())) { + switch (static_cast<uint32_t>(S)) { case AArch64::S_CALL: return ""; case AArch64::S_LO12: return ":lo12:"; case AArch64::S_ABS_G3: return ":abs_g3:"; @@ -124,7 +124,7 @@ static bool evaluate(const MCSpecifierExpr &Expr, MCValue &Res, if (!Expr.getSubExpr()->evaluateAsRelocatable(Res, Asm)) return false; Res.setSpecifier(Expr.getSpecifier()); - return true; + return !Res.getSubSym(); } AArch64MCAsmInfoDarwin::AArch64MCAsmInfoDarwin(bool IsILP32) { @@ -183,7 +183,7 @@ void AArch64MCAsmInfoDarwin::printSpecifierExpr( raw_ostream &OS, const MCSpecifierExpr &Expr) const { if (auto *AE = dyn_cast<AArch64AuthMCExpr>(&Expr)) return AE->print(OS, this); - OS << AArch64::getSpecifierName(Expr); + OS << AArch64::getSpecifierName(Expr.getSpecifier()); printExpr(OS, *Expr.getSubExpr()); } @@ -232,7 +232,7 @@ void AArch64MCAsmInfoELF::printSpecifierExpr( raw_ostream &OS, const MCSpecifierExpr &Expr) const { if (auto *AE = dyn_cast<AArch64AuthMCExpr>(&Expr)) return AE->print(OS, this); - OS << AArch64::getSpecifierName(Expr); + OS << AArch64::getSpecifierName(Expr.getSpecifier()); printExpr(OS, *Expr.getSubExpr()); } @@ -262,7 +262,7 @@ AArch64MCAsmInfoMicrosoftCOFF::AArch64MCAsmInfoMicrosoftCOFF() { void AArch64MCAsmInfoMicrosoftCOFF::printSpecifierExpr( raw_ostream &OS, const MCSpecifierExpr &Expr) const { - OS << AArch64::getSpecifierName(Expr); + OS << AArch64::getSpecifierName(Expr.getSpecifier()); printExpr(OS, *Expr.getSubExpr()); } @@ -292,7 +292,7 @@ AArch64MCAsmInfoGNUCOFF::AArch64MCAsmInfoGNUCOFF() { void AArch64MCAsmInfoGNUCOFF::printSpecifierExpr( raw_ostream &OS, const MCSpecifierExpr &Expr) const { - OS << AArch64::getSpecifierName(Expr); + OS << AArch64::getSpecifierName(Expr.getSpecifier()); printExpr(OS, *Expr.getSubExpr()); } diff --git a/llvm/lib/Target/AArch64/MCTargetDesc/AArch64MCAsmInfo.h b/llvm/lib/Target/AArch64/MCTargetDesc/AArch64MCAsmInfo.h index c28e925d77e2..0dfa61b1dc60 100644 --- a/llvm/lib/Target/AArch64/MCTargetDesc/AArch64MCAsmInfo.h +++ b/llvm/lib/Target/AArch64/MCTargetDesc/AArch64MCAsmInfo.h @@ -181,7 +181,7 @@ enum { /// Return the string representation of the ELF relocation specifier /// (e.g. ":got:", ":lo12:"). -StringRef getSpecifierName(const MCSpecifierExpr &Expr); +StringRef getSpecifierName(Specifier S); inline Specifier getSymbolLoc(Specifier S) { return static_cast<Specifier>(S & AArch64::S_SymLocBits); diff --git a/llvm/lib/Target/AArch64/MCTargetDesc/AArch64MCTargetDesc.h b/llvm/lib/Target/AArch64/MCTargetDesc/AArch64MCTargetDesc.h index 91bdc880998b..7774d07a214b 100644 --- a/llvm/lib/Target/AArch64/MCTargetDesc/AArch64MCTargetDesc.h +++ b/llvm/lib/Target/AArch64/MCTargetDesc/AArch64MCTargetDesc.h @@ -65,15 +65,16 @@ void initLLVMToCVRegMapping(MCRegisterInfo *MRI); bool isHForm(const MCInst &MI, const MCInstrInfo *MCII); bool isQForm(const MCInst &MI, const MCInstrInfo *MCII); bool isFpOrNEON(const MCInst &MI, const MCInstrInfo *MCII); -} +} // namespace AArch64_MC namespace AArch64 { enum OperandType { OPERAND_IMPLICIT_IMM_0 = MCOI::OPERAND_FIRST_TARGET, + OPERAND_SHIFT_MSL, }; } // namespace AArch64 -} // End llvm namespace +} // namespace llvm // Defines symbolic names for AArch64 registers. This defines a mapping from // register name to register number. diff --git a/llvm/lib/Target/AArch64/MCTargetDesc/AArch64WinCOFFObjectWriter.cpp b/llvm/lib/Target/AArch64/MCTargetDesc/AArch64WinCOFFObjectWriter.cpp index a53b676142a0..5fe999389ce7 100644 --- a/llvm/lib/Target/AArch64/MCTargetDesc/AArch64WinCOFFObjectWriter.cpp +++ b/llvm/lib/Target/AArch64/MCTargetDesc/AArch64WinCOFFObjectWriter.cpp @@ -73,9 +73,10 @@ unsigned AArch64WinCOFFObjectWriter::getRelocType( // Supported break; default: - Ctx.reportError(Fixup.getLoc(), "relocation specifier " + - AArch64::getSpecifierName(*A64E) + - " unsupported on COFF targets"); + Ctx.reportError(Fixup.getLoc(), + "relocation specifier " + + AArch64::getSpecifierName(A64E->getSpecifier()) + + " unsupported on COFF targets"); return COFF::IMAGE_REL_ARM64_ABSOLUTE; // Dummy return value } } @@ -83,9 +84,10 @@ unsigned AArch64WinCOFFObjectWriter::getRelocType( switch (FixupKind) { default: { if (auto *A64E = dyn_cast<MCSpecifierExpr>(Expr)) { - Ctx.reportError(Fixup.getLoc(), "relocation specifier " + - AArch64::getSpecifierName(*A64E) + - " unsupported on COFF targets"); + Ctx.reportError(Fixup.getLoc(), + "relocation specifier " + + AArch64::getSpecifierName(A64E->getSpecifier()) + + " unsupported on COFF targets"); } else { MCFixupKindInfo Info = MAB.getFixupKindInfo(Fixup.getKind()); Ctx.reportError(Fixup.getLoc(), Twine("relocation type ") + Info.Name + diff --git a/llvm/lib/Target/AArch64/MachineSMEABIPass.cpp b/llvm/lib/Target/AArch64/MachineSMEABIPass.cpp index b58dfdf32e4a..c39a5cc2fcb1 100644 --- a/llvm/lib/Target/AArch64/MachineSMEABIPass.cpp +++ b/llvm/lib/Target/AArch64/MachineSMEABIPass.cpp @@ -7,7 +7,7 @@ //===----------------------------------------------------------------------===// // // This pass implements the SME ABI requirements for ZA state. This includes -// implementing the lazy ZA state save schemes around calls. +// implementing the lazy (and agnostic) ZA state save schemes around calls. // //===----------------------------------------------------------------------===// // @@ -139,8 +139,8 @@ StringRef getZAStateString(ZAState State) { #undef MAKE_CASE } -static bool isZAorZT0RegOp(const TargetRegisterInfo &TRI, - const MachineOperand &MO) { +static bool isZAorZTRegOp(const TargetRegisterInfo &TRI, + const MachineOperand &MO) { if (!MO.isReg() || !MO.getReg().isPhysical()) return false; return any_of(TRI.subregs_inclusive(MO.getReg()), [](const MCPhysReg &SR) { @@ -166,7 +166,7 @@ getZAStateBeforeInst(const TargetRegisterInfo &TRI, MachineInstr &MI, return {ZAOffAtReturn ? ZAState::OFF : ZAState::ACTIVE, InsertPt}; for (auto &MO : MI.operands()) { - if (isZAorZT0RegOp(TRI, MO)) + if (isZAorZTRegOp(TRI, MO)) return {ZAState::ACTIVE, InsertPt}; } @@ -215,9 +215,44 @@ struct MachineSMEABI : public MachineFunctionPass { void emitZAOff(MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI, bool ClearTPIDR2); + // Emission routines for agnostic ZA functions. + void emitSetupFullZASave(MachineBasicBlock &MBB, + MachineBasicBlock::iterator MBBI, + LiveRegs PhysLiveRegs); + // Emit a "full" ZA save or restore. It is "full" in the sense that this + // function will emit a call to __arm_sme_save or __arm_sme_restore, which + // handles saving and restoring both ZA and ZT0. + void emitFullZASaveRestore(MachineBasicBlock &MBB, + MachineBasicBlock::iterator MBBI, + LiveRegs PhysLiveRegs, bool IsSave); + void emitAllocateFullZASaveBuffer(MachineBasicBlock &MBB, + MachineBasicBlock::iterator MBBI, + LiveRegs PhysLiveRegs); + void emitStateChange(MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI, ZAState From, ZAState To, LiveRegs PhysLiveRegs); + // Helpers for switching between lazy/full ZA save/restore routines. + void emitZASave(MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI, + LiveRegs PhysLiveRegs) { + if (AFI->getSMEFnAttrs().hasAgnosticZAInterface()) + return emitFullZASaveRestore(MBB, MBBI, PhysLiveRegs, /*IsSave=*/true); + return emitSetupLazySave(MBB, MBBI); + } + void emitZARestore(MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI, + LiveRegs PhysLiveRegs) { + if (AFI->getSMEFnAttrs().hasAgnosticZAInterface()) + return emitFullZASaveRestore(MBB, MBBI, PhysLiveRegs, /*IsSave=*/false); + return emitRestoreLazySave(MBB, MBBI, PhysLiveRegs); + } + void emitAllocateZASaveBuffer(MachineBasicBlock &MBB, + MachineBasicBlock::iterator MBBI, + LiveRegs PhysLiveRegs) { + if (AFI->getSMEFnAttrs().hasAgnosticZAInterface()) + return emitAllocateFullZASaveBuffer(MBB, MBBI, PhysLiveRegs); + return emitAllocateLazySaveBuffer(MBB, MBBI); + } + /// Save live physical registers to virtual registers. PhysRegSave createPhysRegSave(LiveRegs PhysLiveRegs, MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI, DebugLoc DL); @@ -228,6 +263,8 @@ struct MachineSMEABI : public MachineFunctionPass { /// Get or create a TPIDR2 block in this function. TPIDR2State getTPIDR2Block(); + Register getAgnosticZABufferPtr(); + private: /// Contains the needed ZA state (and live registers) at an instruction. struct InstInfo { @@ -241,6 +278,7 @@ private: struct BlockInfo { ZAState FixedEntryState{ZAState::ANY}; SmallVector<InstInfo> Insts; + LiveRegs PhysLiveRegsAtEntry = LiveRegs::None; LiveRegs PhysLiveRegsAtExit = LiveRegs::None; }; @@ -249,24 +287,29 @@ private: SmallVector<BlockInfo> Blocks; SmallVector<ZAState> BundleStates; std::optional<TPIDR2State> TPIDR2Block; + std::optional<MachineBasicBlock::iterator> AfterSMEProloguePt; + Register AgnosticZABufferPtr = AArch64::NoRegister; + LiveRegs PhysLiveRegsAfterSMEPrologue = LiveRegs::None; } State; MachineFunction *MF = nullptr; EdgeBundles *Bundles = nullptr; const AArch64Subtarget *Subtarget = nullptr; const AArch64RegisterInfo *TRI = nullptr; + const AArch64FunctionInfo *AFI = nullptr; const TargetInstrInfo *TII = nullptr; MachineRegisterInfo *MRI = nullptr; }; void MachineSMEABI::collectNeededZAStates(SMEAttrs SMEFnAttrs) { - assert((SMEFnAttrs.hasZT0State() || SMEFnAttrs.hasZAState()) && + assert((SMEFnAttrs.hasAgnosticZAInterface() || SMEFnAttrs.hasZT0State() || + SMEFnAttrs.hasZAState()) && "Expected function to have ZA/ZT0 state!"); State.Blocks.resize(MF->getNumBlockIDs()); for (MachineBasicBlock &MBB : *MF) { BlockInfo &Block = State.Blocks[MBB.getNumber()]; - if (&MBB == &MF->front()) { + if (MBB.isEntryBlock()) { // Entry block: Block.FixedEntryState = SMEFnAttrs.hasPrivateZAInterface() ? ZAState::CALLER_DORMANT @@ -294,10 +337,20 @@ void MachineSMEABI::collectNeededZAStates(SMEAttrs SMEFnAttrs) { Block.PhysLiveRegsAtExit = GetPhysLiveRegs(); auto FirstTerminatorInsertPt = MBB.getFirstTerminator(); + auto FirstNonPhiInsertPt = MBB.getFirstNonPHI(); for (MachineInstr &MI : reverse(MBB)) { MachineBasicBlock::iterator MBBI(MI); LiveUnits.stepBackward(MI); LiveRegs PhysLiveRegs = GetPhysLiveRegs(); + // The SMEStateAllocPseudo marker is added to a function if the save + // buffer was allocated in SelectionDAG. It marks the end of the + // allocation -- which is a safe point for this pass to insert any TPIDR2 + // block setup. + if (MI.getOpcode() == AArch64::SMEStateAllocPseudo) { + State.AfterSMEProloguePt = MBBI; + State.PhysLiveRegsAfterSMEPrologue = PhysLiveRegs; + } + // Note: We treat Agnostic ZA as inout_za with an alternate save/restore. auto [NeededState, InsertPt] = getZAStateBeforeInst( *TRI, MI, /*ZAOffAtReturn=*/SMEFnAttrs.hasPrivateZAInterface()); assert((InsertPt == MBBI || @@ -306,6 +359,8 @@ void MachineSMEABI::collectNeededZAStates(SMEAttrs SMEFnAttrs) { // TODO: Do something to avoid state changes where NZCV is live. if (MBBI == FirstTerminatorInsertPt) Block.PhysLiveRegsAtExit = PhysLiveRegs; + if (MBBI == FirstNonPhiInsertPt) + Block.PhysLiveRegsAtEntry = PhysLiveRegs; if (NeededState != ZAState::ANY) Block.Insts.push_back({NeededState, InsertPt, PhysLiveRegs}); } @@ -529,23 +584,25 @@ void MachineSMEABI::emitZAOff(MachineBasicBlock &MBB, void MachineSMEABI::emitAllocateLazySaveBuffer( MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI) { MachineFrameInfo &MFI = MF->getFrameInfo(); - DebugLoc DL = getDebugLoc(MBB, MBBI); Register SP = MRI->createVirtualRegister(&AArch64::GPR64RegClass); Register SVL = MRI->createVirtualRegister(&AArch64::GPR64RegClass); - Register Buffer = MRI->createVirtualRegister(&AArch64::GPR64RegClass); + Register Buffer = AFI->getEarlyAllocSMESaveBuffer(); // Calculate SVL. BuildMI(MBB, MBBI, DL, TII->get(AArch64::RDSVLI_XI), SVL).addImm(1); // 1. Allocate the lazy save buffer. - { - // TODO This function grows the stack with a subtraction, which doesn't work - // on Windows. Some refactoring to share the functionality in - // LowerWindowsDYNAMIC_STACKALLOC will be required once the Windows ABI - // supports SME + if (Buffer == AArch64::NoRegister) { + // TODO: On Windows, we allocate the lazy save buffer in SelectionDAG (so + // Buffer != AArch64::NoRegister). This is done to reuse the existing + // expansions (which can insert stack checks). This works, but it means we + // will always allocate the lazy save buffer (even if the function contains + // no lazy saves). If we want to handle Windows here, we'll need to + // implement something similar to LowerWindowsDYNAMIC_STACKALLOC. assert(!Subtarget->isTargetWindows() && "Lazy ZA save is not yet supported on Windows"); + Buffer = MRI->createVirtualRegister(&AArch64::GPR64RegClass); // Get original stack pointer. BuildMI(MBB, MBBI, DL, TII->get(TargetOpcode::COPY), SP) .addReg(AArch64::SP); @@ -590,8 +647,7 @@ void MachineSMEABI::emitNewZAPrologue(MachineBasicBlock &MBB, .addImm(AArch64SysReg::TPIDR2_EL0); // If TPIDR2_EL0 is non-zero, commit the lazy save. // NOTE: Functions that only use ZT0 don't need to zero ZA. - bool ZeroZA = - MF->getInfo<AArch64FunctionInfo>()->getSMEFnAttrs().hasZAState(); + bool ZeroZA = AFI->getSMEFnAttrs().hasZAState(); auto CommitZASave = BuildMI(MBB, MBBI, DL, TII->get(AArch64::CommitZASavePseudo)) .addReg(TPIDR2EL0) @@ -606,6 +662,86 @@ void MachineSMEABI::emitNewZAPrologue(MachineBasicBlock &MBB, .addImm(1); } +Register MachineSMEABI::getAgnosticZABufferPtr() { + if (State.AgnosticZABufferPtr != AArch64::NoRegister) + return State.AgnosticZABufferPtr; + Register BufferPtr = AFI->getEarlyAllocSMESaveBuffer(); + State.AgnosticZABufferPtr = + BufferPtr != AArch64::NoRegister + ? BufferPtr + : MF->getRegInfo().createVirtualRegister(&AArch64::GPR64RegClass); + return State.AgnosticZABufferPtr; +} + +void MachineSMEABI::emitFullZASaveRestore(MachineBasicBlock &MBB, + MachineBasicBlock::iterator MBBI, + LiveRegs PhysLiveRegs, bool IsSave) { + auto *TLI = Subtarget->getTargetLowering(); + DebugLoc DL = getDebugLoc(MBB, MBBI); + Register BufferPtr = AArch64::X0; + + PhysRegSave RegSave = createPhysRegSave(PhysLiveRegs, MBB, MBBI, DL); + + // Copy the buffer pointer into X0. + BuildMI(MBB, MBBI, DL, TII->get(TargetOpcode::COPY), BufferPtr) + .addReg(getAgnosticZABufferPtr()); + + // Call __arm_sme_save/__arm_sme_restore. + BuildMI(MBB, MBBI, DL, TII->get(AArch64::BL)) + .addReg(BufferPtr, RegState::Implicit) + .addExternalSymbol(TLI->getLibcallName( + IsSave ? RTLIB::SMEABI_SME_SAVE : RTLIB::SMEABI_SME_RESTORE)) + .addRegMask(TRI->getCallPreservedMask( + *MF, + CallingConv::AArch64_SME_ABI_Support_Routines_PreserveMost_From_X1)); + + restorePhyRegSave(RegSave, MBB, MBBI, DL); +} + +void MachineSMEABI::emitAllocateFullZASaveBuffer( + MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI, + LiveRegs PhysLiveRegs) { + // Buffer already allocated in SelectionDAG. + if (AFI->getEarlyAllocSMESaveBuffer()) + return; + + DebugLoc DL = getDebugLoc(MBB, MBBI); + Register BufferPtr = getAgnosticZABufferPtr(); + Register BufferSize = MRI->createVirtualRegister(&AArch64::GPR64RegClass); + + PhysRegSave RegSave = createPhysRegSave(PhysLiveRegs, MBB, MBBI, DL); + + // Calculate the SME state size. + { + auto *TLI = Subtarget->getTargetLowering(); + const AArch64RegisterInfo *TRI = Subtarget->getRegisterInfo(); + BuildMI(MBB, MBBI, DL, TII->get(AArch64::BL)) + .addExternalSymbol(TLI->getLibcallName(RTLIB::SMEABI_SME_STATE_SIZE)) + .addReg(AArch64::X0, RegState::ImplicitDefine) + .addRegMask(TRI->getCallPreservedMask( + *MF, CallingConv:: + AArch64_SME_ABI_Support_Routines_PreserveMost_From_X1)); + BuildMI(MBB, MBBI, DL, TII->get(TargetOpcode::COPY), BufferSize) + .addReg(AArch64::X0); + } + + // Allocate a buffer object of the size given __arm_sme_state_size. + { + MachineFrameInfo &MFI = MF->getFrameInfo(); + BuildMI(MBB, MBBI, DL, TII->get(AArch64::SUBXrx64), AArch64::SP) + .addReg(AArch64::SP) + .addReg(BufferSize) + .addImm(AArch64_AM::getArithExtendImm(AArch64_AM::UXTX, 0)); + BuildMI(MBB, MBBI, DL, TII->get(TargetOpcode::COPY), BufferPtr) + .addReg(AArch64::SP); + + // We have just allocated a variable sized object, tell this to PEI. + MFI.CreateVariableSizedObject(Align(16), nullptr); + } + + restorePhyRegSave(RegSave, MBB, MBBI, DL); +} + void MachineSMEABI::emitStateChange(MachineBasicBlock &MBB, MachineBasicBlock::iterator InsertPt, ZAState From, ZAState To, @@ -623,10 +759,7 @@ void MachineSMEABI::emitStateChange(MachineBasicBlock &MBB, // TODO: Avoid setting up the save buffer if there's no transition to // LOCAL_SAVED. if (From == ZAState::CALLER_DORMANT) { - assert(MBB.getParent() - ->getInfo<AArch64FunctionInfo>() - ->getSMEFnAttrs() - .hasPrivateZAInterface() && + assert(AFI->getSMEFnAttrs().hasPrivateZAInterface() && "CALLER_DORMANT state requires private ZA interface"); assert(&MBB == &MBB.getParent()->front() && "CALLER_DORMANT state only valid in entry block"); @@ -641,12 +774,14 @@ void MachineSMEABI::emitStateChange(MachineBasicBlock &MBB, } if (From == ZAState::ACTIVE && To == ZAState::LOCAL_SAVED) - emitSetupLazySave(MBB, InsertPt); + emitZASave(MBB, InsertPt, PhysLiveRegs); else if (From == ZAState::LOCAL_SAVED && To == ZAState::ACTIVE) - emitRestoreLazySave(MBB, InsertPt, PhysLiveRegs); + emitZARestore(MBB, InsertPt, PhysLiveRegs); else if (To == ZAState::OFF) { assert(From != ZAState::CALLER_DORMANT && "CALLER_DORMANT to OFF should have already been handled"); + assert(!AFI->getSMEFnAttrs().hasAgnosticZAInterface() && + "Should not turn ZA off in agnostic ZA function"); emitZAOff(MBB, InsertPt, /*ClearTPIDR2=*/From == ZAState::LOCAL_SAVED); } else { dbgs() << "Error: Transition from " << getZAStateString(From) << " to " @@ -664,9 +799,10 @@ bool MachineSMEABI::runOnMachineFunction(MachineFunction &MF) { if (!MF.getSubtarget<AArch64Subtarget>().hasSME()) return false; - auto *AFI = MF.getInfo<AArch64FunctionInfo>(); + AFI = MF.getInfo<AArch64FunctionInfo>(); SMEAttrs SMEFnAttrs = AFI->getSMEFnAttrs(); - if (!SMEFnAttrs.hasZAState() && !SMEFnAttrs.hasZT0State()) + if (!SMEFnAttrs.hasZAState() && !SMEFnAttrs.hasZT0State() && + !SMEFnAttrs.hasAgnosticZAInterface()) return false; assert(MF.getRegInfo().isSSA() && "Expected to be run on SSA form!"); @@ -685,9 +821,19 @@ bool MachineSMEABI::runOnMachineFunction(MachineFunction &MF) { insertStateChanges(); // Allocate save buffer (if needed). - if (State.TPIDR2Block) { - MachineBasicBlock &EntryBlock = MF.front(); - emitAllocateLazySaveBuffer(EntryBlock, EntryBlock.getFirstNonPHI()); + if (State.AgnosticZABufferPtr != AArch64::NoRegister || State.TPIDR2Block) { + if (State.AfterSMEProloguePt) { + // Note: With inline stack probes the AfterSMEProloguePt may not be in the + // entry block (due to the probing loop). + emitAllocateZASaveBuffer(*(*State.AfterSMEProloguePt)->getParent(), + *State.AfterSMEProloguePt, + State.PhysLiveRegsAfterSMEPrologue); + } else { + MachineBasicBlock &EntryBlock = MF.front(); + emitAllocateZASaveBuffer( + EntryBlock, EntryBlock.getFirstNonPHI(), + State.Blocks[EntryBlock.getNumber()].PhysLiveRegsAtEntry); + } } return true; diff --git a/llvm/lib/Target/AArch64/SMEABIPass.cpp b/llvm/lib/Target/AArch64/SMEABIPass.cpp index 2008516885c3..79ceb2ababc7 100644 --- a/llvm/lib/Target/AArch64/SMEABIPass.cpp +++ b/llvm/lib/Target/AArch64/SMEABIPass.cpp @@ -50,8 +50,7 @@ private: char SMEABI::ID = 0; static const char *name = "SME ABI Pass"; -INITIALIZE_PASS_BEGIN(SMEABI, DEBUG_TYPE, name, false, false) -INITIALIZE_PASS_END(SMEABI, DEBUG_TYPE, name, false, false) +INITIALIZE_PASS(SMEABI, DEBUG_TYPE, name, false, false) FunctionPass *llvm::createSMEABIPass() { return new SMEABI(); } diff --git a/llvm/lib/Target/AArch64/SMEInstrFormats.td b/llvm/lib/Target/AArch64/SMEInstrFormats.td index b3005d512022..40ec371fe79d 100644 --- a/llvm/lib/Target/AArch64/SMEInstrFormats.td +++ b/llvm/lib/Target/AArch64/SMEInstrFormats.td @@ -1108,6 +1108,10 @@ class sme_spill_fill_base<bit isStore, dag outs, dag ins, string opcodestr> : I<outs, ins, opcodestr, "\t$ZAt[$Rv, $imm4], [$Rn, $offset, mul vl]", "", []>, Sched<[]> { + // 'offset' operand is encoded in the same bits as 'imm4'. There is currently + // no way to tell TableGen about this. + let DecoderMethod = "DecodeSMESpillFillInstruction"; + bits<0> ZAt; bits<2> Rv; bits<5> Rn; bits<4> imm4; diff --git a/llvm/lib/Target/AArch64/SVEInstrFormats.td b/llvm/lib/Target/AArch64/SVEInstrFormats.td index a3a7d0f74e1b..f8c1fe81c678 100644 --- a/llvm/lib/Target/AArch64/SVEInstrFormats.td +++ b/llvm/lib/Target/AArch64/SVEInstrFormats.td @@ -315,10 +315,16 @@ def addsub_imm8_opt_lsl_i16 : imm8_opt_lsl<16, "uint16_t", SVEAddSubImmOperand16 def addsub_imm8_opt_lsl_i32 : imm8_opt_lsl<32, "uint32_t", SVEAddSubImmOperand32>; def addsub_imm8_opt_lsl_i64 : imm8_opt_lsl<64, "uint64_t", SVEAddSubImmOperand64>; -def SVEAddSubImm8Pat : ComplexPattern<i32, 2, "SelectSVEAddSubImm<MVT::i8>", []>; -def SVEAddSubImm16Pat : ComplexPattern<i32, 2, "SelectSVEAddSubImm<MVT::i16>", []>; -def SVEAddSubImm32Pat : ComplexPattern<i32, 2, "SelectSVEAddSubImm<MVT::i32>", []>; -def SVEAddSubImm64Pat : ComplexPattern<i64, 2, "SelectSVEAddSubImm<MVT::i64>", []>; +let Complexity = 1 in { +def SVEAddSubImm8Pat : ComplexPattern<i32, 2, "SelectSVEAddSubImm<MVT::i8, false>", []>; +def SVEAddSubImm16Pat : ComplexPattern<i32, 2, "SelectSVEAddSubImm<MVT::i16, false>", []>; +def SVEAddSubImm32Pat : ComplexPattern<i32, 2, "SelectSVEAddSubImm<MVT::i32, false>", []>; +def SVEAddSubImm64Pat : ComplexPattern<i64, 2, "SelectSVEAddSubImm<MVT::i64, false>", []>; + +def SVEAddSubNegImm8Pat : ComplexPattern<i32, 2, "SelectSVEAddSubImm<MVT::i8, true>", []>; +def SVEAddSubNegImm16Pat : ComplexPattern<i32, 2, "SelectSVEAddSubImm<MVT::i16, true>", []>; +def SVEAddSubNegImm32Pat : ComplexPattern<i32, 2, "SelectSVEAddSubImm<MVT::i32, true>", []>; +def SVEAddSubNegImm64Pat : ComplexPattern<i64, 2, "SelectSVEAddSubImm<MVT::i64, true>", []>; def SVEAddSubSSatNegImm8Pat : ComplexPattern<i32, 2, "SelectSVEAddSubSSatImm<MVT::i8, true>", []>; def SVEAddSubSSatNegImm16Pat : ComplexPattern<i32, 2, "SelectSVEAddSubSSatImm<MVT::i16, true>", []>; @@ -329,6 +335,7 @@ def SVEAddSubSSatPosImm8Pat : ComplexPattern<i32, 2, "SelectSVEAddSubSSatImm<MV def SVEAddSubSSatPosImm16Pat : ComplexPattern<i32, 2, "SelectSVEAddSubSSatImm<MVT::i16, false>", []>; def SVEAddSubSSatPosImm32Pat : ComplexPattern<i32, 2, "SelectSVEAddSubSSatImm<MVT::i32, false>", []>; def SVEAddSubSSatPosImm64Pat : ComplexPattern<i64, 2, "SelectSVEAddSubSSatImm<MVT::i64, false>", []>; +} // Complexity = 1 def SVECpyDupImm8Pat : ComplexPattern<i32, 2, "SelectSVECpyDupImm<MVT::i8>", []>; def SVECpyDupImm16Pat : ComplexPattern<i32, 2, "SelectSVECpyDupImm<MVT::i16>", []>; @@ -886,13 +893,17 @@ class sve_int_ptest<bits<6> opc, string asm, SDPatternOperator op> } multiclass sve_int_ptest<bits<6> opc, string asm, SDPatternOperator op, - SDPatternOperator op_any> { + SDPatternOperator op_any, SDPatternOperator op_first> { def NAME : sve_int_ptest<opc, asm, op>; let hasNoSchedulingInfo = 1, isCompare = 1, Defs = [NZCV] in { def _ANY : Pseudo<(outs), (ins PPRAny:$Pg, PPR8:$Pn), [(set NZCV, (op_any (nxv16i1 PPRAny:$Pg), (nxv16i1 PPR8:$Pn)))]>, PseudoInstExpansion<(!cast<Instruction>(NAME) PPRAny:$Pg, PPR8:$Pn)>; + + def _FIRST : Pseudo<(outs), (ins PPRAny:$Pg, PPR8:$Pn), + [(set NZCV, (op_first (nxv16i1 PPRAny:$Pg), (nxv16i1 PPR8:$Pn)))]>, + PseudoInstExpansion<(!cast<Instruction>(NAME) PPRAny:$Pg, PPR8:$Pn)>; } } @@ -5154,11 +5165,14 @@ multiclass sve_int_dup_imm<string asm> { (!cast<Instruction>(NAME # _D) ZPR64:$Zd, cpy_imm8_opt_lsl_i64:$imm), 1>; def : InstAlias<"fmov $Zd, #0.0", - (!cast<Instruction>(NAME # _H) ZPR16:$Zd, 0, 0), 1>; + (!cast<Instruction>(NAME # _H) ZPR16:$Zd, + (cpy_imm8_opt_lsl_i16 0, 0)), 1>; def : InstAlias<"fmov $Zd, #0.0", - (!cast<Instruction>(NAME # _S) ZPR32:$Zd, 0, 0), 1>; + (!cast<Instruction>(NAME # _S) ZPR32:$Zd, + (cpy_imm8_opt_lsl_i32 0, 0)), 1>; def : InstAlias<"fmov $Zd, #0.0", - (!cast<Instruction>(NAME # _D) ZPR64:$Zd, 0, 0), 1>; + (!cast<Instruction>(NAME # _D) ZPR64:$Zd, + (cpy_imm8_opt_lsl_i64 0, 0)), 1>; } class sve_int_dup_fpimm<bits<2> sz8_64, Operand fpimmtype, @@ -5218,7 +5232,8 @@ class sve_int_arith_imm0<bits<2> sz8_64, bits<3> opc, string asm, let hasSideEffects = 0; } -multiclass sve_int_arith_imm0<bits<3> opc, string asm, SDPatternOperator op> { +multiclass sve_int_arith_imm0<bits<3> opc, string asm, SDPatternOperator op, + SDPatternOperator inv_op = null_frag> { def _B : sve_int_arith_imm0<0b00, opc, asm, ZPR8, addsub_imm8_opt_lsl_i8>; def _H : sve_int_arith_imm0<0b01, opc, asm, ZPR16, addsub_imm8_opt_lsl_i16>; def _S : sve_int_arith_imm0<0b10, opc, asm, ZPR32, addsub_imm8_opt_lsl_i32>; @@ -5228,6 +5243,12 @@ multiclass sve_int_arith_imm0<bits<3> opc, string asm, SDPatternOperator op> { def : SVE_1_Op_Imm_OptLsl_Pat<nxv8i16, op, ZPR16, i32, SVEAddSubImm16Pat, !cast<Instruction>(NAME # _H)>; def : SVE_1_Op_Imm_OptLsl_Pat<nxv4i32, op, ZPR32, i32, SVEAddSubImm32Pat, !cast<Instruction>(NAME # _S)>; def : SVE_1_Op_Imm_OptLsl_Pat<nxv2i64, op, ZPR64, i64, SVEAddSubImm64Pat, !cast<Instruction>(NAME # _D)>; + + // Extra patterns for add(x, splat(-ve)) -> sub(x, +ve). There is no i8 + // pattern as all i8 constants can be handled by an add. + def : SVE_1_Op_Imm_OptLsl_Pat<nxv8i16, inv_op, ZPR16, i32, SVEAddSubNegImm16Pat, !cast<Instruction>(NAME # _H)>; + def : SVE_1_Op_Imm_OptLsl_Pat<nxv4i32, inv_op, ZPR32, i32, SVEAddSubNegImm32Pat, !cast<Instruction>(NAME # _S)>; + def : SVE_1_Op_Imm_OptLsl_Pat<nxv2i64, inv_op, ZPR64, i64, SVEAddSubNegImm64Pat, !cast<Instruction>(NAME # _D)>; } multiclass sve_int_arith_imm0_ssat<bits<3> opc, string asm, SDPatternOperator op, @@ -5549,11 +5570,14 @@ multiclass sve_int_dup_imm_pred_merge<string asm, SDPatternOperator op> { nxv2i64, nxv2i1, i64, SVECpyDupImm64Pat>; def : InstAlias<"fmov $Zd, $Pg/m, #0.0", - (!cast<Instruction>(NAME # _H) ZPR16:$Zd, PPRAny:$Pg, 0, 0), 0>; + (!cast<Instruction>(NAME # _H) ZPR16:$Zd, PPRAny:$Pg, + (cpy_imm8_opt_lsl_i16 0, 0)), 0>; def : InstAlias<"fmov $Zd, $Pg/m, #0.0", - (!cast<Instruction>(NAME # _S) ZPR32:$Zd, PPRAny:$Pg, 0, 0), 0>; + (!cast<Instruction>(NAME # _S) ZPR32:$Zd, PPRAny:$Pg, + (cpy_imm8_opt_lsl_i32 0, 0)), 0>; def : InstAlias<"fmov $Zd, $Pg/m, #0.0", - (!cast<Instruction>(NAME # _D) ZPR64:$Zd, PPRAny:$Pg, 0, 0), 0>; + (!cast<Instruction>(NAME # _D) ZPR64:$Zd, PPRAny:$Pg, + (cpy_imm8_opt_lsl_i64 0, 0)), 0>; def : Pat<(vselect PPRAny:$Pg, (SVEDup0), (nxv8f16 ZPR:$Zd)), (!cast<Instruction>(NAME # _H) $Zd, $Pg, 0, 0)>; @@ -5946,16 +5970,20 @@ class sve2_int_while_rr<bits<2> sz8_64, bits<1> rw, string asm, let isWhile = 1; } -multiclass sve2_int_while_rr<bits<1> rw, string asm, string op> { +multiclass sve2_int_while_rr<bits<1> rw, string asm, SDPatternOperator op> { def _B : sve2_int_while_rr<0b00, rw, asm, PPR8>; def _H : sve2_int_while_rr<0b01, rw, asm, PPR16>; def _S : sve2_int_while_rr<0b10, rw, asm, PPR32>; def _D : sve2_int_while_rr<0b11, rw, asm, PPR64>; - def : SVE_2_Op_Pat<nxv16i1, !cast<SDPatternOperator>(op # _b), i64, i64, !cast<Instruction>(NAME # _B)>; - def : SVE_2_Op_Pat<nxv8i1, !cast<SDPatternOperator>(op # _h), i64, i64, !cast<Instruction>(NAME # _H)>; - def : SVE_2_Op_Pat<nxv4i1, !cast<SDPatternOperator>(op # _s), i64, i64, !cast<Instruction>(NAME # _S)>; - def : SVE_2_Op_Pat<nxv2i1, !cast<SDPatternOperator>(op # _d), i64, i64, !cast<Instruction>(NAME # _D)>; + def : Pat<(nxv16i1 (op i64:$Op1, i64:$Op2, (i64 1))), + (!cast<Instruction>(NAME # _B) $Op1, $Op2)>; + def : Pat<(nxv8i1 (op i64:$Op1, i64:$Op2, (i64 2))), + (!cast<Instruction>(NAME # _H) $Op1, $Op2)>; + def : Pat<(nxv4i1 (op i64:$Op1, i64:$Op2, (i64 4))), + (!cast<Instruction>(NAME # _S) $Op1, $Op2)>; + def : Pat<(nxv2i1 (op i64:$Op1, i64:$Op2, (i64 8))), + (!cast<Instruction>(NAME # _D) $Op1, $Op2)>; } //===----------------------------------------------------------------------===// |