diff options
| author | Michael Kruse <llvm-project@meinersbur.de> | 2025-01-03 10:22:51 +0100 |
|---|---|---|
| committer | Michael Kruse <llvm-project@meinersbur.de> | 2025-01-03 10:22:51 +0100 |
| commit | 38500d63e14ce340236840f60d356cdefb56a52c (patch) | |
| tree | 17edbec446ce9b50d2f215a483b83afb293a635d /llvm/lib/Target/X86 | |
| parent | 1a3d5daaef7a6a63448a497da3eff7fc9e23df26 (diff) | |
| parent | 27f30029741ecf023baece7b3dde1ff9011ffefc (diff) | |
Merge branch 'main' into users/meinersbur/flang_runtime_split-headersusers/meinersbur/flang_runtime_split-headers
Diffstat (limited to 'llvm/lib/Target/X86')
35 files changed, 886 insertions, 712 deletions
diff --git a/llvm/lib/Target/X86/MCTargetDesc/X86ATTInstPrinter.cpp b/llvm/lib/Target/X86/MCTargetDesc/X86ATTInstPrinter.cpp index b67c573e217b..abe0cc6365dd 100644 --- a/llvm/lib/Target/X86/MCTargetDesc/X86ATTInstPrinter.cpp +++ b/llvm/lib/Target/X86/MCTargetDesc/X86ATTInstPrinter.cpp @@ -140,8 +140,8 @@ bool X86ATTInstPrinter::printVecCompareInstr(const MCInst *MI, case X86::VCMPPSZ128rmik: case X86::VCMPPSZ128rrik: case X86::VCMPPSZ256rmik: case X86::VCMPPSZ256rrik: case X86::VCMPPSZrmik: case X86::VCMPPSZrrik: - case X86::VCMPSDZrmi_Intk: case X86::VCMPSDZrri_Intk: - case X86::VCMPSSZrmi_Intk: case X86::VCMPSSZrri_Intk: + case X86::VCMPSDZrmik_Int: case X86::VCMPSDZrrik_Int: + case X86::VCMPSSZrmik_Int: case X86::VCMPSSZrrik_Int: case X86::VCMPPDZ128rmbi: case X86::VCMPPDZ128rmbik: case X86::VCMPPDZ256rmbi: case X86::VCMPPDZ256rmbik: case X86::VCMPPDZrmbi: case X86::VCMPPDZrmbik: @@ -150,8 +150,8 @@ bool X86ATTInstPrinter::printVecCompareInstr(const MCInst *MI, case X86::VCMPPSZrmbi: case X86::VCMPPSZrmbik: case X86::VCMPPDZrrib: case X86::VCMPPDZrribk: case X86::VCMPPSZrrib: case X86::VCMPPSZrribk: - case X86::VCMPSDZrrib_Int: case X86::VCMPSDZrrib_Intk: - case X86::VCMPSSZrrib_Int: case X86::VCMPSSZrrib_Intk: + case X86::VCMPSDZrrib_Int: case X86::VCMPSDZrribk_Int: + case X86::VCMPSSZrrib_Int: case X86::VCMPSSZrribk_Int: case X86::VCMPPHZ128rmi: case X86::VCMPPHZ128rri: case X86::VCMPPHZ256rmi: case X86::VCMPPHZ256rri: case X86::VCMPPHZrmi: case X86::VCMPPHZrri: @@ -160,12 +160,12 @@ bool X86ATTInstPrinter::printVecCompareInstr(const MCInst *MI, case X86::VCMPPHZ128rmik: case X86::VCMPPHZ128rrik: case X86::VCMPPHZ256rmik: case X86::VCMPPHZ256rrik: case X86::VCMPPHZrmik: case X86::VCMPPHZrrik: - case X86::VCMPSHZrmi_Intk: case X86::VCMPSHZrri_Intk: + case X86::VCMPSHZrmik_Int: case X86::VCMPSHZrrik_Int: case X86::VCMPPHZ128rmbi: case X86::VCMPPHZ128rmbik: case X86::VCMPPHZ256rmbi: case X86::VCMPPHZ256rmbik: case X86::VCMPPHZrmbi: case X86::VCMPPHZrmbik: case X86::VCMPPHZrrib: case X86::VCMPPHZrribk: - case X86::VCMPSHZrrib_Int: case X86::VCMPSHZrrib_Intk: + case X86::VCMPSHZrrib_Int: case X86::VCMPSHZrribk_Int: case X86::VCMPPBF16Z128rmi: case X86::VCMPPBF16Z128rri: case X86::VCMPPBF16Z256rmi: case X86::VCMPPBF16Z256rri: case X86::VCMPPBF16Zrmi: case X86::VCMPPBF16Zrri: diff --git a/llvm/lib/Target/X86/MCTargetDesc/X86AsmBackend.cpp b/llvm/lib/Target/X86/MCTargetDesc/X86AsmBackend.cpp index 01b6c84419fc..144a0c99fdf4 100644 --- a/llvm/lib/Target/X86/MCTargetDesc/X86AsmBackend.cpp +++ b/llvm/lib/Target/X86/MCTargetDesc/X86AsmBackend.cpp @@ -634,7 +634,7 @@ const MCFixupKindInfo &X86AsmBackend::getFixupKindInfo(MCFixupKind Kind) const { {"reloc_riprel_4byte_relax", 0, 32, MCFixupKindInfo::FKF_IsPCRel}, {"reloc_riprel_4byte_relax_rex", 0, 32, MCFixupKindInfo::FKF_IsPCRel}, {"reloc_riprel_4byte_relax_rex2", 0, 32, MCFixupKindInfo::FKF_IsPCRel}, - {"reloc_riprel_6byte_relax", 0, 32, MCFixupKindInfo::FKF_IsPCRel}, + {"reloc_riprel_4byte_relax_evex", 0, 32, MCFixupKindInfo::FKF_IsPCRel}, {"reloc_signed_4byte", 0, 32, 0}, {"reloc_signed_4byte_relax", 0, 32, 0}, {"reloc_global_offset_table", 0, 32, 0}, @@ -684,7 +684,7 @@ static unsigned getFixupKindSize(unsigned Kind) { case X86::reloc_riprel_4byte_relax_rex2: case X86::reloc_riprel_4byte_movq_load: case X86::reloc_riprel_4byte_movq_load_rex2: - case X86::reloc_riprel_6byte_relax: + case X86::reloc_riprel_4byte_relax_evex: case X86::reloc_signed_4byte: case X86::reloc_signed_4byte_relax: case X86::reloc_global_offset_table: @@ -811,7 +811,7 @@ bool X86AsmBackend::padInstructionViaPrefix(MCRelaxableFragment &RF, SmallString<256> Code; Code.append(PrefixBytesToAdd, Prefix); Code.append(RF.getContents().begin(), RF.getContents().end()); - RF.getContents() = Code; + RF.setContents(Code); // Adjust the fixups for the change in offsets for (auto &F : RF.getFixups()) { @@ -843,7 +843,7 @@ bool X86AsmBackend::padInstructionViaRelaxation(MCRelaxableFragment &RF, if (Delta > RemainingSize) return false; RF.setInst(Relaxed); - RF.getContents() = Code; + RF.setContents(Code); RF.getFixups() = Fixups; RemainingSize -= Delta; return true; diff --git a/llvm/lib/Target/X86/MCTargetDesc/X86ELFObjectWriter.cpp b/llvm/lib/Target/X86/MCTargetDesc/X86ELFObjectWriter.cpp index 29a1af97d24f..cc712055a861 100644 --- a/llvm/lib/Target/X86/MCTargetDesc/X86ELFObjectWriter.cpp +++ b/llvm/lib/Target/X86/MCTargetDesc/X86ELFObjectWriter.cpp @@ -77,7 +77,7 @@ static X86_64RelType getType64(MCFixupKind Kind, case X86::reloc_riprel_4byte_relax_rex2: case X86::reloc_riprel_4byte_movq_load: case X86::reloc_riprel_4byte_movq_load_rex2: - case X86::reloc_riprel_6byte_relax: + case X86::reloc_riprel_4byte_relax_evex: return RT64_32; case X86::reloc_branch_4byte_pcrel: Modifier = MCSymbolRefExpr::VK_PLT; @@ -203,7 +203,7 @@ static unsigned getRelocType64(MCContext &Ctx, SMLoc Loc, if ((unsigned)Kind == X86::reloc_riprel_4byte_movq_load_rex2 || (unsigned)Kind == X86::reloc_riprel_4byte_relax_rex2) return ELF::R_X86_64_CODE_4_GOTTPOFF; - else if ((unsigned)Kind == X86::reloc_riprel_6byte_relax) + else if ((unsigned)Kind == X86::reloc_riprel_4byte_relax_evex) return ELF::R_X86_64_CODE_6_GOTTPOFF; return ELF::R_X86_64_GOTTPOFF; case MCSymbolRefExpr::VK_TLSLD: @@ -230,8 +230,6 @@ static unsigned getRelocType64(MCContext &Ctx, SMLoc Loc, case X86::reloc_riprel_4byte_relax_rex2: case X86::reloc_riprel_4byte_movq_load_rex2: return ELF::R_X86_64_CODE_4_GOTPCRELX; - case X86::reloc_riprel_6byte_relax: - return ELF::R_X86_64_CODE_6_GOTTPOFF; } llvm_unreachable("unexpected relocation type!"); case MCSymbolRefExpr::VK_GOTPCREL_NORELAX: diff --git a/llvm/lib/Target/X86/MCTargetDesc/X86FixupKinds.h b/llvm/lib/Target/X86/MCTargetDesc/X86FixupKinds.h index 52592a5a13b9..9b951973ab19 100644 --- a/llvm/lib/Target/X86/MCTargetDesc/X86FixupKinds.h +++ b/llvm/lib/Target/X86/MCTargetDesc/X86FixupKinds.h @@ -24,8 +24,9 @@ enum Fixups { // instruction with rex prefix reloc_riprel_4byte_relax_rex2, // 32-bit rip-relative in relaxable // instruction with rex2 prefix - reloc_riprel_6byte_relax, // 32-bit rip-relative in relaxable - // instruction with APX NDD + reloc_riprel_4byte_relax_evex, // 32-bit rip-relative in relaxable + // instruction of APX NDD/NF with + // EVEX prefix reloc_signed_4byte, // 32-bit signed. Unlike FK_Data_4 // this will be sign extended at // runtime. diff --git a/llvm/lib/Target/X86/MCTargetDesc/X86InstComments.cpp b/llvm/lib/Target/X86/MCTargetDesc/X86InstComments.cpp index 9f8bc57fbc76..681d0dab37d0 100644 --- a/llvm/lib/Target/X86/MCTargetDesc/X86InstComments.cpp +++ b/llvm/lib/Target/X86/MCTargetDesc/X86InstComments.cpp @@ -40,6 +40,17 @@ using namespace llvm; CASE_MASK_INS_COMMON(Inst, Suffix, src) \ CASE_MASKZ_INS_COMMON(Inst, Suffix, src) +#define CASE_MASK_INS_COMMON_INT(Inst, Suffix, src) \ + case X86::V##Inst##Suffix##src##k_Int: + +#define CASE_MASKZ_INS_COMMON_INT(Inst, Suffix, src) \ + case X86::V##Inst##Suffix##src##kz_Int: + +#define CASE_AVX512_INS_COMMON_INT(Inst, Suffix, src) \ + CASE_AVX_INS_COMMON(Inst, Suffix, src##_Int) \ + CASE_MASK_INS_COMMON_INT(Inst, Suffix, src) \ + CASE_MASKZ_INS_COMMON_INT(Inst, Suffix, src) + #define CASE_FPCLASS_PACKED(Inst, src) \ CASE_AVX_INS_COMMON(Inst, Z, src##i) \ CASE_AVX_INS_COMMON(Inst, Z256, src##i) \ @@ -196,8 +207,8 @@ using namespace llvm; CASE_AVX_INS_COMMON(Inst##SS, , r_Int) \ CASE_AVX_INS_COMMON(Inst##SD, Z, r) \ CASE_AVX_INS_COMMON(Inst##SS, Z, r) \ - CASE_AVX512_INS_COMMON(Inst##SD, Z, r_Int) \ - CASE_AVX512_INS_COMMON(Inst##SS, Z, r_Int) + CASE_AVX512_INS_COMMON_INT(Inst##SD, Z, r) \ + CASE_AVX512_INS_COMMON_INT(Inst##SS, Z, r) #define CASE_FMA_SCALAR_MEM(Inst) \ CASE_AVX_INS_COMMON(Inst##SD, , m) \ @@ -206,8 +217,8 @@ using namespace llvm; CASE_AVX_INS_COMMON(Inst##SS, , m_Int) \ CASE_AVX_INS_COMMON(Inst##SD, Z, m) \ CASE_AVX_INS_COMMON(Inst##SS, Z, m) \ - CASE_AVX512_INS_COMMON(Inst##SD, Z, m_Int) \ - CASE_AVX512_INS_COMMON(Inst##SS, Z, m_Int) + CASE_AVX512_INS_COMMON_INT(Inst##SD, Z, m) \ + CASE_AVX512_INS_COMMON_INT(Inst##SS, Z, m) #define CASE_FMA4(Inst, suf) \ CASE_AVX_INS_COMMON(Inst, 4, suf) \ diff --git a/llvm/lib/Target/X86/MCTargetDesc/X86InstPrinterCommon.cpp b/llvm/lib/Target/X86/MCTargetDesc/X86InstPrinterCommon.cpp index fafcc737ff98..01e2d4ace977 100644 --- a/llvm/lib/Target/X86/MCTargetDesc/X86InstPrinterCommon.cpp +++ b/llvm/lib/Target/X86/MCTargetDesc/X86InstPrinterCommon.cpp @@ -277,8 +277,8 @@ void X86InstPrinterCommon::printCMPMnemonic(const MCInst *MI, bool IsVCmp, case X86::VCMPSDrmi_Int: case X86::VCMPSDrri_Int: case X86::VCMPSDZrmi: case X86::VCMPSDZrri: case X86::VCMPSDZrmi_Int: case X86::VCMPSDZrri_Int: - case X86::VCMPSDZrmi_Intk: case X86::VCMPSDZrri_Intk: - case X86::VCMPSDZrrib_Int: case X86::VCMPSDZrrib_Intk: + case X86::VCMPSDZrmik_Int: case X86::VCMPSDZrrik_Int: + case X86::VCMPSDZrrib_Int: case X86::VCMPSDZrribk_Int: OS << "sd\t"; break; case X86::CMPSSrmi: case X86::CMPSSrri: @@ -287,8 +287,8 @@ void X86InstPrinterCommon::printCMPMnemonic(const MCInst *MI, bool IsVCmp, case X86::VCMPSSrmi_Int: case X86::VCMPSSrri_Int: case X86::VCMPSSZrmi: case X86::VCMPSSZrri: case X86::VCMPSSZrmi_Int: case X86::VCMPSSZrri_Int: - case X86::VCMPSSZrmi_Intk: case X86::VCMPSSZrri_Intk: - case X86::VCMPSSZrrib_Int: case X86::VCMPSSZrrib_Intk: + case X86::VCMPSSZrmik_Int: case X86::VCMPSSZrrik_Int: + case X86::VCMPSSZrrib_Int: case X86::VCMPSSZrribk_Int: OS << "ss\t"; break; case X86::VCMPPHZ128rmi: case X86::VCMPPHZ128rri: @@ -305,8 +305,8 @@ void X86InstPrinterCommon::printCMPMnemonic(const MCInst *MI, bool IsVCmp, break; case X86::VCMPSHZrmi: case X86::VCMPSHZrri: case X86::VCMPSHZrmi_Int: case X86::VCMPSHZrri_Int: - case X86::VCMPSHZrrib_Int: case X86::VCMPSHZrrib_Intk: - case X86::VCMPSHZrmi_Intk: case X86::VCMPSHZrri_Intk: + case X86::VCMPSHZrrib_Int: case X86::VCMPSHZrribk_Int: + case X86::VCMPSHZrmik_Int: case X86::VCMPSHZrrik_Int: OS << "sh\t"; break; case X86::VCMPPBF16Z128rmi: case X86::VCMPPBF16Z128rri: diff --git a/llvm/lib/Target/X86/MCTargetDesc/X86IntelInstPrinter.cpp b/llvm/lib/Target/X86/MCTargetDesc/X86IntelInstPrinter.cpp index 680092679c90..c26dc2ca5a7a 100644 --- a/llvm/lib/Target/X86/MCTargetDesc/X86IntelInstPrinter.cpp +++ b/llvm/lib/Target/X86/MCTargetDesc/X86IntelInstPrinter.cpp @@ -119,8 +119,8 @@ bool X86IntelInstPrinter::printVecCompareInstr(const MCInst *MI, raw_ostream &OS case X86::VCMPPSZ128rmik: case X86::VCMPPSZ128rrik: case X86::VCMPPSZ256rmik: case X86::VCMPPSZ256rrik: case X86::VCMPPSZrmik: case X86::VCMPPSZrrik: - case X86::VCMPSDZrmi_Intk: case X86::VCMPSDZrri_Intk: - case X86::VCMPSSZrmi_Intk: case X86::VCMPSSZrri_Intk: + case X86::VCMPSDZrmik_Int: case X86::VCMPSDZrrik_Int: + case X86::VCMPSSZrmik_Int: case X86::VCMPSSZrrik_Int: case X86::VCMPPDZ128rmbi: case X86::VCMPPDZ128rmbik: case X86::VCMPPDZ256rmbi: case X86::VCMPPDZ256rmbik: case X86::VCMPPDZrmbi: case X86::VCMPPDZrmbik: @@ -129,8 +129,8 @@ bool X86IntelInstPrinter::printVecCompareInstr(const MCInst *MI, raw_ostream &OS case X86::VCMPPSZrmbi: case X86::VCMPPSZrmbik: case X86::VCMPPDZrrib: case X86::VCMPPDZrribk: case X86::VCMPPSZrrib: case X86::VCMPPSZrribk: - case X86::VCMPSDZrrib_Int: case X86::VCMPSDZrrib_Intk: - case X86::VCMPSSZrrib_Int: case X86::VCMPSSZrrib_Intk: + case X86::VCMPSDZrrib_Int: case X86::VCMPSDZrribk_Int: + case X86::VCMPSSZrrib_Int: case X86::VCMPSSZrribk_Int: case X86::VCMPPHZ128rmi: case X86::VCMPPHZ128rri: case X86::VCMPPHZ256rmi: case X86::VCMPPHZ256rri: case X86::VCMPPHZrmi: case X86::VCMPPHZrri: @@ -139,12 +139,12 @@ bool X86IntelInstPrinter::printVecCompareInstr(const MCInst *MI, raw_ostream &OS case X86::VCMPPHZ128rmik: case X86::VCMPPHZ128rrik: case X86::VCMPPHZ256rmik: case X86::VCMPPHZ256rrik: case X86::VCMPPHZrmik: case X86::VCMPPHZrrik: - case X86::VCMPSHZrmi_Intk: case X86::VCMPSHZrri_Intk: + case X86::VCMPSHZrmik_Int: case X86::VCMPSHZrrik_Int: case X86::VCMPPHZ128rmbi: case X86::VCMPPHZ128rmbik: case X86::VCMPPHZ256rmbi: case X86::VCMPPHZ256rmbik: case X86::VCMPPHZrmbi: case X86::VCMPPHZrmbik: case X86::VCMPPHZrrib: case X86::VCMPPHZrribk: - case X86::VCMPSHZrrib_Int: case X86::VCMPSHZrrib_Intk: + case X86::VCMPSHZrrib_Int: case X86::VCMPSHZrribk_Int: case X86::VCMPPBF16Z128rmi: case X86::VCMPPBF16Z128rri: case X86::VCMPPBF16Z256rmi: case X86::VCMPPBF16Z256rri: case X86::VCMPPBF16Zrmi: case X86::VCMPPBF16Zrri: diff --git a/llvm/lib/Target/X86/MCTargetDesc/X86MCCodeEmitter.cpp b/llvm/lib/Target/X86/MCTargetDesc/X86MCCodeEmitter.cpp index 052d732e4d01..55fb9f54c539 100644 --- a/llvm/lib/Target/X86/MCTargetDesc/X86MCCodeEmitter.cpp +++ b/llvm/lib/Target/X86/MCTargetDesc/X86MCCodeEmitter.cpp @@ -572,15 +572,15 @@ void X86MCCodeEmitter::emitImmediate(const MCOperand &DispOp, SMLoc Loc, FixupKind == MCFixupKind(X86::reloc_riprel_4byte_relax) || FixupKind == MCFixupKind(X86::reloc_riprel_4byte_relax_rex) || FixupKind == MCFixupKind(X86::reloc_riprel_4byte_relax_rex2) || - FixupKind == MCFixupKind(X86::reloc_branch_4byte_pcrel)) { + FixupKind == MCFixupKind(X86::reloc_branch_4byte_pcrel) || + FixupKind == MCFixupKind(X86::reloc_riprel_4byte_relax_evex)) { ImmOffset -= 4; // If this is a pc-relative load off _GLOBAL_OFFSET_TABLE_: // leaq _GLOBAL_OFFSET_TABLE_(%rip), %r15 // this needs to be a GOTPC32 relocation. if (startsWithGlobalOffsetTable(Expr) != GOT_None) FixupKind = MCFixupKind(X86::reloc_global_offset_table); - } else if (FixupKind == MCFixupKind(X86::reloc_riprel_6byte_relax)) - ImmOffset -= 6; + } if (FixupKind == FK_PCRel_2) ImmOffset -= 2; @@ -677,7 +677,7 @@ void X86MCCodeEmitter::emitMemModRMByte( case X86::ADD64mr_ND: case X86::ADD64mr_NF_ND: case X86::ADD64rm_NF_ND: - return X86::reloc_riprel_6byte_relax; + return X86::reloc_riprel_4byte_relax_evex; } }(); diff --git a/llvm/lib/Target/X86/MCTargetDesc/X86MachObjectWriter.cpp b/llvm/lib/Target/X86/MCTargetDesc/X86MachObjectWriter.cpp index 413650e90de6..69eeb09ebf2d 100644 --- a/llvm/lib/Target/X86/MCTargetDesc/X86MachObjectWriter.cpp +++ b/llvm/lib/Target/X86/MCTargetDesc/X86MachObjectWriter.cpp @@ -70,7 +70,7 @@ static bool isFixupKindRIPRel(unsigned Kind) { Kind == X86::reloc_riprel_4byte_relax || Kind == X86::reloc_riprel_4byte_relax_rex || Kind == X86::reloc_riprel_4byte_relax_rex2 || - Kind == X86::reloc_riprel_6byte_relax; + Kind == X86::reloc_riprel_4byte_relax_evex; } static unsigned getFixupKindLog2Size(unsigned Kind) { @@ -92,7 +92,7 @@ static unsigned getFixupKindLog2Size(unsigned Kind) { case X86::reloc_signed_4byte: case X86::reloc_signed_4byte_relax: case X86::reloc_branch_4byte_pcrel: - case X86::reloc_riprel_6byte_relax: + case X86::reloc_riprel_4byte_relax_evex: case FK_Data_4: return 2; case FK_Data_8: return 3; } diff --git a/llvm/lib/Target/X86/MCTargetDesc/X86WinCOFFObjectWriter.cpp b/llvm/lib/Target/X86/MCTargetDesc/X86WinCOFFObjectWriter.cpp index 48d4707bbe1e..f4ac242e9f15 100644 --- a/llvm/lib/Target/X86/MCTargetDesc/X86WinCOFFObjectWriter.cpp +++ b/llvm/lib/Target/X86/MCTargetDesc/X86WinCOFFObjectWriter.cpp @@ -70,7 +70,7 @@ unsigned X86WinCOFFObjectWriter::getRelocType(MCContext &Ctx, case X86::reloc_riprel_4byte_relax: case X86::reloc_riprel_4byte_relax_rex: case X86::reloc_riprel_4byte_relax_rex2: - case X86::reloc_riprel_6byte_relax: + case X86::reloc_riprel_4byte_relax_evex: case X86::reloc_branch_4byte_pcrel: return COFF::IMAGE_REL_AMD64_REL32; case FK_Data_4: diff --git a/llvm/lib/Target/X86/X86.td b/llvm/lib/Target/X86/X86.td index 5722167beca9..38761e1fd7ee 100644 --- a/llvm/lib/Target/X86/X86.td +++ b/llvm/lib/Target/X86/X86.td @@ -1163,6 +1163,7 @@ def ProcessorFeatures { FeatureAVXNECONVERT, FeatureAVXVNNIINT8, FeatureAVXVNNIINT16, + FeatureUSERMSR, FeatureSHA512, FeatureSM3, FeatureEGPR, diff --git a/llvm/lib/Target/X86/X86ArgumentStackSlotRebase.cpp b/llvm/lib/Target/X86/X86ArgumentStackSlotRebase.cpp index 19c4751e50f7..bf9b8e573059 100644 --- a/llvm/lib/Target/X86/X86ArgumentStackSlotRebase.cpp +++ b/llvm/lib/Target/X86/X86ArgumentStackSlotRebase.cpp @@ -40,9 +40,7 @@ class X86ArgumentStackSlotPass : public MachineFunctionPass { public: static char ID; // Pass identification, replacement for typeid - explicit X86ArgumentStackSlotPass() : MachineFunctionPass(ID) { - initializeX86ArgumentStackSlotPassPass(*PassRegistry::getPassRegistry()); - } + explicit X86ArgumentStackSlotPass() : MachineFunctionPass(ID) {} bool runOnMachineFunction(MachineFunction &MF) override; diff --git a/llvm/lib/Target/X86/X86FlagsCopyLowering.cpp b/llvm/lib/Target/X86/X86FlagsCopyLowering.cpp index ea1b6e97aa32..ab4c70aee486 100644 --- a/llvm/lib/Target/X86/X86FlagsCopyLowering.cpp +++ b/llvm/lib/Target/X86/X86FlagsCopyLowering.cpp @@ -277,8 +277,7 @@ bool X86FlagsCopyLoweringPass::runOnMachineFunction(MachineFunction &MF) { if (MDTWrapper) { MDT = &MDTWrapper->getDomTree(); } else { - OwnedMDT = std::make_unique<MachineDominatorTree>(); - OwnedMDT->getBase().recalculate(MF); + OwnedMDT = std::make_unique<MachineDominatorTree>(MF); MDT = OwnedMDT.get(); } diff --git a/llvm/lib/Target/X86/X86FloatingPoint.cpp b/llvm/lib/Target/X86/X86FloatingPoint.cpp index 34d8b774a186..24129de9b717 100644 --- a/llvm/lib/Target/X86/X86FloatingPoint.cpp +++ b/llvm/lib/Target/X86/X86FloatingPoint.cpp @@ -1652,24 +1652,25 @@ void FPS::handleSpecialFP(MachineBasicBlock::iterator &Inst) { } if (STUses && !isMask_32(STUses)) - MI.emitError("fixed input regs must be last on the x87 stack"); + MI.emitGenericError("fixed input regs must be last on the x87 stack"); unsigned NumSTUses = llvm::countr_one(STUses); // Defs must be contiguous from the stack top. ST0-STn. if (STDefs && !isMask_32(STDefs)) { - MI.emitError("output regs must be last on the x87 stack"); + MI.emitGenericError("output regs must be last on the x87 stack"); STDefs = NextPowerOf2(STDefs) - 1; } unsigned NumSTDefs = llvm::countr_one(STDefs); // So must the clobbered stack slots. ST0-STm, m >= n. if (STClobbers && !isMask_32(STDefs | STClobbers)) - MI.emitError("clobbers must be last on the x87 stack"); + MI.emitGenericError("clobbers must be last on the x87 stack"); // Popped inputs are the ones that are also clobbered or defined. unsigned STPopped = STUses & (STDefs | STClobbers); if (STPopped && !isMask_32(STPopped)) - MI.emitError("implicitly popped regs must be last on the x87 stack"); + MI.emitGenericError( + "implicitly popped regs must be last on the x87 stack"); unsigned NumSTPopped = llvm::countr_one(STPopped); LLVM_DEBUG(dbgs() << "Asm uses " << NumSTUses << " fixed regs, pops " diff --git a/llvm/lib/Target/X86/X86ISelDAGToDAG.cpp b/llvm/lib/Target/X86/X86ISelDAGToDAG.cpp index 93d196d0fde5..9b340a778b36 100644 --- a/llvm/lib/Target/X86/X86ISelDAGToDAG.cpp +++ b/llvm/lib/Target/X86/X86ISelDAGToDAG.cpp @@ -370,7 +370,7 @@ namespace { return false; // Walk all the users of the immediate. - for (const SDNode *User : N->uses()) { + for (const SDNode *User : N->users()) { if (UseCount >= 2) break; @@ -1095,7 +1095,7 @@ void X86DAGToDAGISel::PreprocessISelDAG() { SDNode *MaxLd = nullptr; SDValue Ptr = Ld->getBasePtr(); SDValue Chain = Ld->getChain(); - for (SDNode *User : Ptr->uses()) { + for (SDNode *User : Ptr->users()) { auto *UserLd = dyn_cast<LoadSDNode>(User); MVT UserVT = User->getSimpleValueType(0); if (User != N && UserLd && ISD::isNormalLoad(User) && @@ -2423,11 +2423,11 @@ SDValue X86DAGToDAGISel::matchIndexRecursively(SDValue N, if (CurDAG->isBaseWithConstantOffset(Src)) { SDValue AddSrc = Src.getOperand(0); auto *AddVal = cast<ConstantSDNode>(Src.getOperand(1)); - uint64_t Offset = (uint64_t)AddVal->getSExtValue(); - if (!foldOffsetIntoAddress(Offset * AM.Scale, AM)) { + int64_t Offset = AddVal->getSExtValue(); + if (!foldOffsetIntoAddress((uint64_t)Offset * AM.Scale, AM)) { SDLoc DL(N); SDValue ExtSrc = CurDAG->getNode(Opc, DL, VT, AddSrc); - SDValue ExtVal = CurDAG->getConstant(Offset, DL, VT); + SDValue ExtVal = CurDAG->getSignedConstant(Offset, DL, VT); SDValue ExtAdd = CurDAG->getNode(ISD::ADD, DL, VT, ExtSrc, ExtVal); insertDAGNode(*CurDAG, N, ExtSrc); insertDAGNode(*CurDAG, N, ExtVal); @@ -3318,24 +3318,25 @@ X86::CondCode X86DAGToDAGISel::getCondFromNode(SDNode *N) const { /// other than ZF. bool X86DAGToDAGISel::onlyUsesZeroFlag(SDValue Flags) const { // Examine each user of the node. - for (SDNode::use_iterator UI = Flags->use_begin(), UE = Flags->use_end(); - UI != UE; ++UI) { + for (SDUse &Use : Flags->uses()) { // Only check things that use the flags. - if (UI.getUse().getResNo() != Flags.getResNo()) + if (Use.getResNo() != Flags.getResNo()) continue; + SDNode *User = Use.getUser(); // Only examine CopyToReg uses that copy to EFLAGS. - if (UI->getOpcode() != ISD::CopyToReg || - cast<RegisterSDNode>(UI->getOperand(1))->getReg() != X86::EFLAGS) + if (User->getOpcode() != ISD::CopyToReg || + cast<RegisterSDNode>(User->getOperand(1))->getReg() != X86::EFLAGS) return false; // Examine each user of the CopyToReg use. - for (SDNode::use_iterator FlagUI = UI->use_begin(), - FlagUE = UI->use_end(); FlagUI != FlagUE; ++FlagUI) { + for (SDUse &FlagUse : User->uses()) { // Only examine the Flag result. - if (FlagUI.getUse().getResNo() != 1) continue; + if (FlagUse.getResNo() != 1) + continue; // Anything unusual: assume conservatively. - if (!FlagUI->isMachineOpcode()) return false; + if (!FlagUse.getUser()->isMachineOpcode()) + return false; // Examine the condition code of the user. - X86::CondCode CC = getCondFromNode(*FlagUI); + X86::CondCode CC = getCondFromNode(FlagUse.getUser()); switch (CC) { // Comparisons which only use the zero flag. @@ -3354,24 +3355,25 @@ bool X86DAGToDAGISel::onlyUsesZeroFlag(SDValue Flags) const { /// flag to be accurate. bool X86DAGToDAGISel::hasNoSignFlagUses(SDValue Flags) const { // Examine each user of the node. - for (SDNode::use_iterator UI = Flags->use_begin(), UE = Flags->use_end(); - UI != UE; ++UI) { + for (SDUse &Use : Flags->uses()) { // Only check things that use the flags. - if (UI.getUse().getResNo() != Flags.getResNo()) + if (Use.getResNo() != Flags.getResNo()) continue; + SDNode *User = Use.getUser(); // Only examine CopyToReg uses that copy to EFLAGS. - if (UI->getOpcode() != ISD::CopyToReg || - cast<RegisterSDNode>(UI->getOperand(1))->getReg() != X86::EFLAGS) + if (User->getOpcode() != ISD::CopyToReg || + cast<RegisterSDNode>(User->getOperand(1))->getReg() != X86::EFLAGS) return false; // Examine each user of the CopyToReg use. - for (SDNode::use_iterator FlagUI = UI->use_begin(), - FlagUE = UI->use_end(); FlagUI != FlagUE; ++FlagUI) { + for (SDUse &FlagUse : User->uses()) { // Only examine the Flag result. - if (FlagUI.getUse().getResNo() != 1) continue; + if (FlagUse.getResNo() != 1) + continue; // Anything unusual: assume conservatively. - if (!FlagUI->isMachineOpcode()) return false; + if (!FlagUse.getUser()->isMachineOpcode()) + return false; // Examine the condition code of the user. - X86::CondCode CC = getCondFromNode(*FlagUI); + X86::CondCode CC = getCondFromNode(FlagUse.getUser()); switch (CC) { // Comparisons which don't examine the SF flag. @@ -3410,29 +3412,28 @@ static bool mayUseCarryFlag(X86::CondCode CC) { /// CF flag to be accurate. bool X86DAGToDAGISel::hasNoCarryFlagUses(SDValue Flags) const { // Examine each user of the node. - for (SDNode::use_iterator UI = Flags->use_begin(), UE = Flags->use_end(); - UI != UE; ++UI) { + for (SDUse &Use : Flags->uses()) { // Only check things that use the flags. - if (UI.getUse().getResNo() != Flags.getResNo()) + if (Use.getResNo() != Flags.getResNo()) continue; - unsigned UIOpc = UI->getOpcode(); + SDNode *User = Use.getUser(); + unsigned UserOpc = User->getOpcode(); - if (UIOpc == ISD::CopyToReg) { + if (UserOpc == ISD::CopyToReg) { // Only examine CopyToReg uses that copy to EFLAGS. - if (cast<RegisterSDNode>(UI->getOperand(1))->getReg() != X86::EFLAGS) + if (cast<RegisterSDNode>(User->getOperand(1))->getReg() != X86::EFLAGS) return false; // Examine each user of the CopyToReg use. - for (SDNode::use_iterator FlagUI = UI->use_begin(), FlagUE = UI->use_end(); - FlagUI != FlagUE; ++FlagUI) { + for (SDUse &FlagUse : User->uses()) { // Only examine the Flag result. - if (FlagUI.getUse().getResNo() != 1) + if (FlagUse.getResNo() != 1) continue; // Anything unusual: assume conservatively. - if (!FlagUI->isMachineOpcode()) + if (!FlagUse.getUser()->isMachineOpcode()) return false; // Examine the condition code of the user. - X86::CondCode CC = getCondFromNode(*FlagUI); + X86::CondCode CC = getCondFromNode(FlagUse.getUser()); if (mayUseCarryFlag(CC)) return false; @@ -3445,7 +3446,7 @@ static bool mayUseCarryFlag(X86::CondCode CC) { // This might be an unselected node. So look for the pre-isel opcodes that // use flags. unsigned CCOpNo; - switch (UIOpc) { + switch (UserOpc) { default: // Something unusual. Be conservative. return false; @@ -3455,7 +3456,7 @@ static bool mayUseCarryFlag(X86::CondCode CC) { case X86ISD::BRCOND: CCOpNo = 2; break; } - X86::CondCode CC = (X86::CondCode)UI->getConstantOperandVal(CCOpNo); + X86::CondCode CC = (X86::CondCode)User->getConstantOperandVal(CCOpNo); if (mayUseCarryFlag(CC)) return false; } diff --git a/llvm/lib/Target/X86/X86ISelLowering.cpp b/llvm/lib/Target/X86/X86ISelLowering.cpp index 4ae3e8fbe0f8..a0514e93d659 100644 --- a/llvm/lib/Target/X86/X86ISelLowering.cpp +++ b/llvm/lib/Target/X86/X86ISelLowering.cpp @@ -2333,6 +2333,9 @@ X86TargetLowering::X86TargetLowering(const X86TargetMachine &TM, setLoadExtAction(ISD::EXTLOAD, MVT::v8f64, MVT::v8f16, Legal); setLoadExtAction(ISD::EXTLOAD, MVT::v16f32, MVT::v16f16, Legal); + + setOperationAction(ISD::FMINIMUM, MVT::v32f16, Custom); + setOperationAction(ISD::FMAXIMUM, MVT::v32f16, Custom); } if (Subtarget.hasVLX()) { @@ -2377,6 +2380,12 @@ X86TargetLowering::X86TargetLowering(const X86TargetMachine &TM, // Need to custom widen these to prevent scalarization. setOperationAction(ISD::LOAD, MVT::v4f16, Custom); setOperationAction(ISD::STORE, MVT::v4f16, Custom); + + setOperationAction(ISD::FMINIMUM, MVT::v8f16, Custom); + setOperationAction(ISD::FMAXIMUM, MVT::v8f16, Custom); + + setOperationAction(ISD::FMINIMUM, MVT::v16f16, Custom); + setOperationAction(ISD::FMAXIMUM, MVT::v16f16, Custom); } } @@ -2433,6 +2442,8 @@ X86TargetLowering::X86TargetLowering(const X86TargetMachine &TM, setOperationAction(ISD::FSQRT, VT, Legal); setOperationAction(ISD::FMA, VT, Legal); setOperationAction(ISD::SETCC, VT, Custom); + setOperationAction(ISD::FMINIMUM, VT, Custom); + setOperationAction(ISD::FMAXIMUM, VT, Custom); } if (Subtarget.hasAVX10_2_512()) { setOperationAction(ISD::FADD, MVT::v32bf16, Legal); @@ -2442,6 +2453,8 @@ X86TargetLowering::X86TargetLowering(const X86TargetMachine &TM, setOperationAction(ISD::FSQRT, MVT::v32bf16, Legal); setOperationAction(ISD::FMA, MVT::v32bf16, Legal); setOperationAction(ISD::SETCC, MVT::v32bf16, Custom); + setOperationAction(ISD::FMINIMUM, MVT::v32bf16, Custom); + setOperationAction(ISD::FMAXIMUM, MVT::v32bf16, Custom); } for (auto VT : {MVT::f16, MVT::f32, MVT::f64}) { setCondCodeAction(ISD::SETOEQ, VT, Custom); @@ -2766,12 +2779,12 @@ bool X86::mayFoldLoadIntoBroadcastFromMem(SDValue Op, MVT EltVT, } bool X86::mayFoldIntoStore(SDValue Op) { - return Op.hasOneUse() && ISD::isNormalStore(*Op.getNode()->use_begin()); + return Op.hasOneUse() && ISD::isNormalStore(*Op.getNode()->user_begin()); } bool X86::mayFoldIntoZeroExtend(SDValue Op) { if (Op.hasOneUse()) { - unsigned Opcode = Op.getNode()->use_begin()->getOpcode(); + unsigned Opcode = Op.getNode()->user_begin()->getOpcode(); return (ISD::ZERO_EXTEND == Opcode); } return false; @@ -3208,14 +3221,16 @@ bool X86TargetLowering::shouldReduceLoadWidth(SDNode *Load, // can be store-folded. Therefore, it's probably not worth splitting the load. EVT VT = Load->getValueType(0); if ((VT.is256BitVector() || VT.is512BitVector()) && !Load->hasOneUse()) { - for (auto UI = Load->use_begin(), UE = Load->use_end(); UI != UE; ++UI) { + for (SDUse &Use : Load->uses()) { // Skip uses of the chain value. Result 0 of the node is the load value. - if (UI.getUse().getResNo() != 0) + if (Use.getResNo() != 0) continue; + SDNode *User = Use.getUser(); + // If this use is not an extract + store, it's probably worth splitting. - if (UI->getOpcode() != ISD::EXTRACT_SUBVECTOR || !UI->hasOneUse() || - UI->use_begin()->getOpcode() != ISD::STORE) + if (User->getOpcode() != ISD::EXTRACT_SUBVECTOR || !User->hasOneUse() || + User->user_begin()->getOpcode() != ISD::STORE) return true; } // All non-chain uses are extract + store. @@ -3306,7 +3321,7 @@ bool X86TargetLowering::shouldScalarizeBinop(SDValue VecOp) const { // Assume target opcodes can't be scalarized. // TODO - do we have any exceptions? - if (Opc >= ISD::BUILTIN_OP_END) + if (Opc >= ISD::BUILTIN_OP_END || !isBinOp(Opc)) return false; // If the vector op is not supported, try to convert to scalar. @@ -5244,7 +5259,8 @@ static SDValue IsNOT(SDValue V, SelectionDAG &DAG) { SmallVector<APInt> EltBits; if (getTargetConstantBitsFromNode(V.getOperand(0), V.getScalarValueSizeInBits(), UndefElts, - EltBits)) { + EltBits) && + !ISD::isBuildVectorOfConstantSDNodes(V.getOperand(1).getNode())) { // Don't fold min_signed_value -> (min_signed_value - 1) bool MinSigned = false; for (APInt &Elt : EltBits) { @@ -7397,7 +7413,7 @@ static Constant *getConstantVector(MVT VT, const APInt &SplatValue, } static bool isFoldableUseOfShuffle(SDNode *N) { - for (auto *U : N->uses()) { + for (auto *U : N->users()) { unsigned Opc = U->getOpcode(); // VPERMV/VPERMV3 shuffles can never fold their index operands. if (Opc == X86ISD::VPERMV && U->getOperand(0).getNode() == N) @@ -16004,7 +16020,7 @@ static SDValue lowerShufflePairAsUNPCKAndPermute(const SDLoc &DL, MVT VT, // Find the intersection between shuffle users of V1 and V2. SmallVector<SDNode *, 2> Shuffles; - for (SDNode *User : V1->uses()) + for (SDNode *User : V1->users()) if (User->getOpcode() == ISD::VECTOR_SHUFFLE && User->getOperand(0) == V1 && User->getOperand(1) == V2) Shuffles.push_back(User); @@ -17318,6 +17334,13 @@ static SDValue lowerV32I16Shuffle(const SDLoc &DL, ArrayRef<int> Mask, Zeroable, Subtarget, DAG)) return PSHUFB; + // Try to simplify this by merging 128-bit lanes to enable a lane-based + // shuffle. + if (!V2.isUndef()) + if (SDValue Result = lowerShuffleAsLanePermuteAndRepeatedMask( + DL, MVT::v32i16, V1, V2, Mask, Subtarget, DAG)) + return Result; + return lowerShuffleWithPERMV(DL, MVT::v32i16, Mask, V1, V2, Subtarget, DAG); } @@ -18205,7 +18228,7 @@ static SDValue LowerEXTRACT_VECTOR_ELT_SSE4(SDValue Op, SelectionDAG &DAG) { // because a MOVSSmr can be used instead, which is smaller and faster. if (!Op.hasOneUse()) return SDValue(); - SDNode *User = *Op.getNode()->use_begin(); + SDNode *User = *Op.getNode()->user_begin(); if ((User->getOpcode() != ISD::STORE || isNullConstant(Idx)) && (User->getOpcode() != ISD::BITCAST || User->getValueType(0) != MVT::i32)) @@ -18273,7 +18296,7 @@ static APInt getExtractedDemandedElts(SDNode *N) { MVT VT = N->getSimpleValueType(0); unsigned NumElts = VT.getVectorNumElements(); APInt DemandedElts = APInt::getZero(NumElts); - for (SDNode *User : N->uses()) { + for (SDNode *User : N->users()) { switch (User->getOpcode()) { case X86ISD::PEXTRB: case X86ISD::PEXTRW: @@ -18958,11 +18981,10 @@ static SDValue GetTLSADDR(SelectionDAG &DAG, GlobalAddressSDNode *GA, SDValue Ret; if (LocalDynamic && UseTLSDESC) { TGA = DAG.getTargetExternalSymbol("_TLS_MODULE_BASE_", PtrVT, OperandFlags); - auto UI = TGA->use_begin(); // Reuse existing GetTLSADDR node if we can find it. - if (UI != TGA->use_end()) { + if (TGA->hasOneUse()) { // TLSDESC uses TGA. - auto TLSDescOp = UI; + SDNode *TLSDescOp = *TGA->user_begin(); assert(TLSDescOp->getOpcode() == X86ISD::TLSDESC && "Unexpected TLSDESC DAG"); // CALLSEQ_END uses TGA via a chain and glue. @@ -22136,7 +22158,7 @@ static SDValue LowerFABSorFNEG(SDValue Op, SelectionDAG &DAG) { // If this is a FABS and it has an FNEG user, bail out to fold the combination // into an FNABS. We'll lower the FABS after that if it is still in use. if (IsFABS) - for (SDNode *User : Op->uses()) + for (SDNode *User : Op->users()) if (User->getOpcode() == ISD::FNEG) return Op; @@ -22860,14 +22882,13 @@ static SDValue MatchVectorAllEqualTest(SDValue LHS, SDValue RHS, /// return true if \c Op has a use that doesn't just read flags. static bool hasNonFlagsUse(SDValue Op) { - for (SDNode::use_iterator UI = Op->use_begin(), UE = Op->use_end(); UI != UE; - ++UI) { - SDNode *User = *UI; - unsigned UOpNo = UI.getOperandNo(); + for (SDUse &Use : Op->uses()) { + SDNode *User = Use.getUser(); + unsigned UOpNo = Use.getOperandNo(); if (User->getOpcode() == ISD::TRUNCATE && User->hasOneUse()) { - // Look pass truncate. - UOpNo = User->use_begin().getOperandNo(); - User = *User->use_begin(); + // Look past truncate. + UOpNo = User->use_begin()->getOperandNo(); + User = User->use_begin()->getUser(); } if (User->getOpcode() != ISD::BRCOND && User->getOpcode() != ISD::SETCC && @@ -22881,7 +22902,7 @@ static bool hasNonFlagsUse(SDValue Op) { // using an RMW op or only the flags are used. Otherwise, leave // the node alone and emit a 'cmp' or 'test' instruction. static bool isProfitableToUseFlagOp(SDValue Op) { - for (SDNode *U : Op->uses()) + for (SDNode *U : Op->users()) if (U->getOpcode() != ISD::CopyToReg && U->getOpcode() != ISD::SETCC && U->getOpcode() != ISD::STORE) @@ -23559,6 +23580,9 @@ static SDValue LowerVSETCC(SDValue Op, const X86Subtarget &Subtarget, SDValue Chain = IsStrict ? Op.getOperand(0) : SDValue(); if (isSoftF16(EltVT, Subtarget)) { + if (Subtarget.hasAVX512() && !Subtarget.hasVLX()) + return SDValue(); + // Break 256-bit FP vector compare into smaller ones. if (OpVT.is256BitVector() && !Subtarget.useAVX512Regs()) return splitVSETCC(VT, Op0, Op1, Cond, DAG, dl); @@ -24228,8 +24252,10 @@ SDValue X86TargetLowering::LowerSETCC(SDValue Op, SelectionDAG &DAG) const { if (Subtarget.hasAVX10_2()) { if (CC == ISD::SETOEQ || CC == ISD::SETUNE) { auto NewCC = (CC == ISD::SETOEQ) ? X86::COND_E : (X86::COND_NE); - return getSETCC(NewCC, DAG.getNode(X86ISD::UCOMX, dl, MVT::i32, Op0, Op1), - dl, DAG); + assert(Op0.getSimpleValueType() != MVT::bf16 && "Unsupported Type"); + if (Op0.getSimpleValueType() != MVT::f80) + return getSETCC( + NewCC, DAG.getNode(X86ISD::UCOMX, dl, MVT::i32, Op0, Op1), dl, DAG); } } // Handle floating point. @@ -25253,7 +25279,7 @@ SDValue X86TargetLowering::LowerBRCOND(SDValue Op, SelectionDAG &DAG) const { // have a fall-through edge, because this requires an explicit // jmp when the condition is false. if (Op.getNode()->hasOneUse()) { - SDNode *User = *Op.getNode()->use_begin(); + SDNode *User = *Op.getNode()->user_begin(); // Look for an unconditional branch following this conditional branch. // We need this because we need to reverse the successors in order // to implement FCMP_OEQ. @@ -28820,6 +28846,20 @@ static SDValue LowerFMINIMUM_FMAXIMUM(SDValue Op, const X86Subtarget &Subtarget, SDValue X = Op.getOperand(0); SDValue Y = Op.getOperand(1); SDLoc DL(Op); + if (Subtarget.hasAVX10_2() && TLI.isTypeLegal(VT)) { + unsigned Opc = 0; + if (VT.isVector()) + Opc = X86ISD::VMINMAX; + else if (VT == MVT::f16 || VT == MVT::f32 || VT == MVT::f64) + Opc = X86ISD::VMINMAXS; + + if (Opc) { + SDValue Imm = + DAG.getTargetConstant(Op.getOpcode() == ISD::FMAXIMUM, DL, MVT::i32); + return DAG.getNode(Opc, DL, VT, X, Y, Imm, Op->getFlags()); + } + } + uint64_t SizeInBits = VT.getScalarSizeInBits(); APInt PreferredZero = APInt::getZero(SizeInBits); APInt OppositeZero = PreferredZero; @@ -29988,6 +30028,7 @@ static SDValue LowerShift(SDValue Op, const X86Subtarget &Subtarget, SDLoc dl(Op); SDValue R = Op.getOperand(0); SDValue Amt = Op.getOperand(1); + unsigned NumElts = VT.getVectorNumElements(); unsigned EltSizeInBits = VT.getScalarSizeInBits(); bool ConstantAmt = ISD::isBuildVectorOfConstantSDNodes(Amt.getNode()); @@ -30044,6 +30085,23 @@ static SDValue LowerShift(SDValue Op, const X86Subtarget &Subtarget, return DAG.getVectorShuffle(VT, dl, R0, R1, {0, 3}); } + // Build a map of inrange constant amounts with element mask where they occur. + SmallDenseMap<unsigned, APInt, 16> UniqueCstAmt; + if (ConstantAmt) { + for (unsigned I = 0; I != NumElts; ++I) { + SDValue A = Amt.getOperand(I); + if (A.isUndef() || A->getAsAPIntVal().uge(EltSizeInBits)) + continue; + unsigned CstAmt = A->getAsAPIntVal().getZExtValue(); + if (UniqueCstAmt.count(CstAmt)) { + UniqueCstAmt[CstAmt].setBit(I); + continue; + } + UniqueCstAmt[CstAmt] = APInt::getOneBitSet(NumElts, I); + } + assert(!UniqueCstAmt.empty() && "Illegal constant shift amounts"); + } + // If possible, lower this shift as a sequence of two shifts by // constant plus a BLENDing shuffle instead of scalarizing it. // Example: @@ -30054,46 +30112,31 @@ static SDValue LowerShift(SDValue Op, const X86Subtarget &Subtarget, // // The advantage is that the two shifts from the example would be // lowered as X86ISD::VSRLI nodes in parallel before blending. - if (ConstantAmt && (VT == MVT::v8i16 || VT == MVT::v4i32 || - (VT == MVT::v16i16 && Subtarget.hasInt256()))) { - SDValue Amt1, Amt2; - unsigned NumElts = VT.getVectorNumElements(); - SmallVector<int, 8> ShuffleMask; - for (unsigned i = 0; i != NumElts; ++i) { - SDValue A = Amt->getOperand(i); - if (A.isUndef()) { - ShuffleMask.push_back(SM_SentinelUndef); - continue; - } - if (!Amt1 || Amt1 == A) { - ShuffleMask.push_back(i); - Amt1 = A; - continue; - } - if (!Amt2 || Amt2 == A) { - ShuffleMask.push_back(i + NumElts); - Amt2 = A; - continue; - } - break; + if (UniqueCstAmt.size() == 2 && + (VT == MVT::v8i16 || VT == MVT::v4i32 || + (VT == MVT::v16i16 && Subtarget.hasInt256()))) { + unsigned AmtA = UniqueCstAmt.begin()->first; + unsigned AmtB = std::next(UniqueCstAmt.begin())->first; + const APInt &MaskA = UniqueCstAmt.begin()->second; + const APInt &MaskB = std::next(UniqueCstAmt.begin())->second; + SmallVector<int, 8> ShuffleMask(NumElts, SM_SentinelUndef); + for (unsigned I = 0; I != NumElts; ++I) { + if (MaskA[I]) + ShuffleMask[I] = I; + if (MaskB[I]) + ShuffleMask[I] = I + NumElts; } // Only perform this blend if we can perform it without loading a mask. - if (ShuffleMask.size() == NumElts && Amt1 && Amt2 && - (VT != MVT::v16i16 || + if ((VT != MVT::v16i16 || is128BitLaneRepeatedShuffleMask(VT, ShuffleMask)) && (VT == MVT::v4i32 || Subtarget.hasSSE41() || Opc != ISD::SHL || canWidenShuffleElements(ShuffleMask))) { - auto *Cst1 = dyn_cast<ConstantSDNode>(Amt1); - auto *Cst2 = dyn_cast<ConstantSDNode>(Amt2); - if (Cst1 && Cst2 && Cst1->getAPIntValue().ult(EltSizeInBits) && - Cst2->getAPIntValue().ult(EltSizeInBits)) { - SDValue Shift1 = getTargetVShiftByConstNode(X86OpcI, dl, VT, R, - Cst1->getZExtValue(), DAG); - SDValue Shift2 = getTargetVShiftByConstNode(X86OpcI, dl, VT, R, - Cst2->getZExtValue(), DAG); - return DAG.getVectorShuffle(VT, dl, Shift1, Shift2, ShuffleMask); - } + SDValue Shift1 = + DAG.getNode(Opc, dl, VT, R, DAG.getConstant(AmtA, dl, VT)); + SDValue Shift2 = + DAG.getNode(Opc, dl, VT, R, DAG.getConstant(AmtB, dl, VT)); + return DAG.getVectorShuffle(VT, dl, Shift1, Shift2, ShuffleMask); } } @@ -30104,18 +30147,13 @@ static SDValue LowerShift(SDValue Op, const X86Subtarget &Subtarget, VT == MVT::v8i16 || VT == MVT::v16i16 || VT == MVT::v32i16) && !Subtarget.hasXOP()) { MVT NarrowScalarVT = VT.getScalarType(); - int NumElts = VT.getVectorNumElements(); // We can do this extra fast if each pair of narrow elements is shifted by // the same amount by doing this SWAR style: use a shift to move the valid // bits to the right position, mask out any bits which crossed from one // element to the other. // This optimized lowering is only valid if the elements in a pair can // be treated identically. - SmallVector<SDValue, 32> AmtWideElts; - AmtWideElts.reserve(NumElts); - for (int I = 0; I != NumElts; ++I) { - AmtWideElts.push_back(Amt.getOperand(I)); - } + SmallVector<SDValue, 32> AmtWideElts(Amt->op_begin(), Amt->op_end()); SmallVector<SDValue, 32> TmpAmtWideElts; int WideEltSizeInBits = EltSizeInBits; while (WideEltSizeInBits < 32) { @@ -30189,7 +30227,7 @@ static SDValue LowerShift(SDValue Op, const X86Subtarget &Subtarget, (WideEltSizeInBits < 32 || IsConstantSplat) && !Subtarget.hasAVX2()) { Profitable = false; } - int WideNumElts = AmtWideElts.size(); + unsigned WideNumElts = AmtWideElts.size(); // We are only dealing with identical pairs. if (Profitable && WideNumElts != NumElts) { MVT WideScalarVT = MVT::getIntegerVT(WideEltSizeInBits); @@ -30213,8 +30251,7 @@ static SDValue LowerShift(SDValue Op, const X86Subtarget &Subtarget, // another. The construction of this mask will be constant folded. // The mask for a logical right shift is nearly identical, the only // difference is that the all ones mask is shifted right instead of left. - SDValue CstFullMask = DAG.getAllOnesConstant(dl, NarrowScalarVT); - SDValue SplatFullMask = DAG.getSplat(VT, dl, CstFullMask); + SDValue SplatFullMask = DAG.getAllOnesConstant(dl, VT); SDValue Mask = DAG.getNode(LogicalOpc, dl, VT, SplatFullMask, Amt); Mask = DAG.getBitcast(WideVT, Mask); // Finally, we mask the shifted vector with the SWAR mask. @@ -30247,9 +30284,8 @@ static SDValue LowerShift(SDValue Op, const X86Subtarget &Subtarget, // // This is equal to Masked - 2*SignBitMask which will correctly sign // extend our result. - SDValue CstHighBit = - DAG.getConstant(1ULL << (EltSizeInBits - 1), dl, NarrowScalarVT); - SDValue SplatHighBit = DAG.getSplat(VT, dl, CstHighBit); + SDValue SplatHighBit = + DAG.getConstant(APInt::getSignMask(EltSizeInBits), dl, VT); // This does not induce recursion, all operands are constants. SDValue SignBitMask = DAG.getNode(LogicalOpc, dl, VT, SplatHighBit, Amt); SDValue FlippedSignBit = @@ -30365,7 +30401,6 @@ static SDValue LowerShift(SDValue Op, const X86Subtarget &Subtarget, if ((VT == MVT::v16i8 && Subtarget.hasSSSE3()) || (VT == MVT::v32i8 && Subtarget.hasInt256()) || (VT == MVT::v64i8 && Subtarget.hasBWI())) { - unsigned NumElts = VT.getVectorNumElements(); unsigned NumLanes = VT.getSizeInBits() / 128u; unsigned NumEltsPerLane = NumElts / NumLanes; SmallVector<APInt, 16> LUT; @@ -30405,7 +30440,7 @@ static SDValue LowerShift(SDValue Op, const X86Subtarget &Subtarget, assert((!Subtarget.hasBWI() || VT == MVT::v32i8 || VT == MVT::v16i8) && "Unexpected vector type"); MVT EvtSVT = Subtarget.hasBWI() ? MVT::i16 : MVT::i32; - MVT ExtVT = MVT::getVectorVT(EvtSVT, VT.getVectorNumElements()); + MVT ExtVT = MVT::getVectorVT(EvtSVT, NumElts); unsigned ExtOpc = Opc == ISD::SRA ? ISD::SIGN_EXTEND : ISD::ZERO_EXTEND; R = DAG.getNode(ExtOpc, dl, ExtVT, R); Amt = DAG.getNode(ISD::ZERO_EXTEND, dl, ExtVT, Amt); @@ -30419,7 +30454,6 @@ static SDValue LowerShift(SDValue Op, const X86Subtarget &Subtarget, (VT == MVT::v16i8 || (VT == MVT::v32i8 && Subtarget.hasInt256()) || (VT == MVT::v64i8 && Subtarget.hasBWI())) && !Subtarget.hasXOP()) { - int NumElts = VT.getVectorNumElements(); MVT VT16 = MVT::getVectorVT(MVT::i16, NumElts / 2); SDValue Cst8 = DAG.getTargetConstant(8, dl, MVT::i8); @@ -30441,7 +30475,7 @@ static SDValue LowerShift(SDValue Op, const X86Subtarget &Subtarget, } SmallVector<SDValue, 16> LoAmt, HiAmt; - for (int i = 0; i != NumElts; i += 16) { + for (unsigned i = 0; i != NumElts; i += 16) { for (int j = 0; j != 8; ++j) { LoAmt.push_back(Amt.getOperand(i + j)); HiAmt.push_back(Amt.getOperand(i + j + 8)); @@ -30465,14 +30499,14 @@ static SDValue LowerShift(SDValue Op, const X86Subtarget &Subtarget, if (VT == MVT::v16i8 || (VT == MVT::v32i8 && Subtarget.hasInt256() && !Subtarget.hasXOP()) || (VT == MVT::v64i8 && Subtarget.hasBWI())) { - MVT ExtVT = MVT::getVectorVT(MVT::i16, VT.getVectorNumElements() / 2); + MVT ExtVT = MVT::getVectorVT(MVT::i16, NumElts / 2); auto SignBitSelect = [&](MVT SelVT, SDValue Sel, SDValue V0, SDValue V1) { if (VT.is512BitVector()) { // On AVX512BW targets we make use of the fact that VSELECT lowers // to a masked blend which selects bytes based just on the sign bit // extracted to a mask. - MVT MaskVT = MVT::getVectorVT(MVT::i1, VT.getVectorNumElements()); + MVT MaskVT = MVT::getVectorVT(MVT::i1, NumElts); V0 = DAG.getBitcast(VT, V0); V1 = DAG.getBitcast(VT, V1); Sel = DAG.getBitcast(VT, Sel); @@ -30599,7 +30633,7 @@ static SDValue LowerShift(SDValue Op, const X86Subtarget &Subtarget, // On SSE41 targets we can use PBLENDVB which selects bytes based just on // the sign bit. if (UseSSE41) { - MVT ExtVT = MVT::getVectorVT(MVT::i8, VT.getVectorNumElements() * 2); + MVT ExtVT = MVT::getVectorVT(MVT::i8, NumElts * 2); V0 = DAG.getBitcast(ExtVT, V0); V1 = DAG.getBitcast(ExtVT, V1); Sel = DAG.getBitcast(ExtVT, Sel); @@ -31465,7 +31499,7 @@ static bool shouldExpandCmpArithRMWInIR(AtomicRMWInst *AI) { return false; Value *Op = AI->getOperand(1); - ICmpInst::Predicate Pred; + CmpPredicate Pred; Instruction *I = AI->user_back(); AtomicRMWInst::BinOp Opc = AI->getOperation(); if (Opc == AtomicRMWInst::Add) { @@ -32643,8 +32677,7 @@ static SDValue ExtendToType(SDValue InOp, MVT NVT, SelectionDAG &DAG, "Unexpected request for vector widening"); SDLoc dl(InOp); - if (InOp.getOpcode() == ISD::CONCAT_VECTORS && - InOp.getNumOperands() == 2) { + if (InOp.getOpcode() == ISD::CONCAT_VECTORS && InOp.getNumOperands() == 2) { SDValue N1 = InOp.getOperand(1); if ((ISD::isBuildVectorAllZeros(N1.getNode()) && FillWithZeroes) || N1.isUndef()) { @@ -32655,22 +32688,17 @@ static SDValue ExtendToType(SDValue InOp, MVT NVT, SelectionDAG &DAG, } if (ISD::isBuildVectorOfConstantSDNodes(InOp.getNode()) || ISD::isBuildVectorOfConstantFPSDNodes(InOp.getNode())) { - SmallVector<SDValue, 16> Ops; - for (unsigned i = 0; i < InNumElts; ++i) - Ops.push_back(InOp.getOperand(i)); - EVT EltVT = InOp.getOperand(0).getValueType(); - - SDValue FillVal = FillWithZeroes ? DAG.getConstant(0, dl, EltVT) : - DAG.getUNDEF(EltVT); - for (unsigned i = 0; i < WidenNumElts - InNumElts; ++i) - Ops.push_back(FillVal); + SDValue FillVal = + FillWithZeroes ? DAG.getConstant(0, dl, EltVT) : DAG.getUNDEF(EltVT); + SmallVector<SDValue, 16> Ops(InOp->op_begin(), InOp->op_end()); + Ops.append(WidenNumElts - InNumElts, FillVal); return DAG.getBuildVector(NVT, dl, Ops); } - SDValue FillVal = FillWithZeroes ? DAG.getConstant(0, dl, NVT) : - DAG.getUNDEF(NVT); - return DAG.getNode(ISD::INSERT_SUBVECTOR, dl, NVT, FillVal, - InOp, DAG.getIntPtrConstant(0, dl)); + SDValue FillVal = + FillWithZeroes ? DAG.getConstant(0, dl, NVT) : DAG.getUNDEF(NVT); + return DAG.getNode(ISD::INSERT_SUBVECTOR, dl, NVT, FillVal, InOp, + DAG.getIntPtrConstant(0, dl)); } static SDValue LowerMSCATTER(SDValue Op, const X86Subtarget &Subtarget, @@ -33250,7 +33278,8 @@ void X86TargetLowering::ReplaceNodeResults(SDNode *N, SmallVectorImpl<SDValue>&Results, SelectionDAG &DAG) const { SDLoc dl(N); - switch (N->getOpcode()) { + unsigned Opc = N->getOpcode(); + switch (Opc) { default: #ifndef NDEBUG dbgs() << "ReplaceNodeResults: "; @@ -33346,7 +33375,7 @@ void X86TargetLowering::ReplaceNodeResults(SDNode *N, EVT VT = N->getValueType(0); assert(getTypeAction(*DAG.getContext(), VT) == TypeWidenVector && VT == MVT::v2i32 && "Unexpected VT!"); - bool IsSigned = N->getOpcode() == ISD::SMULO; + bool IsSigned = Opc == ISD::SMULO; unsigned ExtOpc = IsSigned ? ISD::SIGN_EXTEND : ISD::ZERO_EXTEND; SDValue Op0 = DAG.getNode(ExtOpc, dl, MVT::v2i64, N->getOperand(0)); SDValue Op1 = DAG.getNode(ExtOpc, dl, MVT::v2i64, N->getOperand(1)); @@ -33403,7 +33432,7 @@ void X86TargetLowering::ReplaceNodeResults(SDNode *N, Ops[0] = N->getOperand(1); SDValue InVec1 = DAG.getNode(ISD::CONCAT_VECTORS, dl, InWideVT, Ops); - SDValue Res = DAG.getNode(N->getOpcode(), dl, WideVT, InVec0, InVec1); + SDValue Res = DAG.getNode(Opc, dl, WideVT, InVec0, InVec1); Results.push_back(Res); return; } @@ -33411,15 +33440,26 @@ void X86TargetLowering::ReplaceNodeResults(SDNode *N, case X86ISD::FMINC: case X86ISD::FMIN: case X86ISD::FMAXC: - case X86ISD::FMAX: { + case X86ISD::FMAX: + case X86ISD::STRICT_FMIN: + case X86ISD::STRICT_FMAX: { EVT VT = N->getValueType(0); assert(VT == MVT::v2f32 && "Unexpected type (!= v2f32) on FMIN/FMAX."); + bool IsStrict = Opc == X86ISD::STRICT_FMIN || Opc == X86ISD::STRICT_FMAX; SDValue UNDEF = DAG.getUNDEF(VT); SDValue LHS = DAG.getNode(ISD::CONCAT_VECTORS, dl, MVT::v4f32, - N->getOperand(0), UNDEF); + N->getOperand(IsStrict ? 1 : 0), UNDEF); SDValue RHS = DAG.getNode(ISD::CONCAT_VECTORS, dl, MVT::v4f32, - N->getOperand(1), UNDEF); - Results.push_back(DAG.getNode(N->getOpcode(), dl, MVT::v4f32, LHS, RHS)); + N->getOperand(IsStrict ? 2 : 1), UNDEF); + SDValue Res; + if (IsStrict) + Res = DAG.getNode(Opc, dl, {MVT::v4f32, MVT::Other}, + {N->getOperand(0), LHS, RHS}); + else + Res = DAG.getNode(Opc, dl, MVT::v4f32, LHS, RHS); + Results.push_back(Res); + if (IsStrict) + Results.push_back(Res.getValue(1)); return; } case ISD::SDIV: @@ -33441,7 +33481,7 @@ void X86TargetLowering::ReplaceNodeResults(SDNode *N, EVT ResVT = getTypeToTransformTo(*DAG.getContext(), VT); SDValue N0 = DAG.getNode(ISD::CONCAT_VECTORS, dl, ResVT, Ops0); SDValue N1 = DAG.getConstant(SplatVal, dl, ResVT); - SDValue Res = DAG.getNode(N->getOpcode(), dl, ResVT, N0, N1); + SDValue Res = DAG.getNode(Opc, dl, ResVT, N0, N1); Results.push_back(Res); } return; @@ -33562,7 +33602,7 @@ void X86TargetLowering::ReplaceNodeResults(SDNode *N, (InVT == MVT::v4i16 || InVT == MVT::v4i8)){ assert(getTypeAction(*DAG.getContext(), InVT) == TypeWidenVector && "Unexpected type action!"); - assert(N->getOpcode() == ISD::SIGN_EXTEND && "Unexpected opcode"); + assert(Opc == ISD::SIGN_EXTEND && "Unexpected opcode"); // Custom split this so we can extend i8/i16->i32 invec. This is better // since sign_extend_inreg i8/i16->i64 requires an extend to i32 using // sra. Then extending from i32 to i64 using pcmpgt. By custom splitting @@ -33599,7 +33639,7 @@ void X86TargetLowering::ReplaceNodeResults(SDNode *N, // Promote the input to 128 bits. Type legalization will turn this into // zext_inreg/sext_inreg. - In = DAG.getNode(N->getOpcode(), dl, InVT, In); + In = DAG.getNode(Opc, dl, InVT, In); } // Perform custom splitting instead of the two stage extend we would get @@ -33608,7 +33648,7 @@ void X86TargetLowering::ReplaceNodeResults(SDNode *N, std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(N->getValueType(0)); assert(isTypeLegal(LoVT) && "Split VT not legal?"); - SDValue Lo = getEXTEND_VECTOR_INREG(N->getOpcode(), dl, LoVT, In, DAG); + SDValue Lo = getEXTEND_VECTOR_INREG(Opc, dl, LoVT, In, DAG); // We need to shift the input over by half the number of elements. unsigned NumElts = InVT.getVectorNumElements(); @@ -33618,7 +33658,7 @@ void X86TargetLowering::ReplaceNodeResults(SDNode *N, ShufMask[i] = i + HalfNumElts; SDValue Hi = DAG.getVectorShuffle(InVT, dl, In, In, ShufMask); - Hi = getEXTEND_VECTOR_INREG(N->getOpcode(), dl, HiVT, Hi, DAG); + Hi = getEXTEND_VECTOR_INREG(Opc, dl, HiVT, Hi, DAG); SDValue Res = DAG.getNode(ISD::CONCAT_VECTORS, dl, VT, Lo, Hi); Results.push_back(Res); @@ -33630,8 +33670,7 @@ void X86TargetLowering::ReplaceNodeResults(SDNode *N, case ISD::FP_TO_UINT: case ISD::STRICT_FP_TO_UINT: { bool IsStrict = N->isStrictFPOpcode(); - bool IsSigned = N->getOpcode() == ISD::FP_TO_SINT || - N->getOpcode() == ISD::STRICT_FP_TO_SINT; + bool IsSigned = Opc == ISD::FP_TO_SINT || Opc == ISD::STRICT_FP_TO_SINT; EVT VT = N->getValueType(0); SDValue Src = N->getOperand(IsStrict ? 1 : 0); SDValue Chain = IsStrict ? N->getOperand(0) : SDValue(); @@ -33642,13 +33681,13 @@ void X86TargetLowering::ReplaceNodeResults(SDNode *N, EVT NVT = VT.isVector() ? VT.changeVectorElementType(MVT::f32) : MVT::f32; if (IsStrict) { Res = - DAG.getNode(N->getOpcode(), dl, {VT, MVT::Other}, + DAG.getNode(Opc, dl, {VT, MVT::Other}, {Chain, DAG.getNode(ISD::STRICT_FP_EXTEND, dl, {NVT, MVT::Other}, {Chain, Src})}); Chain = Res.getValue(1); } else { - Res = DAG.getNode(N->getOpcode(), dl, VT, - DAG.getNode(ISD::FP_EXTEND, dl, NVT, Src)); + Res = + DAG.getNode(Opc, dl, VT, DAG.getNode(ISD::FP_EXTEND, dl, NVT, Src)); } Results.push_back(Res); if (IsStrict) @@ -33671,13 +33710,12 @@ void X86TargetLowering::ReplaceNodeResults(SDNode *N, } if (IsStrict) { - unsigned Opc = - IsSigned ? X86ISD::STRICT_CVTTP2SI : X86ISD::STRICT_CVTTP2UI; + Opc = IsSigned ? X86ISD::STRICT_CVTTP2SI : X86ISD::STRICT_CVTTP2UI; Res = DAG.getNode(Opc, dl, {ResVT, MVT::Other}, {N->getOperand(0), Src}); Chain = Res.getValue(1); } else { - unsigned Opc = IsSigned ? X86ISD::CVTTP2SI : X86ISD::CVTTP2UI; + Opc = IsSigned ? X86ISD::CVTTP2SI : X86ISD::CVTTP2UI; Res = DAG.getNode(Opc, dl, ResVT, Src); } @@ -33763,7 +33801,6 @@ void X86TargetLowering::ReplaceNodeResults(SDNode *N, return; } - unsigned Opc; if (IsStrict) Opc = IsSigned ? X86ISD::STRICT_CVTTP2SI : X86ISD::STRICT_CVTTP2UI; else @@ -33802,7 +33839,7 @@ void X86TargetLowering::ReplaceNodeResults(SDNode *N, if (Src.getValueType() == MVT::v2f32 && IsStrict) { Src = DAG.getNode(ISD::CONCAT_VECTORS, dl, MVT::v4f32, Src, DAG.getConstantFP(0.0, dl, MVT::v2f32)); - SDValue Res = DAG.getNode(N->getOpcode(), dl, {MVT::v4i32, MVT::Other}, + SDValue Res = DAG.getNode(Opc, dl, {MVT::v4i32, MVT::Other}, {N->getOperand(0), Src}); Results.push_back(Res); Results.push_back(Res.getValue(1)); @@ -33826,7 +33863,6 @@ void X86TargetLowering::ReplaceNodeResults(SDNode *N, std::max(NumElts, 128U / (unsigned)SrcVT.getSizeInBits()); MVT VecVT = MVT::getVectorVT(MVT::i64, NumElts); MVT VecInVT = MVT::getVectorVT(SrcVT.getSimpleVT(), SrcElts); - unsigned Opc = N->getOpcode(); if (NumElts != SrcElts) { if (IsStrict) Opc = IsSigned ? X86ISD::STRICT_CVTTP2SI : X86ISD::STRICT_CVTTP2UI; @@ -33880,8 +33916,7 @@ void X86TargetLowering::ReplaceNodeResults(SDNode *N, case ISD::UINT_TO_FP: case ISD::STRICT_UINT_TO_FP: { bool IsStrict = N->isStrictFPOpcode(); - bool IsSigned = N->getOpcode() == ISD::SINT_TO_FP || - N->getOpcode() == ISD::STRICT_SINT_TO_FP; + bool IsSigned = Opc == ISD::SINT_TO_FP || Opc == ISD::STRICT_SINT_TO_FP; EVT VT = N->getValueType(0); SDValue Src = N->getOperand(IsStrict ? 1 : 0); if (VT.getVectorElementType() == MVT::f16 && Subtarget.hasFP16() && @@ -33975,7 +34010,7 @@ void X86TargetLowering::ReplaceNodeResults(SDNode *N, // FIXME: Should generic type legalizer do this? Src = DAG.getNode(ISD::CONCAT_VECTORS, dl, MVT::v4i32, Src, DAG.getConstant(0, dl, MVT::v2i32)); - SDValue Res = DAG.getNode(N->getOpcode(), dl, {MVT::v4f32, MVT::Other}, + SDValue Res = DAG.getNode(Opc, dl, {MVT::v4f32, MVT::Other}, {N->getOperand(0), Src}); Results.push_back(Res); Results.push_back(Res.getValue(1)); @@ -39406,8 +39441,8 @@ static SDValue combineX86ShuffleChain(ArrayRef<SDValue> Inputs, SDValue Root, // from being reused. bool IsMaskedShuffle = false; if (RootSizeInBits == 512 || (Subtarget.hasVLX() && RootSizeInBits >= 128)) { - if (Root.hasOneUse() && Root->use_begin()->getOpcode() == ISD::VSELECT && - Root->use_begin()->getOperand(0).getScalarValueSizeInBits() == 1) { + if (Root.hasOneUse() && Root->user_begin()->getOpcode() == ISD::VSELECT && + Root->user_begin()->getOperand(0).getScalarValueSizeInBits() == 1) { IsMaskedShuffle = true; } } @@ -39835,6 +39870,10 @@ static SDValue combineX86ShuffleChain(ArrayRef<SDValue> Inputs, SDValue Root, bool AllowBWIVPERMV3 = (Depth >= (VariableCrossLaneShuffleDepth + 2) || HasVariableMask); + // If root was a VPERMV3 node, always allow a variable shuffle. + if (Root.getOpcode() == X86ISD::VPERMV3) + AllowVariableCrossLaneMask = AllowVariablePerLaneMask = true; + bool MaskContainsZeros = isAnyZero(Mask); if (is128BitLaneCrossingShuffleMask(MaskVT, Mask)) { @@ -41691,7 +41730,7 @@ static SDValue combineTargetShuffle(SDValue N, const SDLoc &DL, // Share broadcast with the longest vector and extract low subvector (free). // Ensure the same SDValue from the SDNode use is being used. - for (SDNode *User : Src->uses()) + for (SDNode *User : Src->users()) if (User != N.getNode() && User->getOpcode() == X86ISD::VBROADCAST && Src == User->getOperand(0) && User->getValueSizeInBits(0).getFixedValue() > @@ -42288,11 +42327,11 @@ static SDValue combineTargetShuffle(SDValue N, const SDLoc &DL, return SDValue(); } case X86ISD::VPERMV3: { - // Combine VPERMV3 to widened VPERMV if the two source operands are split - // from the same vector. SDValue V1 = peekThroughBitcasts(N.getOperand(0)); SDValue V2 = peekThroughBitcasts(N.getOperand(2)); MVT SVT = V1.getSimpleValueType(); + // Combine VPERMV3 to widened VPERMV if the two source operands are split + // from the same vector. if (V1.getOpcode() == ISD::EXTRACT_SUBVECTOR && V1.getConstantOperandVal(1) == 0 && V2.getOpcode() == ISD::EXTRACT_SUBVECTOR && @@ -42313,14 +42352,25 @@ static SDValue combineTargetShuffle(SDValue N, const SDLoc &DL, } SmallVector<SDValue, 2> Ops; SmallVector<int, 32> Mask; - if (isShuffleFoldableLoad(N.getOperand(0)) && - !isShuffleFoldableLoad(N.getOperand(2)) && - getTargetShuffleMask(N, /*AllowSentinelZero=*/false, Ops, Mask)) { - ShuffleVectorSDNode::commuteMask(Mask); - SDValue NewMask = getConstVector( - Mask, N.getOperand(1).getSimpleValueType(), DAG, DL, /*IsMask=*/true); - return DAG.getNode(X86ISD::VPERMV3, DL, VT, N.getOperand(2), NewMask, - N.getOperand(0)); + if (getTargetShuffleMask(N, /*AllowSentinelZero=*/false, Ops, Mask)) { + MVT MaskVT = N.getOperand(1).getSimpleValueType(); + // Canonicalize to VPERMV if both sources are the same. + if (V1 == V2) { + for (int &M : Mask) + M = (M < 0 ? M : M & Mask.size() - 1); + SDValue NewMask = getConstVector(Mask, MaskVT, DAG, DL, + /*IsMask=*/true); + return DAG.getNode(X86ISD::VPERMV, DL, VT, NewMask, N.getOperand(0)); + } + // Commute foldable source to the RHS. + if (isShuffleFoldableLoad(N.getOperand(0)) && + !isShuffleFoldableLoad(N.getOperand(2))) { + ShuffleVectorSDNode::commuteMask(Mask); + SDValue NewMask = + getConstVector(Mask, MaskVT, DAG, DL, /*IsMask=*/true); + return DAG.getNode(X86ISD::VPERMV3, DL, VT, N.getOperand(2), NewMask, + N.getOperand(0)); + } } return SDValue(); } @@ -42878,7 +42928,7 @@ bool X86TargetLowering::SimplifyDemandedVectorEltsForTargetNode( // If we reuse the shift amount just for sse shift amounts then we know that // only the bottom 64-bits are only ever used. - bool AssumeSingleUse = llvm::all_of(Amt->uses(), [&Amt](SDNode *Use) { + bool AssumeSingleUse = llvm::all_of(Amt->users(), [&Amt](SDNode *Use) { unsigned UseOpc = Use->getOpcode(); return (UseOpc == X86ISD::VSHL || UseOpc == X86ISD::VSRL || UseOpc == X86ISD::VSRA) && @@ -45638,7 +45688,7 @@ combineExtractFromVectorLoad(SDNode *N, EVT VecVT, SDValue SrcVec, uint64_t Idx, const TargetLowering &TLI = DAG.getTargetLoweringInfo(); EVT VT = N->getValueType(0); - bool LikelyUsedAsVector = any_of(N->uses(), [](SDNode *Use) { + bool LikelyUsedAsVector = any_of(N->users(), [](SDNode *Use) { return Use->getOpcode() == ISD::STORE || Use->getOpcode() == ISD::INSERT_VECTOR_ELT || Use->getOpcode() == ISD::SCALAR_TO_VECTOR; @@ -46306,7 +46356,7 @@ static SDValue combineExtractVectorElt(SDNode *N, SelectionDAG &DAG, return false; }; // TODO: Can we drop the oneuse check for constant extracts? - if (all_of(InputVector->uses(), IsBoolExtract) && + if (all_of(InputVector->users(), IsBoolExtract) && (IsVar || BoolExtracts.size() > 1)) { EVT BCVT = EVT::getIntegerVT(*DAG.getContext(), NumSrcElts); if (SDValue BC = @@ -46698,11 +46748,10 @@ static SDValue combineVSelectToBLENDV(SDNode *N, SelectionDAG &DAG, return SDValue(); auto OnlyUsedAsSelectCond = [](SDValue Cond) { - for (SDNode::use_iterator UI = Cond->use_begin(), UE = Cond->use_end(); - UI != UE; ++UI) - if ((UI->getOpcode() != ISD::VSELECT && - UI->getOpcode() != X86ISD::BLENDV) || - UI.getOperandNo() != 0) + for (SDUse &Use : Cond->uses()) + if ((Use.getUser()->getOpcode() != ISD::VSELECT && + Use.getUser()->getOpcode() != X86ISD::BLENDV) || + Use.getOperandNo() != 0) return false; return true; @@ -46722,7 +46771,7 @@ static SDValue combineVSelectToBLENDV(SDNode *N, SelectionDAG &DAG, // the generic VSELECT anymore. Otherwise, we may perform wrong // optimizations as we messed with the actual expectation for the vector // boolean values. - for (SDNode *U : Cond->uses()) { + for (SDNode *U : Cond->users()) { if (U->getOpcode() == X86ISD::BLENDV) continue; @@ -48950,7 +48999,7 @@ static SDValue combineMul(SDNode *N, SelectionDAG &DAG, (SignMulAmt >= 0 && (MulAmt2 == 3 || MulAmt2 == 5 || MulAmt2 == 9)))) { if (isPowerOf2_64(MulAmt2) && !(SignMulAmt >= 0 && N->hasOneUse() && - N->use_begin()->getOpcode() == ISD::ADD)) + N->user_begin()->getOpcode() == ISD::ADD)) // If second multiplifer is pow2, issue it first. We want the multiply // by 3, 5, or 9 to be folded into the addressing mode unless the lone // use is an add. Only do this for positive multiply amounts since the @@ -49905,7 +49954,7 @@ static SDValue combineCompareEqual(SDNode *N, SelectionDAG &DAG, (VT == MVT::f16 && Subtarget.hasFP16())) { bool ExpectingFlags = false; // Check for any users that want flags: - for (const SDNode *U : N->uses()) { + for (const SDNode *U : N->users()) { if (ExpectingFlags) break; @@ -50733,7 +50782,7 @@ static SDValue combineX86SubCmpForFlags(SDNode *N, SDValue Flag, return SDValue(); // Check the only user of flag is `brcond ne`. - SDNode *BrCond = *Flag->uses().begin(); + SDNode *BrCond = *Flag->user_begin(); if (BrCond->getOpcode() != X86ISD::BRCOND) return SDValue(); unsigned CondNo = 2; @@ -52147,7 +52196,7 @@ static SDValue combineConstantPoolLoads(SDNode *N, const SDLoc &dl, // Look through all other loads/broadcasts in the chain for another constant // pool entry. - for (SDNode *User : Chain->uses()) { + for (SDNode *User : Chain->users()) { auto *UserLd = dyn_cast<MemSDNode>(User); if (User != N && UserLd && (User->getOpcode() == X86ISD::SUBV_BROADCAST_LOAD || @@ -52257,7 +52306,7 @@ static SDValue combineLoad(SDNode *N, SelectionDAG &DAG, (RegVT.is128BitVector() || RegVT.is256BitVector())) { SDValue Ptr = Ld->getBasePtr(); SDValue Chain = Ld->getChain(); - for (SDNode *User : Chain->uses()) { + for (SDNode *User : Chain->users()) { auto *UserLd = dyn_cast<MemSDNode>(User); if (User != N && UserLd && User->getOpcode() == X86ISD::SUBV_BROADCAST_LOAD && @@ -52662,7 +52711,7 @@ static SDValue combineStore(SDNode *N, SelectionDAG &DAG, } // Convert scalar fabs/fneg load-store to integer equivalents. - if ((VT == MVT::f32 || VT == MVT::f64) && + if ((VT == MVT::f16 || VT == MVT::bf16 || VT == MVT::f32 || VT == MVT::f64) && (StoredVal.getOpcode() == ISD::FABS || StoredVal.getOpcode() == ISD::FNEG) && ISD::isNormalLoad(StoredVal.getOperand(0).getNode()) && @@ -52809,6 +52858,44 @@ static SDValue combineStore(SDNode *N, SelectionDAG &DAG, } } + // Convert store(cmov(load(p), x, CC), p) to cstore(x, p, CC) + // store(cmov(x, load(p), CC), p) to cstore(x, p, InvertCC) + if ((VT == MVT::i16 || VT == MVT::i32 || VT == MVT::i64) && + Subtarget.hasCF() && St->isSimple()) { + SDValue Cmov; + if (StoredVal.getOpcode() == X86ISD::CMOV) + Cmov = StoredVal; + else if (StoredVal.getOpcode() == ISD::TRUNCATE && + StoredVal.getOperand(0).getOpcode() == X86ISD::CMOV) + Cmov = StoredVal.getOperand(0); + else + return SDValue(); + + auto *Ld = dyn_cast<LoadSDNode>(St->getChain()); + if (!Ld || !Ld->isSimple() || Ld->getBasePtr() != St->getBasePtr()) + return SDValue(); + + bool InvertCC = false; + SDValue V = SDValue(Ld, 0); + if (V == Cmov.getOperand(1)) + InvertCC = true; + else if (V != Cmov.getOperand(0)) + return SDValue(); + + SDVTList Tys = DAG.getVTList(MVT::Other); + SDValue CC = Cmov.getOperand(2); + SDValue Src = DAG.getAnyExtOrTrunc(Cmov.getOperand(!InvertCC), dl, VT); + if (InvertCC) + CC = DAG.getTargetConstant( + GetOppositeBranchCondition( + (X86::CondCode)Cmov.getConstantOperandVal(2)), + dl, MVT::i8); + SDValue Ops[] = {St->getChain(), Src, St->getBasePtr(), CC, + Cmov.getOperand(3)}; + return DAG.getMemIntrinsicNode(X86ISD::CSTORE, dl, Tys, Ops, VT, + St->getMemOperand()); + } + // Turn load->store of MMX types into GPR load/stores. This avoids clobbering // the FP state in cases where an emms may be missing. // A preferable solution to the general problem is to figure out the right @@ -53080,8 +53167,8 @@ static bool isHorizontalBinOp(unsigned HOpcode, SDValue &LHS, SDValue &RHS, return User->getOpcode() == HOpcode && User->getValueType(0) == VT; }; ForceHorizOp = - ForceHorizOp || (llvm::any_of(NewLHS->uses(), FoundHorizUser) && - llvm::any_of(NewRHS->uses(), FoundHorizUser)); + ForceHorizOp || (llvm::any_of(NewLHS->users(), FoundHorizUser) && + llvm::any_of(NewRHS->users(), FoundHorizUser)); // Assume a SingleSource HOP if we only shuffle one input and don't need to // shuffle the result. @@ -53106,9 +53193,9 @@ static SDValue combineToHorizontalAddSub(SDNode *N, SelectionDAG &DAG, auto MergableHorizOp = [N](unsigned HorizOpcode) { return N->hasOneUse() && - N->use_begin()->getOpcode() == ISD::VECTOR_SHUFFLE && - (N->use_begin()->getOperand(0).getOpcode() == HorizOpcode || - N->use_begin()->getOperand(1).getOpcode() == HorizOpcode); + N->user_begin()->getOpcode() == ISD::VECTOR_SHUFFLE && + (N->user_begin()->getOperand(0).getOpcode() == HorizOpcode || + N->user_begin()->getOperand(1).getOpcode() == HorizOpcode); }; switch (Opcode) { @@ -54422,7 +54509,8 @@ static SDValue combineX86INT_TO_FP(SDNode *N, SelectionDAG &DAG, static SDValue combineCVTP2I_CVTTP2I(SDNode *N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI) { - bool IsStrict = N->isTargetStrictFPOpcode(); + const SelectionDAGTargetInfo &TSI = DAG.getSelectionDAGInfo(); + bool IsStrict = TSI.isTargetStrictFPOpcode(N->getOpcode()); EVT VT = N->getValueType(0); // Convert a full vector load into vzload when not all bits are needed. @@ -54808,7 +54896,7 @@ static SDValue promoteExtBeforeAdd(SDNode *Ext, SelectionDAG &DAG, // of single 'add' instructions, but the cost model for selecting an LEA // currently has a high threshold. bool HasLEAPotential = false; - for (auto *User : Ext->uses()) { + for (auto *User : Ext->users()) { if (User->getOpcode() == ISD::ADD || User->getOpcode() == ISD::SHL) { HasLEAPotential = true; break; @@ -54996,10 +55084,11 @@ static SDValue getInvertedVectorForFMA(SDValue V, SelectionDAG &DAG) { // Check if we can eliminate V. We assume if a value is only used in FMAs, we // can eliminate it. Since this function is invoked for each FMA with this // vector. - auto IsNotFMA = [](SDNode *Use) { - return Use->getOpcode() != ISD::FMA && Use->getOpcode() != ISD::STRICT_FMA; + auto IsNotFMA = [](SDNode *User) { + return User->getOpcode() != ISD::FMA && + User->getOpcode() != ISD::STRICT_FMA; }; - if (llvm::any_of(V->uses(), IsNotFMA)) + if (llvm::any_of(V->users(), IsNotFMA)) return SDValue(); SmallVector<SDValue, 8> Ops; @@ -55020,7 +55109,7 @@ static SDValue getInvertedVectorForFMA(SDValue V, SelectionDAG &DAG) { // If an inverted version cannot be eliminated, choose it instead of the // original version. - if (llvm::any_of(NV->uses(), IsNotFMA)) + if (llvm::any_of(NV->users(), IsNotFMA)) return SDValue(NV, 0); // If the inverted version also can be eliminated, we have to consistently @@ -55042,7 +55131,10 @@ static SDValue combineFMA(SDNode *N, SelectionDAG &DAG, const X86Subtarget &Subtarget) { SDLoc dl(N); EVT VT = N->getValueType(0); - bool IsStrict = N->isStrictFPOpcode() || N->isTargetStrictFPOpcode(); + const SelectionDAGTargetInfo &TSI = DAG.getSelectionDAGInfo(); + bool IsStrict = N->isTargetOpcode() + ? TSI.isTargetStrictFPOpcode(N->getOpcode()) + : N->isStrictFPOpcode(); // Let legalize expand this if it isn't a legal type yet. const TargetLowering &TLI = DAG.getTargetLoweringInfo(); @@ -56113,7 +56205,7 @@ static SDValue combineSIntToFP(SDNode *N, SelectionDAG &DAG, static bool needCarryOrOverflowFlag(SDValue Flags) { assert(Flags.getValueType() == MVT::i32 && "Unexpected VT!"); - for (const SDNode *User : Flags->uses()) { + for (const SDNode *User : Flags->users()) { X86::CondCode CC; switch (User->getOpcode()) { default: @@ -56148,7 +56240,7 @@ static bool needCarryOrOverflowFlag(SDValue Flags) { static bool onlyZeroFlagUsed(SDValue Flags) { assert(Flags.getValueType() == MVT::i32 && "Unexpected VT!"); - for (const SDNode *User : Flags->uses()) { + for (const SDNode *User : Flags->users()) { unsigned CCOpNo; switch (User->getOpcode()) { default: @@ -56351,7 +56443,7 @@ static SDValue combineX86AddSub(SDNode *N, SelectionDAG &DAG, if (Negate) { // Bail if this is only used by a user of the x86 add/sub. if (GenericAddSub->hasOneUse() && - GenericAddSub->use_begin()->isOnlyUserOf(N)) + GenericAddSub->user_begin()->isOnlyUserOf(N)) return; Op = DAG.getNegative(Op, DL, VT); } @@ -56447,9 +56539,11 @@ static SDValue combineADC(SDNode *N, SelectionDAG &DAG, return SDValue(); } -static SDValue matchPMADDWD(SelectionDAG &DAG, SDValue Op0, SDValue Op1, +static SDValue matchPMADDWD(SelectionDAG &DAG, SDNode *N, const SDLoc &DL, EVT VT, const X86Subtarget &Subtarget) { + using namespace SDPatternMatch; + // Example of pattern we try to detect: // t := (v8i32 mul (sext (v8i16 x0), (sext (v8i16 x1)))) //(add (build_vector (extract_elt t, 0), @@ -56464,15 +56558,19 @@ static SDValue matchPMADDWD(SelectionDAG &DAG, SDValue Op0, SDValue Op1, if (!Subtarget.hasSSE2()) return SDValue(); - if (Op0.getOpcode() != ISD::BUILD_VECTOR || - Op1.getOpcode() != ISD::BUILD_VECTOR) - return SDValue(); - if (!VT.isVector() || VT.getVectorElementType() != MVT::i32 || VT.getVectorNumElements() < 4 || !isPowerOf2_32(VT.getVectorNumElements())) return SDValue(); + SDValue Op0, Op1, Accum; + if (!sd_match(N, m_Add(m_AllOf(m_Opc(ISD::BUILD_VECTOR), m_Value(Op0)), + m_AllOf(m_Opc(ISD::BUILD_VECTOR), m_Value(Op1)))) && + !sd_match(N, m_Add(m_AllOf(m_Opc(ISD::BUILD_VECTOR), m_Value(Op0)), + m_Add(m_Value(Accum), m_AllOf(m_Opc(ISD::BUILD_VECTOR), + m_Value(Op1)))))) + return SDValue(); + // Check if one of Op0,Op1 is of the form: // (build_vector (extract_elt Mul, 0), // (extract_elt Mul, 2), @@ -56489,26 +56587,20 @@ static SDValue matchPMADDWD(SelectionDAG &DAG, SDValue Op0, SDValue Op1, SDValue Op0L = Op0->getOperand(i), Op1L = Op1->getOperand(i), Op0H = Op0->getOperand(i + 1), Op1H = Op1->getOperand(i + 1); // TODO: Be more tolerant to undefs. - if (Op0L.getOpcode() != ISD::EXTRACT_VECTOR_ELT || - Op1L.getOpcode() != ISD::EXTRACT_VECTOR_ELT || - Op0H.getOpcode() != ISD::EXTRACT_VECTOR_ELT || - Op1H.getOpcode() != ISD::EXTRACT_VECTOR_ELT) + APInt Idx0L, Idx0H, Idx1L, Idx1H; + SDValue Vec0L, Vec0H, Vec1L, Vec1H; + if (!sd_match(Op0L, m_ExtractElt(m_Value(Vec0L), m_ConstInt(Idx0L))) || + !sd_match(Op0H, m_ExtractElt(m_Value(Vec0H), m_ConstInt(Idx0H))) || + !sd_match(Op1L, m_ExtractElt(m_Value(Vec1L), m_ConstInt(Idx1L))) || + !sd_match(Op1H, m_ExtractElt(m_Value(Vec1H), m_ConstInt(Idx1H)))) return SDValue(); - auto *Const0L = dyn_cast<ConstantSDNode>(Op0L->getOperand(1)); - auto *Const1L = dyn_cast<ConstantSDNode>(Op1L->getOperand(1)); - auto *Const0H = dyn_cast<ConstantSDNode>(Op0H->getOperand(1)); - auto *Const1H = dyn_cast<ConstantSDNode>(Op1H->getOperand(1)); - if (!Const0L || !Const1L || !Const0H || !Const1H) - return SDValue(); - unsigned Idx0L = Const0L->getZExtValue(), Idx1L = Const1L->getZExtValue(), - Idx0H = Const0H->getZExtValue(), Idx1H = Const1H->getZExtValue(); // Commutativity of mul allows factors of a product to reorder. - if (Idx0L > Idx1L) + if (Idx0L.getZExtValue() > Idx1L.getZExtValue()) std::swap(Idx0L, Idx1L); - if (Idx0H > Idx1H) + if (Idx0H.getZExtValue() > Idx1H.getZExtValue()) std::swap(Idx0H, Idx1H); // Commutativity of add allows pairs of factors to reorder. - if (Idx0L > Idx0H) { + if (Idx0L.getZExtValue() > Idx0H.getZExtValue()) { std::swap(Idx0L, Idx0H); std::swap(Idx1L, Idx1H); } @@ -56519,14 +56611,13 @@ static SDValue matchPMADDWD(SelectionDAG &DAG, SDValue Op0, SDValue Op1, // First time an extract_elt's source vector is visited. Must be a MUL // with 2X number of vector elements than the BUILD_VECTOR. // Both extracts must be from same MUL. - Mul = Op0L->getOperand(0); - if (Mul->getOpcode() != ISD::MUL || + Mul = Vec0L; + if (Mul.getOpcode() != ISD::MUL || Mul.getValueType().getVectorNumElements() != 2 * e) return SDValue(); } // Check that the extract is from the same MUL previously seen. - if (Mul != Op0L->getOperand(0) || Mul != Op1L->getOperand(0) || - Mul != Op0H->getOperand(0) || Mul != Op1H->getOperand(0)) + if (Mul != Vec0L || Mul != Vec1L || Mul != Vec0H || Mul != Vec1H) return SDValue(); } @@ -56549,19 +56640,21 @@ static SDValue matchPMADDWD(SelectionDAG &DAG, SDValue Op0, SDValue Op1, InVT.getVectorNumElements() / 2); return DAG.getNode(X86ISD::VPMADDWD, DL, ResVT, Ops[0], Ops[1]); }; - return SplitOpsAndApply(DAG, Subtarget, DL, VT, { N0, N1 }, PMADDBuilder); + SDValue R = SplitOpsAndApply(DAG, Subtarget, DL, VT, {N0, N1}, PMADDBuilder); + if (Accum) + R = DAG.getNode(ISD::ADD, DL, VT, R, Accum); + return R; } // Attempt to turn this pattern into PMADDWD. // (add (mul (sext (build_vector)), (sext (build_vector))), // (mul (sext (build_vector)), (sext (build_vector))) -static SDValue matchPMADDWD_2(SelectionDAG &DAG, SDValue N0, SDValue N1, +static SDValue matchPMADDWD_2(SelectionDAG &DAG, SDNode *N, const SDLoc &DL, EVT VT, const X86Subtarget &Subtarget) { - if (!Subtarget.hasSSE2()) - return SDValue(); + using namespace SDPatternMatch; - if (N0.getOpcode() != ISD::MUL || N1.getOpcode() != ISD::MUL) + if (!Subtarget.hasSSE2()) return SDValue(); if (!VT.isVector() || VT.getVectorElementType() != MVT::i32 || @@ -56569,25 +56662,13 @@ static SDValue matchPMADDWD_2(SelectionDAG &DAG, SDValue N0, SDValue N1, !isPowerOf2_32(VT.getVectorNumElements())) return SDValue(); - SDValue N00 = N0.getOperand(0); - SDValue N01 = N0.getOperand(1); - SDValue N10 = N1.getOperand(0); - SDValue N11 = N1.getOperand(1); - // All inputs need to be sign extends. // TODO: Support ZERO_EXTEND from known positive? - if (N00.getOpcode() != ISD::SIGN_EXTEND || - N01.getOpcode() != ISD::SIGN_EXTEND || - N10.getOpcode() != ISD::SIGN_EXTEND || - N11.getOpcode() != ISD::SIGN_EXTEND) + SDValue N00, N01, N10, N11; + if (!sd_match(N, m_Add(m_Mul(m_SExt(m_Value(N00)), m_SExt(m_Value(N01))), + m_Mul(m_SExt(m_Value(N10)), m_SExt(m_Value(N11)))))) return SDValue(); - // Peek through the extends. - N00 = N00.getOperand(0); - N01 = N01.getOperand(0); - N10 = N10.getOperand(0); - N11 = N11.getOperand(0); - // Must be extending from vXi16. EVT InVT = N00.getValueType(); if (InVT.getVectorElementType() != MVT::i16 || N01.getValueType() != InVT || @@ -56614,34 +56695,22 @@ static SDValue matchPMADDWD_2(SelectionDAG &DAG, SDValue N0, SDValue N1, SDValue N10Elt = N10.getOperand(i); SDValue N11Elt = N11.getOperand(i); // TODO: Be more tolerant to undefs. - if (N00Elt.getOpcode() != ISD::EXTRACT_VECTOR_ELT || - N01Elt.getOpcode() != ISD::EXTRACT_VECTOR_ELT || - N10Elt.getOpcode() != ISD::EXTRACT_VECTOR_ELT || - N11Elt.getOpcode() != ISD::EXTRACT_VECTOR_ELT) + SDValue N00In, N01In, N10In, N11In; + APInt IdxN00, IdxN01, IdxN10, IdxN11; + if (!sd_match(N00Elt, m_ExtractElt(m_Value(N00In), m_ConstInt(IdxN00))) || + !sd_match(N01Elt, m_ExtractElt(m_Value(N01In), m_ConstInt(IdxN01))) || + !sd_match(N10Elt, m_ExtractElt(m_Value(N10In), m_ConstInt(IdxN10))) || + !sd_match(N11Elt, m_ExtractElt(m_Value(N11In), m_ConstInt(IdxN11)))) return SDValue(); - auto *ConstN00Elt = dyn_cast<ConstantSDNode>(N00Elt.getOperand(1)); - auto *ConstN01Elt = dyn_cast<ConstantSDNode>(N01Elt.getOperand(1)); - auto *ConstN10Elt = dyn_cast<ConstantSDNode>(N10Elt.getOperand(1)); - auto *ConstN11Elt = dyn_cast<ConstantSDNode>(N11Elt.getOperand(1)); - if (!ConstN00Elt || !ConstN01Elt || !ConstN10Elt || !ConstN11Elt) - return SDValue(); - unsigned IdxN00 = ConstN00Elt->getZExtValue(); - unsigned IdxN01 = ConstN01Elt->getZExtValue(); - unsigned IdxN10 = ConstN10Elt->getZExtValue(); - unsigned IdxN11 = ConstN11Elt->getZExtValue(); // Add is commutative so indices can be reordered. - if (IdxN00 > IdxN10) { + if (IdxN00.getZExtValue() > IdxN10.getZExtValue()) { std::swap(IdxN00, IdxN10); std::swap(IdxN01, IdxN11); } // N0 indices be the even element. N1 indices must be the next odd element. - if (IdxN00 != 2 * i || IdxN10 != 2 * i + 1 || - IdxN01 != 2 * i || IdxN11 != 2 * i + 1) + if (IdxN00 != 2 * i || IdxN10 != 2 * i + 1 || IdxN01 != 2 * i || + IdxN11 != 2 * i + 1) return SDValue(); - SDValue N00In = N00Elt.getOperand(0); - SDValue N01In = N01Elt.getOperand(0); - SDValue N10In = N10Elt.getOperand(0); - SDValue N11In = N11Elt.getOperand(0); // First time we find an input capture it. if (!In0) { @@ -56782,7 +56851,7 @@ static SDValue pushAddIntoCmovOfConsts(SDNode *N, const SDLoc &DL, // TODO: If target has "slow3OpsLEA", do this even without the trailing memop? if (OtherOp.getOpcode() == ISD::ADD && OtherOp.hasOneUse() && !isa<ConstantSDNode>(OtherOp.getOperand(0)) && - all_of(N->uses(), [&](SDNode *Use) { + all_of(N->users(), [&](SDNode *Use) { auto *MemNode = dyn_cast<MemSDNode>(Use); return MemNode && MemNode->getBasePtr().getNode() == N; })) { @@ -56815,9 +56884,9 @@ static SDValue combineAdd(SDNode *N, SelectionDAG &DAG, if (SDValue Select = pushAddIntoCmovOfConsts(N, DL, DAG, Subtarget)) return Select; - if (SDValue MAdd = matchPMADDWD(DAG, Op0, Op1, DL, VT, Subtarget)) + if (SDValue MAdd = matchPMADDWD(DAG, N, DL, VT, Subtarget)) return MAdd; - if (SDValue MAdd = matchPMADDWD_2(DAG, Op0, Op1, DL, VT, Subtarget)) + if (SDValue MAdd = matchPMADDWD_2(DAG, N, DL, VT, Subtarget)) return MAdd; if (SDValue MAdd = combineAddOfPMADDWD(DAG, Op0, Op1, DL, VT)) return MAdd; @@ -56862,6 +56931,23 @@ static SDValue combineAdd(SDNode *N, SelectionDAG &DAG, } } + // Peephole for 512-bit VPDPBSSD on non-VLX targets. + // TODO: Should this be part of matchPMADDWD/matchPMADDWD_2? + if (Subtarget.hasVNNI() && Subtarget.useAVX512Regs() && VT == MVT::v16i32) { + using namespace SDPatternMatch; + SDValue Accum, Lo0, Lo1, Hi0, Hi1; + if (sd_match(N, m_Add(m_Value(Accum), + m_Node(ISD::CONCAT_VECTORS, + m_BinOp(X86ISD::VPMADDWD, m_Value(Lo0), + m_Value(Lo1)), + m_BinOp(X86ISD::VPMADDWD, m_Value(Hi0), + m_Value(Hi1)))))) { + return DAG.getNode(X86ISD::VPDPWSSD, DL, VT, Accum, + concatSubVectors(Lo0, Hi0, DAG, DL), + concatSubVectors(Lo1, Hi1, DAG, DL)); + } + } + // Fold ADD(ADC(Y,0,W),X) -> ADC(X,Y,W) if (Op0.getOpcode() == X86ISD::ADC && Op0->hasOneUse() && X86::isZeroNode(Op0.getOperand(1))) { @@ -58273,6 +58359,17 @@ static SDValue combineEXTRACT_SUBVECTOR(SDNode *N, SelectionDAG &DAG, return DAG.getNode( InOpcode, DL, VT, extractSubVector(InVec.getOperand(0), IdxVal, DAG, DL, SizeInBits)); + case X86ISD::PSHUFD: + case X86ISD::VPERMILPI: + if (InVec.getOperand(0).hasOneUse()) { + uint64_t M = InVec.getConstantOperandVal(1) & 255; + M = VT.getScalarSizeInBits() < 64 ? M : (M >> IdxVal); + return DAG.getNode(InOpcode, DL, VT, + extractSubVector(InVec.getOperand(0), IdxVal, DAG, + DL, SizeInBits), + DAG.getTargetConstant(M, DL, MVT::i8)); + } + break; case X86ISD::PCMPEQ: case X86ISD::PCMPGT: case X86ISD::UNPCKH: @@ -58410,7 +58507,7 @@ static SDValue combineScalarToVector(SDNode *N, SelectionDAG &DAG, // See if we're broadcasting the scalar value, in which case just reuse that. // Ensure the same SDValue from the SDNode use is being used. if (VT.getScalarType() == Src.getValueType()) - for (SDNode *User : Src->uses()) + for (SDNode *User : Src->users()) if (User->getOpcode() == X86ISD::VBROADCAST && Src == User->getOperand(0)) { unsigned SizeInBits = VT.getFixedSizeInBits(); @@ -58634,11 +58731,30 @@ static SDValue combineEXTEND_VECTOR_INREG(SDNode *N, SelectionDAG &DAG, static SDValue combineKSHIFT(SDNode *N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI) { EVT VT = N->getValueType(0); - + const TargetLowering &TLI = DAG.getTargetLoweringInfo(); if (ISD::isBuildVectorAllZeros(N->getOperand(0).getNode())) return DAG.getConstant(0, SDLoc(N), VT); - const TargetLowering &TLI = DAG.getTargetLoweringInfo(); + // Fold kshiftr(extract_subvector(X,C1),C2) + // --> extract_subvector(kshiftr(X,C1+C2),0) + // Fold kshiftr(kshiftr(X,C1),C2) --> kshiftr(X,C1+C2) + if (N->getOpcode() == X86ISD::KSHIFTR) { + SDLoc DL(N); + if (N->getOperand(0).getOpcode() == ISD::EXTRACT_SUBVECTOR || + N->getOperand(0).getOpcode() == X86ISD::KSHIFTR) { + SDValue Src = N->getOperand(0).getOperand(0); + uint64_t Amt = N->getConstantOperandVal(1) + + N->getOperand(0).getConstantOperandVal(1); + EVT SrcVT = Src.getValueType(); + if (TLI.isTypeLegal(SrcVT) && Amt < SrcVT.getVectorNumElements()) { + SDValue Shift = DAG.getNode(X86ISD::KSHIFTR, DL, SrcVT, Src, + DAG.getTargetConstant(Amt, DL, MVT::i8)); + return DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, VT, Shift, + DAG.getIntPtrConstant(0, DL)); + } + } + } + APInt DemandedElts = APInt::getAllOnes(VT.getVectorNumElements()); if (TLI.SimplifyDemandedVectorElts(SDValue(N, 0), DemandedElts, DCI)) return SDValue(N, 0); @@ -58787,7 +58903,7 @@ static SDValue combineBROADCAST_LOAD(SDNode *N, SelectionDAG &DAG, // Look at other users of our base pointer and try to find a wider broadcast. // The input chain and the size of the memory VT must match. - for (SDNode *User : Ptr->uses()) + for (SDNode *User : Ptr->users()) if (User != N && User->getOpcode() == N->getOpcode() && cast<MemIntrinsicSDNode>(User)->getBasePtr() == Ptr && cast<MemIntrinsicSDNode>(User)->getChain() == Chain && @@ -59324,7 +59440,7 @@ bool X86TargetLowering::IsDesirableToPromoteOp(SDValue Op, EVT &PVT) const { auto IsFoldableRMW = [](SDValue Load, SDValue Op) { if (!Op.hasOneUse()) return false; - SDNode *User = *Op->use_begin(); + SDNode *User = *Op->user_begin(); if (!ISD::isNormalStore(User)) return false; auto *Ld = cast<LoadSDNode>(Load); @@ -59337,7 +59453,7 @@ bool X86TargetLowering::IsDesirableToPromoteOp(SDValue Op, EVT &PVT) const { return false; if (!Op.hasOneUse()) return false; - SDNode *User = *Op->use_begin(); + SDNode *User = *Op->user_begin(); if (User->getOpcode() != ISD::ATOMIC_STORE) return false; auto *Ld = cast<AtomicSDNode>(Load); @@ -59348,7 +59464,7 @@ bool X86TargetLowering::IsDesirableToPromoteOp(SDValue Op, EVT &PVT) const { auto IsFoldableZext = [](SDValue Op) { if (!Op.hasOneUse()) return false; - SDNode *User = *Op->use_begin(); + SDNode *User = *Op->user_begin(); EVT VT = User->getValueType(0); return (User->getOpcode() == ISD::ZERO_EXTEND && (VT == MVT::i32 || VT == MVT::i64)); diff --git a/llvm/lib/Target/X86/X86ISelLowering.h b/llvm/lib/Target/X86/X86ISelLowering.h index 2db25d6dda06..2b7a8eaf249d 100644 --- a/llvm/lib/Target/X86/X86ISelLowering.h +++ b/llvm/lib/Target/X86/X86ISelLowering.h @@ -809,7 +809,8 @@ namespace llvm { CTEST, /// X86 strict FP compare instructions. - STRICT_FCMP = ISD::FIRST_TARGET_STRICTFP_OPCODE, + FIRST_STRICTFP_OPCODE, + STRICT_FCMP = FIRST_STRICTFP_OPCODE, STRICT_FCMPS, // Vector packed double/float comparison. @@ -853,12 +854,11 @@ namespace llvm { /// Floating point max and min. STRICT_FMAX, STRICT_FMIN, - - // WARNING: Only add nodes here if they are strict FP nodes. Non-memory and - // non-strict FP nodes should be above FIRST_TARGET_STRICTFP_OPCODE. + LAST_STRICTFP_OPCODE = STRICT_FMIN, // Compare and swap. - LCMPXCHG_DAG = ISD::FIRST_TARGET_MEMORY_OPCODE, + FIRST_MEMORY_OPCODE, + LCMPXCHG_DAG = FIRST_MEMORY_OPCODE, LCMPXCHG8_DAG, LCMPXCHG16_DAG, LCMPXCHG16_SAVE_RBX_DAG, @@ -979,10 +979,7 @@ namespace llvm { // Conditional load/store instructions CLOAD, CSTORE, - - // WARNING: Do not add anything in the end unless you want the node to - // have memop! In fact, starting from FIRST_TARGET_MEMORY_OPCODE all - // opcodes will be thought as target memory ops! + LAST_MEMORY_OPCODE = CSTORE, }; } // end namespace X86ISD @@ -1079,8 +1076,7 @@ namespace llvm { /// function arguments in the caller parameter area. For X86, aggregates /// that contains are placed at 16-byte boundaries while the rest are at /// 4-byte boundaries. - uint64_t getByValTypeAlignment(Type *Ty, - const DataLayout &DL) const override; + Align getByValTypeAlignment(Type *Ty, const DataLayout &DL) const override; EVT getOptimalMemOpType(const MemOp &Op, const AttributeList &FuncAttributes) const override; diff --git a/llvm/lib/Target/X86/X86ISelLoweringCall.cpp b/llvm/lib/Target/X86/X86ISelLoweringCall.cpp index 8a764de56141..b1c1ab4aa855 100644 --- a/llvm/lib/Target/X86/X86ISelLoweringCall.cpp +++ b/llvm/lib/Target/X86/X86ISelLoweringCall.cpp @@ -262,20 +262,15 @@ static void getMaxByValAlign(Type *Ty, Align &MaxAlign) { /// function arguments in the caller parameter area. For X86, aggregates /// that contain SSE vectors are placed at 16-byte boundaries while the rest /// are at 4-byte boundaries. -uint64_t X86TargetLowering::getByValTypeAlignment(Type *Ty, - const DataLayout &DL) const { - if (Subtarget.is64Bit()) { - // Max of 8 and alignment of type. - Align TyAlign = DL.getABITypeAlign(Ty); - if (TyAlign > 8) - return TyAlign.value(); - return 8; - } +Align X86TargetLowering::getByValTypeAlignment(Type *Ty, + const DataLayout &DL) const { + if (Subtarget.is64Bit()) + return std::max(DL.getABITypeAlign(Ty), Align::Constant<8>()); Align Alignment(4); if (Subtarget.hasSSE1()) getMaxByValAlign(Ty, Alignment); - return Alignment.value(); + return Alignment; } /// It returns EVT::Other if the type should be determined using generic @@ -949,7 +944,7 @@ bool X86TargetLowering::isUsedByReturnOnly(SDNode *N, SDValue &Chain) const { return false; SDValue TCChain = Chain; - SDNode *Copy = *N->use_begin(); + SDNode *Copy = *N->user_begin(); if (Copy->getOpcode() == ISD::CopyToReg) { // If the copy has a glue operand, we conservatively assume it isn't safe to // perform a tail call. @@ -960,7 +955,7 @@ bool X86TargetLowering::isUsedByReturnOnly(SDNode *N, SDValue &Chain) const { return false; bool HasRet = false; - for (const SDNode *U : Copy->uses()) { + for (const SDNode *U : Copy->users()) { if (U->getOpcode() != X86ISD::RET_GLUE) return false; // If we are returning more than one value, we can definitely diff --git a/llvm/lib/Target/X86/X86InstrAVX10.td b/llvm/lib/Target/X86/X86InstrAVX10.td index 0301c07dfb54..cda6998778bc 100644 --- a/llvm/lib/Target/X86/X86InstrAVX10.td +++ b/llvm/lib/Target/X86/X86InstrAVX10.td @@ -403,28 +403,45 @@ multiclass avx10_minmax_scalar<string OpStr, X86VectorVTInfo _, SDNode OpNode, SDNode OpNodeSAE> { let ExeDomain = _.ExeDomain, Predicates = [HasAVX10_2] in { let mayRaiseFPException = 1 in { + let isCodeGenOnly = 1 in { + def rri : AVX512Ii8<0x53, MRMSrcReg, (outs _.FRC:$dst), + (ins _.FRC:$src1, _.FRC:$src2, i32u8imm:$src3), + !strconcat(OpStr, "\t{$src3, $src2, $src1|$src1, $src2, $src3}"), + [(set _.FRC:$dst, (OpNode _.FRC:$src1, _.FRC:$src2, (i32 timm:$src3)))]>, + Sched<[WriteFMAX]>; + + def rmi : AVX512Ii8<0x53, MRMSrcMem, (outs _.FRC:$dst), + (ins _.FRC:$src1, _.ScalarMemOp:$src2, i32u8imm:$src3), + !strconcat(OpStr, "\t{$src3, $src2, $src1|$src1, $src2, $src3}"), + [(set _.FRC:$dst, (OpNode _.FRC:$src1, (_.ScalarLdFrag addr:$src2), + (i32 timm:$src3)))]>, + Sched<[WriteFMAX.Folded, WriteFMAX.ReadAfterFold]>; + } defm rri : AVX512_maskable<0x53, MRMSrcReg, _, (outs VR128X:$dst), - (ins VR128X:$src1, VR128X:$src2, i32u8imm:$src3), - OpStr, "$src3, $src2, $src1", "$src1, $src2, $src3", - (_.VT (OpNode (_.VT _.RC:$src1), (_.VT _.RC:$src2), - (i32 timm:$src3)))>, - Sched<[WriteFMAX]>; + (ins VR128X:$src1, VR128X:$src2, i32u8imm:$src3), + OpStr, "$src3, $src2, $src1", "$src1, $src2, $src3", + (_.VT (OpNode (_.VT _.RC:$src1), (_.VT _.RC:$src2), + (i32 timm:$src3))), + 0, 0, 0, vselect_mask, "", "_Int">, + Sched<[WriteFMAX]>; defm rmi : AVX512_maskable<0x53, MRMSrcMem, _, (outs VR128X:$dst), - (ins VR128X:$src1, _.ScalarMemOp:$src2, i32u8imm:$src3), - OpStr, "$src3, $src2, $src1", "$src1, $src2, $src3", - (_.VT (OpNode (_.VT _.RC:$src1), (_.ScalarIntMemFrags addr:$src2), - (i32 timm:$src3)))>, + (ins VR128X:$src1, _.ScalarMemOp:$src2, i32u8imm:$src3), + OpStr, "$src3, $src2, $src1", "$src1, $src2, $src3", + (_.VT (OpNode (_.VT _.RC:$src1), (_.ScalarIntMemFrags addr:$src2), + (i32 timm:$src3))), + 0, 0, 0, vselect_mask, "", "_Int">, Sched<[WriteFMAX.Folded, WriteFMAX.ReadAfterFold]>; } let Uses = []<Register>, mayRaiseFPException = 0 in defm rrib : AVX512_maskable<0x53, MRMSrcReg, _, (outs VR128X:$dst), - (ins VR128X:$src1, VR128X:$src2, i32u8imm:$src3), - OpStr, "$src3, {sae}, $src2, $src1", - "$src1, $src2, {sae}, $src3", - (_.VT (OpNodeSAE (_.VT _.RC:$src1), (_.VT _.RC:$src2), - (i32 timm:$src3)))>, - Sched<[WriteFMAX]>, EVEX_B; + (ins VR128X:$src1, VR128X:$src2, i32u8imm:$src3), + OpStr, "$src3, {sae}, $src2, $src1", + "$src1, $src2, {sae}, $src3", + (_.VT (OpNodeSAE (_.VT _.RC:$src1), (_.VT _.RC:$src2), + (i32 timm:$src3))), + 0, 0, 0, vselect_mask, "", "_Int">, + Sched<[WriteFMAX]>, EVEX_B; } } diff --git a/llvm/lib/Target/X86/X86InstrAVX512.td b/llvm/lib/Target/X86/X86InstrAVX512.td index 83a2e981ffd7..d6ca4b142afe 100644 --- a/llvm/lib/Target/X86/X86InstrAVX512.td +++ b/llvm/lib/Target/X86/X86InstrAVX512.td @@ -28,19 +28,20 @@ multiclass AVX512_maskable_custom<bits<8> O, Format F, bit IsCommutable = 0, bit IsKCommutable = 0, bit IsKZCommutable = IsCommutable, - string ClobberConstraint = ""> { + string ClobberConstraint = "", + string Suffix = ""> { let isCommutable = IsCommutable, Constraints = ClobberConstraint in - def NAME: AVX512<O, F, Outs, Ins, - OpcodeStr#"\t{"#AttSrcAsm#", $dst|"# - "$dst, "#IntelSrcAsm#"}", - Pattern>; + def Suffix: AVX512<O, F, Outs, Ins, + OpcodeStr#"\t{"#AttSrcAsm#", $dst|"# + "$dst, "#IntelSrcAsm#"}", + Pattern>; // Prefer over VMOV*rrk Pat<> let isCommutable = IsKCommutable in - def NAME#k: AVX512<O, F, Outs, MaskingIns, - OpcodeStr#"\t{"#AttSrcAsm#", $dst {${mask}}|"# - "$dst {${mask}}, "#IntelSrcAsm#"}", - MaskingPattern>, + def k#Suffix: AVX512<O, F, Outs, MaskingIns, + OpcodeStr#"\t{"#AttSrcAsm#", $dst {${mask}}|"# + "$dst {${mask}}, "#IntelSrcAsm#"}", + MaskingPattern>, EVEX_K { // In case of the 3src subclass this is overridden with a let. string Constraints = !if(!eq(ClobberConstraint, ""), MaskingConstraint, @@ -52,10 +53,10 @@ multiclass AVX512_maskable_custom<bits<8> O, Format F, // So, it is Ok to use IsCommutable instead of IsKCommutable. let isCommutable = IsKZCommutable, // Prefer over VMOV*rrkz Pat<> Constraints = ClobberConstraint in - def NAME#kz: AVX512<O, F, Outs, ZeroMaskingIns, - OpcodeStr#"\t{"#AttSrcAsm#", $dst {${mask}} {z}|"# - "$dst {${mask}} {z}, "#IntelSrcAsm#"}", - ZeroMaskingPattern>, + def kz#Suffix: AVX512<O, F, Outs, ZeroMaskingIns, + OpcodeStr#"\t{"#AttSrcAsm#", $dst {${mask}} {z}|"# + "$dst {${mask}} {z}, "#IntelSrcAsm#"}", + ZeroMaskingPattern>, EVEX_KZ; } @@ -72,7 +73,8 @@ multiclass AVX512_maskable_common<bits<8> O, Format F, X86VectorVTInfo _, bit IsCommutable = 0, bit IsKCommutable = 0, bit IsKZCommutable = IsCommutable, - string ClobberConstraint = ""> : + string ClobberConstraint = "", + string Suffix = ""> : AVX512_maskable_custom<O, F, Outs, Ins, MaskingIns, ZeroMaskingIns, OpcodeStr, AttSrcAsm, IntelSrcAsm, [(set _.RC:$dst, RHS)], @@ -80,7 +82,8 @@ multiclass AVX512_maskable_common<bits<8> O, Format F, X86VectorVTInfo _, [(set _.RC:$dst, (Select _.KRCWM:$mask, RHS, _.ImmAllZerosV))], MaskingConstraint, IsCommutable, - IsKCommutable, IsKZCommutable, ClobberConstraint>; + IsKCommutable, IsKZCommutable, ClobberConstraint, + Suffix>; // This multiclass generates the unconditional/non-masking, the masking and // the zero-masking variant of the vector instruction. In the masking case, the @@ -115,23 +118,24 @@ multiclass AVX512_maskable<bits<8> O, Format F, X86VectorVTInfo _, bit IsCommutable = 0, bit IsKCommutable = 0, bit IsKZCommutable = IsCommutable, SDPatternOperator Select = vselect_mask, - string ClobberConstraint = ""> : + string ClobberConstraint = "", + string Suffix = ""> : AVX512_maskable_common<O, F, _, Outs, Ins, !con((ins _.RC:$src0, _.KRCWM:$mask), Ins), !con((ins _.KRCWM:$mask), Ins), OpcodeStr, AttSrcAsm, IntelSrcAsm, RHS, (Select _.KRCWM:$mask, RHS, _.RC:$src0), Select, "$src0 = $dst", IsCommutable, IsKCommutable, - IsKZCommutable, ClobberConstraint>; + IsKZCommutable, ClobberConstraint, Suffix>; // This multiclass generates the unconditional/non-masking, the masking and // the zero-masking variant of the scalar instruction. multiclass AVX512_maskable_scalar<bits<8> O, Format F, X86VectorVTInfo _, dag Outs, dag Ins, string OpcodeStr, string AttSrcAsm, string IntelSrcAsm, - dag RHS> : + dag RHS, string Suffix = ""> : AVX512_maskable<O, F, _, Outs, Ins, OpcodeStr, AttSrcAsm, IntelSrcAsm, - RHS, 0, 0, 0, X86selects_mask>; + RHS, 0, 0, 0, X86selects_mask, "", Suffix>; // Similar to AVX512_maskable but in this case one of the source operands // ($src1) is already tied to $dst so we just use that for the preserved @@ -144,7 +148,7 @@ multiclass AVX512_maskable_3src<bits<8> O, Format F, X86VectorVTInfo _, bit IsCommutable = 0, bit IsKCommutable = 0, SDPatternOperator Select = vselect_mask, - bit MaskOnly = 0> : + bit MaskOnly = 0, string Suffix = ""> : AVX512_maskable_common<O, F, _, Outs, !con((ins _.RC:$src1), NonTiedIns), !con((ins _.RC:$src1, _.KRCWM:$mask), NonTiedIns), @@ -152,7 +156,8 @@ multiclass AVX512_maskable_3src<bits<8> O, Format F, X86VectorVTInfo _, OpcodeStr, AttSrcAsm, IntelSrcAsm, !if(MaskOnly, (null_frag), RHS), (Select _.KRCWM:$mask, RHS, _.RC:$src1), - Select, "", IsCommutable, IsKCommutable>; + Select, "", IsCommutable, IsKCommutable, + IsCommutable, "", Suffix>; // Similar to AVX512_maskable_3src but in this case the input VT for the tied // operand differs from the output VT. This requires a bitconvert on @@ -178,10 +183,10 @@ multiclass AVX512_maskable_3src_scalar<bits<8> O, Format F, X86VectorVTInfo _, dag RHS, bit IsCommutable = 0, bit IsKCommutable = 0, - bit MaskOnly = 0> : + bit MaskOnly = 0, string Suffix = ""> : AVX512_maskable_3src<O, F, _, Outs, NonTiedIns, OpcodeStr, AttSrcAsm, IntelSrcAsm, RHS, IsCommutable, IsKCommutable, - X86selects_mask, MaskOnly>; + X86selects_mask, MaskOnly, Suffix>; multiclass AVX512_maskable_in_asm<bits<8> O, Format F, X86VectorVTInfo _, dag Outs, dag Ins, @@ -215,17 +220,18 @@ multiclass AVX512_maskable_custom_cmp<bits<8> O, Format F, string AttSrcAsm, string IntelSrcAsm, list<dag> Pattern, list<dag> MaskingPattern, - bit IsCommutable = 0> { + bit IsCommutable = 0, + string Suffix = ""> { let isCommutable = IsCommutable in { - def NAME: AVX512<O, F, Outs, Ins, + def Suffix: AVX512<O, F, Outs, Ins, OpcodeStr#"\t{"#AttSrcAsm#", $dst|"# "$dst, "#IntelSrcAsm#"}", Pattern>; - def NAME#k: AVX512<O, F, Outs, MaskingIns, - OpcodeStr#"\t{"#AttSrcAsm#", $dst {${mask}}|"# - "$dst {${mask}}, "#IntelSrcAsm#"}", - MaskingPattern>, EVEX_K; + def k#Suffix: AVX512<O, F, Outs, MaskingIns, + OpcodeStr#"\t{"#AttSrcAsm#", $dst {${mask}}|"# + "$dst {${mask}}, "#IntelSrcAsm#"}", + MaskingPattern>, EVEX_K; } } @@ -235,20 +241,22 @@ multiclass AVX512_maskable_common_cmp<bits<8> O, Format F, X86VectorVTInfo _, string OpcodeStr, string AttSrcAsm, string IntelSrcAsm, dag RHS, dag MaskingRHS, - bit IsCommutable = 0> : + bit IsCommutable = 0, + string Suffix = ""> : AVX512_maskable_custom_cmp<O, F, Outs, Ins, MaskingIns, OpcodeStr, AttSrcAsm, IntelSrcAsm, [(set _.KRC:$dst, RHS)], - [(set _.KRC:$dst, MaskingRHS)], IsCommutable>; + [(set _.KRC:$dst, MaskingRHS)], IsCommutable, Suffix>; multiclass AVX512_maskable_cmp<bits<8> O, Format F, X86VectorVTInfo _, dag Outs, dag Ins, string OpcodeStr, string AttSrcAsm, string IntelSrcAsm, - dag RHS, dag RHS_su, bit IsCommutable = 0> : + dag RHS, dag RHS_su, bit IsCommutable = 0, + string Suffix = ""> : AVX512_maskable_common_cmp<O, F, _, Outs, Ins, !con((ins _.KRCWM:$mask), Ins), OpcodeStr, AttSrcAsm, IntelSrcAsm, RHS, - (and _.KRCWM:$mask, RHS_su), IsCommutable>; + (and _.KRCWM:$mask, RHS_su), IsCommutable, Suffix>; // Used by conversion instructions. multiclass AVX512_maskable_cvt<bits<8> O, Format F, X86VectorVTInfo _, @@ -1937,37 +1945,37 @@ defm VPBLENDMW : blendmask_bw<0x66, "vpblendmw", SchedWriteVarBlend, multiclass avx512_cmp_scalar<X86VectorVTInfo _, SDNode OpNode, SDNode OpNodeSAE, PatFrag OpNode_su, PatFrag OpNodeSAE_su, X86FoldableSchedWrite sched> { - defm rri_Int : AVX512_maskable_cmp<0xC2, MRMSrcReg, _, - (outs _.KRC:$dst), - (ins _.RC:$src1, _.RC:$src2, u8imm:$cc), - "vcmp"#_.Suffix, - "$cc, $src2, $src1", "$src1, $src2, $cc", - (OpNode (_.VT _.RC:$src1), (_.VT _.RC:$src2), timm:$cc), - (OpNode_su (_.VT _.RC:$src1), (_.VT _.RC:$src2), timm:$cc)>, - EVEX, VVVV, VEX_LIG, Sched<[sched]>, SIMD_EXC; + defm rri : AVX512_maskable_cmp<0xC2, MRMSrcReg, _, + (outs _.KRC:$dst), + (ins _.RC:$src1, _.RC:$src2, u8imm:$cc), + "vcmp"#_.Suffix, + "$cc, $src2, $src1", "$src1, $src2, $cc", + (OpNode (_.VT _.RC:$src1), (_.VT _.RC:$src2), timm:$cc), + (OpNode_su (_.VT _.RC:$src1), (_.VT _.RC:$src2), timm:$cc), 0, "_Int">, + EVEX, VVVV, VEX_LIG, Sched<[sched]>, SIMD_EXC; let mayLoad = 1 in - defm rmi_Int : AVX512_maskable_cmp<0xC2, MRMSrcMem, _, - (outs _.KRC:$dst), - (ins _.RC:$src1, _.IntScalarMemOp:$src2, u8imm:$cc), - "vcmp"#_.Suffix, - "$cc, $src2, $src1", "$src1, $src2, $cc", - (OpNode (_.VT _.RC:$src1), (_.ScalarIntMemFrags addr:$src2), - timm:$cc), - (OpNode_su (_.VT _.RC:$src1), (_.ScalarIntMemFrags addr:$src2), - timm:$cc)>, EVEX, VVVV, VEX_LIG, EVEX_CD8<_.EltSize, CD8VT1>, - Sched<[sched.Folded, sched.ReadAfterFold]>, SIMD_EXC; + defm rmi : AVX512_maskable_cmp<0xC2, MRMSrcMem, _, + (outs _.KRC:$dst), + (ins _.RC:$src1, _.IntScalarMemOp:$src2, u8imm:$cc), + "vcmp"#_.Suffix, + "$cc, $src2, $src1", "$src1, $src2, $cc", + (OpNode (_.VT _.RC:$src1), (_.ScalarIntMemFrags addr:$src2), + timm:$cc), + (OpNode_su (_.VT _.RC:$src1), (_.ScalarIntMemFrags addr:$src2), + timm:$cc), 0, "_Int">, EVEX, VVVV, VEX_LIG, EVEX_CD8<_.EltSize, CD8VT1>, + Sched<[sched.Folded, sched.ReadAfterFold]>, SIMD_EXC; let Uses = [MXCSR] in - defm rrib_Int : AVX512_maskable_cmp<0xC2, MRMSrcReg, _, - (outs _.KRC:$dst), - (ins _.RC:$src1, _.RC:$src2, u8imm:$cc), - "vcmp"#_.Suffix, - "$cc, {sae}, $src2, $src1","$src1, $src2, {sae}, $cc", - (OpNodeSAE (_.VT _.RC:$src1), (_.VT _.RC:$src2), - timm:$cc), - (OpNodeSAE_su (_.VT _.RC:$src1), (_.VT _.RC:$src2), - timm:$cc)>, - EVEX, VVVV, VEX_LIG, EVEX_B, Sched<[sched]>; + defm rrib : AVX512_maskable_cmp<0xC2, MRMSrcReg, _, + (outs _.KRC:$dst), + (ins _.RC:$src1, _.RC:$src2, u8imm:$cc), + "vcmp"#_.Suffix, + "$cc, {sae}, $src2, $src1","$src1, $src2, {sae}, $cc", + (OpNodeSAE (_.VT _.RC:$src1), (_.VT _.RC:$src2), + timm:$cc), + (OpNodeSAE_su (_.VT _.RC:$src1), (_.VT _.RC:$src2), + timm:$cc), 0, "_Int">, + EVEX, VVVV, VEX_LIG, EVEX_B, Sched<[sched]>; let isCodeGenOnly = 1 in { let isCommutable = 1 in @@ -5354,17 +5362,17 @@ multiclass avx512_fp_scalar<bits<8> opc, string OpcodeStr,X86VectorVTInfo _, SDPatternOperator OpNode, SDNode VecNode, X86FoldableSchedWrite sched, bit IsCommutable> { let ExeDomain = _.ExeDomain, Uses = [MXCSR], mayRaiseFPException = 1 in { - defm rr_Int : AVX512_maskable_scalar<opc, MRMSrcReg, _, (outs _.RC:$dst), + defm rr : AVX512_maskable_scalar<opc, MRMSrcReg, _, (outs _.RC:$dst), (ins _.RC:$src1, _.RC:$src2), OpcodeStr, "$src2, $src1", "$src1, $src2", - (_.VT (VecNode _.RC:$src1, _.RC:$src2))>, + (_.VT (VecNode _.RC:$src1, _.RC:$src2)), "_Int">, Sched<[sched]>; - defm rm_Int : AVX512_maskable_scalar<opc, MRMSrcMem, _, (outs _.RC:$dst), + defm rm : AVX512_maskable_scalar<opc, MRMSrcMem, _, (outs _.RC:$dst), (ins _.RC:$src1, _.IntScalarMemOp:$src2), OpcodeStr, "$src2, $src1", "$src1, $src2", (_.VT (VecNode _.RC:$src1, - (_.ScalarIntMemFrags addr:$src2)))>, + (_.ScalarIntMemFrags addr:$src2))), "_Int">, Sched<[sched.Folded, sched.ReadAfterFold]>; let isCodeGenOnly = 1, Predicates = [HasAVX512] in { def rr : I< opc, MRMSrcReg, (outs _.FRC:$dst), @@ -5387,28 +5395,28 @@ multiclass avx512_fp_scalar<bits<8> opc, string OpcodeStr,X86VectorVTInfo _, multiclass avx512_fp_scalar_round<bits<8> opc, string OpcodeStr,X86VectorVTInfo _, SDNode VecNode, X86FoldableSchedWrite sched> { let ExeDomain = _.ExeDomain, Uses = [MXCSR] in - defm rrb_Int : AVX512_maskable_scalar<opc, MRMSrcReg, _, (outs _.RC:$dst), + defm rrb : AVX512_maskable_scalar<opc, MRMSrcReg, _, (outs _.RC:$dst), (ins _.RC:$src1, _.RC:$src2, AVX512RC:$rc), OpcodeStr, "$rc, $src2, $src1", "$src1, $src2, $rc", (VecNode (_.VT _.RC:$src1), (_.VT _.RC:$src2), - (i32 timm:$rc))>, + (i32 timm:$rc)), "_Int">, EVEX_B, EVEX_RC, Sched<[sched]>; } multiclass avx512_fp_scalar_sae<bits<8> opc, string OpcodeStr,X86VectorVTInfo _, SDPatternOperator OpNode, SDNode VecNode, SDNode SaeNode, X86FoldableSchedWrite sched, bit IsCommutable> { let ExeDomain = _.ExeDomain in { - defm rr_Int : AVX512_maskable_scalar<opc, MRMSrcReg, _, (outs _.RC:$dst), + defm rr : AVX512_maskable_scalar<opc, MRMSrcReg, _, (outs _.RC:$dst), (ins _.RC:$src1, _.RC:$src2), OpcodeStr, "$src2, $src1", "$src1, $src2", - (_.VT (VecNode _.RC:$src1, _.RC:$src2))>, + (_.VT (VecNode _.RC:$src1, _.RC:$src2)), "_Int">, Sched<[sched]>, SIMD_EXC; - defm rm_Int : AVX512_maskable_scalar<opc, MRMSrcMem, _, (outs _.RC:$dst), + defm rm : AVX512_maskable_scalar<opc, MRMSrcMem, _, (outs _.RC:$dst), (ins _.RC:$src1, _.IntScalarMemOp:$src2), OpcodeStr, "$src2, $src1", "$src1, $src2", (_.VT (VecNode _.RC:$src1, - (_.ScalarIntMemFrags addr:$src2)))>, + (_.ScalarIntMemFrags addr:$src2))), "_Int">, Sched<[sched.Folded, sched.ReadAfterFold]>, SIMD_EXC; let isCodeGenOnly = 1, Predicates = [HasAVX512], @@ -5429,10 +5437,10 @@ multiclass avx512_fp_scalar_sae<bits<8> opc, string OpcodeStr,X86VectorVTInfo _, } let Uses = [MXCSR] in - defm rrb_Int : AVX512_maskable_scalar<opc, MRMSrcReg, _, (outs _.RC:$dst), + defm rrb : AVX512_maskable_scalar<opc, MRMSrcReg, _, (outs _.RC:$dst), (ins _.RC:$src1, _.RC:$src2), OpcodeStr, "{sae}, $src2, $src1", "$src1, $src2, {sae}", - (SaeNode (_.VT _.RC:$src1), (_.VT _.RC:$src2))>, + (SaeNode (_.VT _.RC:$src1), (_.VT _.RC:$src2)), "_Int">, EVEX_B, Sched<[sched]>; } } @@ -6835,22 +6843,22 @@ defm VFNMSUB132 : avx512_fma3p_132_f<0x9E, "vfnmsub132", X86any_Fnmsub, multiclass avx512_fma3s_common<bits<8> opc, string OpcodeStr, X86VectorVTInfo _, dag RHS_r, dag RHS_m, dag RHS_b, bit MaskOnlyReg> { let Constraints = "$src1 = $dst", hasSideEffects = 0 in { - defm r_Int: AVX512_maskable_3src_scalar<opc, MRMSrcReg, _, (outs _.RC:$dst), + defm r: AVX512_maskable_3src_scalar<opc, MRMSrcReg, _, (outs _.RC:$dst), (ins _.RC:$src2, _.RC:$src3), OpcodeStr, - "$src3, $src2", "$src2, $src3", (null_frag), 1, 1>, + "$src3, $src2", "$src2, $src3", (null_frag), 1, 1, 0, "_Int">, EVEX, VVVV, Sched<[SchedWriteFMA.Scl]>, SIMD_EXC; let mayLoad = 1 in - defm m_Int: AVX512_maskable_3src_scalar<opc, MRMSrcMem, _, (outs _.RC:$dst), + defm m: AVX512_maskable_3src_scalar<opc, MRMSrcMem, _, (outs _.RC:$dst), (ins _.RC:$src2, _.IntScalarMemOp:$src3), OpcodeStr, - "$src3, $src2", "$src2, $src3", (null_frag), 1, 1>, + "$src3, $src2", "$src2, $src3", (null_frag), 1, 1, 0, "_Int">, EVEX, VVVV, Sched<[SchedWriteFMA.Scl.Folded, SchedWriteFMA.Scl.ReadAfterFold, SchedWriteFMA.Scl.ReadAfterFold]>, SIMD_EXC; let Uses = [MXCSR] in - defm rb_Int: AVX512_maskable_3src_scalar<opc, MRMSrcReg, _, (outs _.RC:$dst), + defm rb: AVX512_maskable_3src_scalar<opc, MRMSrcReg, _, (outs _.RC:$dst), (ins _.RC:$src2, _.RC:$src3, AVX512RC:$rc), - OpcodeStr, "$rc, $src3, $src2", "$src2, $src3, $rc", (null_frag), 1, 1>, + OpcodeStr, "$rc, $src3, $src2", "$src2, $src3, $rc", (null_frag), 1, 1, 0, "_Int">, EVEX, VVVV, EVEX_B, EVEX_RC, Sched<[SchedWriteFMA.Scl]>; let isCodeGenOnly = 1, isCommutable = 1 in { @@ -6982,7 +6990,7 @@ multiclass avx512_scalar_fma_patterns<SDPatternOperator Op, SDNode MaskedOp, (_.EltVT (extractelt (_.VT VR128X:$src1), (iPTR 0))), _.FRC:$src3), (_.EltVT (extractelt (_.VT VR128X:$src1), (iPTR 0)))))))), - (!cast<I>(Prefix#"213"#Suffix#"Zr_Intk") + (!cast<I>(Prefix#"213"#Suffix#"Zrk_Int") VR128X:$src1, VK1WM:$mask, (_.VT (COPY_TO_REGCLASS _.FRC:$src2, VR128X)), (_.VT (COPY_TO_REGCLASS _.FRC:$src3, VR128X)))>; @@ -6993,7 +7001,7 @@ multiclass avx512_scalar_fma_patterns<SDPatternOperator Op, SDNode MaskedOp, (_.EltVT (extractelt (_.VT VR128X:$src1), (iPTR 0))), (_.ScalarLdFrag addr:$src3)), (_.EltVT (extractelt (_.VT VR128X:$src1), (iPTR 0)))))))), - (!cast<I>(Prefix#"213"#Suffix#"Zm_Intk") + (!cast<I>(Prefix#"213"#Suffix#"Zmk_Int") VR128X:$src1, VK1WM:$mask, (_.VT (COPY_TO_REGCLASS _.FRC:$src2, VR128X)), addr:$src3)>; @@ -7002,7 +7010,7 @@ multiclass avx512_scalar_fma_patterns<SDPatternOperator Op, SDNode MaskedOp, (MaskedOp (_.EltVT (extractelt (_.VT VR128X:$src1), (iPTR 0))), (_.ScalarLdFrag addr:$src3), _.FRC:$src2), (_.EltVT (extractelt (_.VT VR128X:$src1), (iPTR 0)))))))), - (!cast<I>(Prefix#"132"#Suffix#"Zm_Intk") + (!cast<I>(Prefix#"132"#Suffix#"Zmk_Int") VR128X:$src1, VK1WM:$mask, (_.VT (COPY_TO_REGCLASS _.FRC:$src2, VR128X)), addr:$src3)>; @@ -7011,7 +7019,7 @@ multiclass avx512_scalar_fma_patterns<SDPatternOperator Op, SDNode MaskedOp, (MaskedOp _.FRC:$src2, _.FRC:$src3, (_.EltVT (extractelt (_.VT VR128X:$src1), (iPTR 0)))), (_.EltVT (extractelt (_.VT VR128X:$src1), (iPTR 0)))))))), - (!cast<I>(Prefix#"231"#Suffix#"Zr_Intk") + (!cast<I>(Prefix#"231"#Suffix#"Zrk_Int") VR128X:$src1, VK1WM:$mask, (_.VT (COPY_TO_REGCLASS _.FRC:$src2, VR128X)), (_.VT (COPY_TO_REGCLASS _.FRC:$src3, VR128X)))>; @@ -7021,7 +7029,7 @@ multiclass avx512_scalar_fma_patterns<SDPatternOperator Op, SDNode MaskedOp, (MaskedOp _.FRC:$src2, (_.ScalarLdFrag addr:$src3), (_.EltVT (extractelt (_.VT VR128X:$src1), (iPTR 0)))), (_.EltVT (extractelt (_.VT VR128X:$src1), (iPTR 0)))))))), - (!cast<I>(Prefix#"231"#Suffix#"Zm_Intk") + (!cast<I>(Prefix#"231"#Suffix#"Zmk_Int") VR128X:$src1, VK1WM:$mask, (_.VT (COPY_TO_REGCLASS _.FRC:$src2, VR128X)), addr:$src3)>; @@ -7031,7 +7039,7 @@ multiclass avx512_scalar_fma_patterns<SDPatternOperator Op, SDNode MaskedOp, (_.EltVT (extractelt (_.VT VR128X:$src1), (iPTR 0))), _.FRC:$src3), (_.EltVT ZeroFP)))))), - (!cast<I>(Prefix#"213"#Suffix#"Zr_Intkz") + (!cast<I>(Prefix#"213"#Suffix#"Zrkz_Int") VR128X:$src1, VK1WM:$mask, (_.VT (COPY_TO_REGCLASS _.FRC:$src2, VR128X)), (_.VT (COPY_TO_REGCLASS _.FRC:$src3, VR128X)))>; @@ -7041,7 +7049,7 @@ multiclass avx512_scalar_fma_patterns<SDPatternOperator Op, SDNode MaskedOp, (MaskedOp _.FRC:$src2, _.FRC:$src3, (_.EltVT (extractelt (_.VT VR128X:$src1), (iPTR 0)))), (_.EltVT ZeroFP)))))), - (!cast<I>(Prefix#"231"#Suffix#"Zr_Intkz") + (!cast<I>(Prefix#"231"#Suffix#"Zrkz_Int") VR128X:$src1, VK1WM:$mask, (_.VT (COPY_TO_REGCLASS _.FRC:$src2, VR128X)), (_.VT (COPY_TO_REGCLASS _.FRC:$src3, VR128X)))>; @@ -7052,7 +7060,7 @@ multiclass avx512_scalar_fma_patterns<SDPatternOperator Op, SDNode MaskedOp, (_.EltVT (extractelt (_.VT VR128X:$src1), (iPTR 0))), (_.ScalarLdFrag addr:$src3)), (_.EltVT ZeroFP)))))), - (!cast<I>(Prefix#"213"#Suffix#"Zm_Intkz") + (!cast<I>(Prefix#"213"#Suffix#"Zmkz_Int") VR128X:$src1, VK1WM:$mask, (_.VT (COPY_TO_REGCLASS _.FRC:$src2, VR128X)), addr:$src3)>; @@ -7061,7 +7069,7 @@ multiclass avx512_scalar_fma_patterns<SDPatternOperator Op, SDNode MaskedOp, (MaskedOp (_.EltVT (extractelt (_.VT VR128X:$src1), (iPTR 0))), _.FRC:$src2, (_.ScalarLdFrag addr:$src3)), (_.EltVT ZeroFP)))))), - (!cast<I>(Prefix#"132"#Suffix#"Zm_Intkz") + (!cast<I>(Prefix#"132"#Suffix#"Zmkz_Int") VR128X:$src1, VK1WM:$mask, (_.VT (COPY_TO_REGCLASS _.FRC:$src2, VR128X)), addr:$src3)>; @@ -7070,7 +7078,7 @@ multiclass avx512_scalar_fma_patterns<SDPatternOperator Op, SDNode MaskedOp, (MaskedOp _.FRC:$src2, (_.ScalarLdFrag addr:$src3), (_.EltVT (extractelt (_.VT VR128X:$src1), (iPTR 0)))), (_.EltVT ZeroFP)))))), - (!cast<I>(Prefix#"231"#Suffix#"Zm_Intkz") + (!cast<I>(Prefix#"231"#Suffix#"Zmkz_Int") VR128X:$src1, VK1WM:$mask, (_.VT (COPY_TO_REGCLASS _.FRC:$src2, VR128X)), addr:$src3)>; @@ -7097,7 +7105,7 @@ multiclass avx512_scalar_fma_patterns<SDPatternOperator Op, SDNode MaskedOp, (_.EltVT (extractelt (_.VT VR128X:$src1), (iPTR 0))), _.FRC:$src3, (i32 timm:$rc)), (_.EltVT (extractelt (_.VT VR128X:$src1), (iPTR 0)))))))), - (!cast<I>(Prefix#"213"#Suffix#"Zrb_Intk") + (!cast<I>(Prefix#"213"#Suffix#"Zrbk_Int") VR128X:$src1, VK1WM:$mask, (_.VT (COPY_TO_REGCLASS _.FRC:$src2, VR128X)), (_.VT (COPY_TO_REGCLASS _.FRC:$src3, VR128X)), AVX512RC:$rc)>; @@ -7108,7 +7116,7 @@ multiclass avx512_scalar_fma_patterns<SDPatternOperator Op, SDNode MaskedOp, (_.EltVT (extractelt (_.VT VR128X:$src1), (iPTR 0))), (i32 timm:$rc)), (_.EltVT (extractelt (_.VT VR128X:$src1), (iPTR 0)))))))), - (!cast<I>(Prefix#"231"#Suffix#"Zrb_Intk") + (!cast<I>(Prefix#"231"#Suffix#"Zrbk_Int") VR128X:$src1, VK1WM:$mask, (_.VT (COPY_TO_REGCLASS _.FRC:$src2, VR128X)), (_.VT (COPY_TO_REGCLASS _.FRC:$src3, VR128X)), AVX512RC:$rc)>; @@ -7119,7 +7127,7 @@ multiclass avx512_scalar_fma_patterns<SDPatternOperator Op, SDNode MaskedOp, (_.EltVT (extractelt (_.VT VR128X:$src1), (iPTR 0))), _.FRC:$src3, (i32 timm:$rc)), (_.EltVT ZeroFP)))))), - (!cast<I>(Prefix#"213"#Suffix#"Zrb_Intkz") + (!cast<I>(Prefix#"213"#Suffix#"Zrbkz_Int") VR128X:$src1, VK1WM:$mask, (_.VT (COPY_TO_REGCLASS _.FRC:$src2, VR128X)), (_.VT (COPY_TO_REGCLASS _.FRC:$src3, VR128X)), AVX512RC:$rc)>; @@ -7130,7 +7138,7 @@ multiclass avx512_scalar_fma_patterns<SDPatternOperator Op, SDNode MaskedOp, (_.EltVT (extractelt (_.VT VR128X:$src1), (iPTR 0))), (i32 timm:$rc)), (_.EltVT ZeroFP)))))), - (!cast<I>(Prefix#"231"#Suffix#"Zrb_Intkz") + (!cast<I>(Prefix#"231"#Suffix#"Zrbkz_Int") VR128X:$src1, VK1WM:$mask, (_.VT (COPY_TO_REGCLASS _.FRC:$src2, VR128X)), (_.VT (COPY_TO_REGCLASS _.FRC:$src3, VR128X)), AVX512RC:$rc)>; @@ -7628,17 +7636,17 @@ let Uses = [MXCSR], mayRaiseFPException = 1 in multiclass avx512_cvt_fp_scalar<bits<8> opc, string OpcodeStr, X86VectorVTInfo _, X86VectorVTInfo _Src, SDNode OpNode, X86FoldableSchedWrite sched> { - defm rr_Int : AVX512_maskable_scalar<opc, MRMSrcReg, _, (outs _.RC:$dst), + defm rr : AVX512_maskable_scalar<opc, MRMSrcReg, _, (outs _.RC:$dst), (ins _.RC:$src1, _Src.RC:$src2), OpcodeStr, "$src2, $src1", "$src1, $src2", (_.VT (OpNode (_.VT _.RC:$src1), - (_Src.VT _Src.RC:$src2)))>, + (_Src.VT _Src.RC:$src2))), "_Int">, EVEX, VVVV, VEX_LIG, Sched<[sched]>; - defm rm_Int : AVX512_maskable_scalar<opc, MRMSrcMem, _, (outs _.RC:$dst), + defm rm : AVX512_maskable_scalar<opc, MRMSrcMem, _, (outs _.RC:$dst), (ins _.RC:$src1, _Src.IntScalarMemOp:$src2), OpcodeStr, "$src2, $src1", "$src1, $src2", (_.VT (OpNode (_.VT _.RC:$src1), - (_Src.ScalarIntMemFrags addr:$src2)))>, + (_Src.ScalarIntMemFrags addr:$src2))), "_Int">, EVEX, VVVV, VEX_LIG, Sched<[sched.Folded, sched.ReadAfterFold]>; @@ -7660,11 +7668,11 @@ multiclass avx512_cvt_fp_sae_scalar<bits<8> opc, string OpcodeStr, X86VectorVTIn X86VectorVTInfo _Src, SDNode OpNodeSAE, X86FoldableSchedWrite sched> { let Uses = [MXCSR] in - defm rrb_Int : AVX512_maskable_scalar<opc, MRMSrcReg, _, (outs _.RC:$dst), + defm rrb : AVX512_maskable_scalar<opc, MRMSrcReg, _, (outs _.RC:$dst), (ins _.RC:$src1, _Src.RC:$src2), OpcodeStr, "{sae}, $src2, $src1", "$src1, $src2, {sae}", (_.VT (OpNodeSAE (_.VT _.RC:$src1), - (_Src.VT _Src.RC:$src2)))>, + (_Src.VT _Src.RC:$src2))), "_Int">, EVEX, VVVV, VEX_LIG, EVEX_B, Sched<[sched]>; } @@ -7673,11 +7681,11 @@ multiclass avx512_cvt_fp_rc_scalar<bits<8> opc, string OpcodeStr, X86VectorVTInf X86VectorVTInfo _Src, SDNode OpNodeRnd, X86FoldableSchedWrite sched> { let Uses = [MXCSR] in - defm rrb_Int : AVX512_maskable_scalar<opc, MRMSrcReg, _, (outs _.RC:$dst), + defm rrb : AVX512_maskable_scalar<opc, MRMSrcReg, _, (outs _.RC:$dst), (ins _.RC:$src1, _Src.RC:$src2, AVX512RC:$rc), OpcodeStr, "$rc, $src2, $src1", "$src1, $src2, $rc", (_.VT (OpNodeRnd (_.VT _.RC:$src1), - (_Src.VT _Src.RC:$src2), (i32 timm:$rc)))>, + (_Src.VT _Src.RC:$src2), (i32 timm:$rc))), "_Int">, EVEX, VVVV, VEX_LIG, Sched<[sched]>, EVEX_B, EVEX_RC; } @@ -9531,25 +9539,25 @@ multiclass avx512_sqrt_packed_all_round<bits<8> opc, string OpcodeStr, multiclass avx512_sqrt_scalar<bits<8> opc, string OpcodeStr, X86FoldableSchedWrite sched, X86VectorVTInfo _, string Name, Predicate prd = HasAVX512> { let ExeDomain = _.ExeDomain, Predicates = [prd] in { - defm r_Int : AVX512_maskable_scalar<opc, MRMSrcReg, _, (outs _.RC:$dst), + defm r : AVX512_maskable_scalar<opc, MRMSrcReg, _, (outs _.RC:$dst), (ins _.RC:$src1, _.RC:$src2), OpcodeStr, "$src2, $src1", "$src1, $src2", (X86fsqrts (_.VT _.RC:$src1), - (_.VT _.RC:$src2))>, + (_.VT _.RC:$src2)), "_Int">, Sched<[sched]>, SIMD_EXC; - defm m_Int : AVX512_maskable_scalar<opc, MRMSrcMem, _, (outs _.RC:$dst), + defm m : AVX512_maskable_scalar<opc, MRMSrcMem, _, (outs _.RC:$dst), (ins _.RC:$src1, _.IntScalarMemOp:$src2), OpcodeStr, "$src2, $src1", "$src1, $src2", (X86fsqrts (_.VT _.RC:$src1), - (_.ScalarIntMemFrags addr:$src2))>, + (_.ScalarIntMemFrags addr:$src2)), "_Int">, Sched<[sched.Folded, sched.ReadAfterFold]>, SIMD_EXC; let Uses = [MXCSR] in - defm rb_Int : AVX512_maskable_scalar<opc, MRMSrcReg, _, (outs _.RC:$dst), + defm rb : AVX512_maskable_scalar<opc, MRMSrcReg, _, (outs _.RC:$dst), (ins _.RC:$src1, _.RC:$src2, AVX512RC:$rc), OpcodeStr, "$rc, $src2, $src1", "$src1, $src2, $rc", (X86fsqrtRnds (_.VT _.RC:$src1), (_.VT _.RC:$src2), - (i32 timm:$rc))>, + (i32 timm:$rc)), "_Int">, EVEX_B, EVEX_RC, Sched<[sched]>; let isCodeGenOnly = 1, hasSideEffects = 0 in { @@ -9596,27 +9604,27 @@ defm VSQRT : avx512_sqrt_scalar_all<0x51, "vsqrt", SchedWriteFSqrtSizes>, VEX_LI multiclass avx512_rndscale_scalar<bits<8> opc, string OpcodeStr, X86FoldableSchedWrite sched, X86VectorVTInfo _> { let ExeDomain = _.ExeDomain in { - defm rri_Int : AVX512_maskable_scalar<opc, MRMSrcReg, _, (outs _.RC:$dst), + defm rri : AVX512_maskable_scalar<opc, MRMSrcReg, _, (outs _.RC:$dst), (ins _.RC:$src1, _.RC:$src2, i32u8imm:$src3), OpcodeStr, "$src3, $src2, $src1", "$src1, $src2, $src3", (_.VT (X86RndScales (_.VT _.RC:$src1), (_.VT _.RC:$src2), - (i32 timm:$src3)))>, + (i32 timm:$src3))), "_Int">, Sched<[sched]>, SIMD_EXC; let Uses = [MXCSR] in - defm rrib_Int : AVX512_maskable_scalar<opc, MRMSrcReg, _, (outs _.RC:$dst), + defm rrib : AVX512_maskable_scalar<opc, MRMSrcReg, _, (outs _.RC:$dst), (ins _.RC:$src1, _.RC:$src2, i32u8imm:$src3), OpcodeStr, "$src3, {sae}, $src2, $src1", "$src1, $src2, {sae}, $src3", (_.VT (X86RndScalesSAE (_.VT _.RC:$src1), (_.VT _.RC:$src2), - (i32 timm:$src3)))>, EVEX_B, + (i32 timm:$src3))), "_Int">, EVEX_B, Sched<[sched]>; - defm rmi_Int : AVX512_maskable_scalar<opc, MRMSrcMem, _, (outs _.RC:$dst), + defm rmi : AVX512_maskable_scalar<opc, MRMSrcMem, _, (outs _.RC:$dst), (ins _.RC:$src1, _.IntScalarMemOp:$src2, i32u8imm:$src3), OpcodeStr, "$src3, $src2, $src1", "$src1, $src2, $src3", (_.VT (X86RndScales _.RC:$src1, - (_.ScalarIntMemFrags addr:$src2), (i32 timm:$src3)))>, + (_.ScalarIntMemFrags addr:$src2), (i32 timm:$src3))), "_Int">, Sched<[sched.Folded, sched.ReadAfterFold]>, SIMD_EXC; let isCodeGenOnly = 1, hasSideEffects = 0, Predicates = [HasAVX512] in { @@ -9669,13 +9677,13 @@ multiclass avx512_masked_scalar<SDNode OpNode, string OpcPrefix, SDNode Move, def : Pat<(Move _.VT:$src1, (scalar_to_vector (X86selects_mask Mask, (OpNode (extractelt _.VT:$src2, (iPTR 0))), (extractelt _.VT:$dst, (iPTR 0))))), - (!cast<Instruction>("V"#OpcPrefix#r_Intk) + (!cast<Instruction>("V"#OpcPrefix#rk_Int) _.VT:$dst, OutMask, _.VT:$src2, _.VT:$src1)>; def : Pat<(Move _.VT:$src1, (scalar_to_vector (X86selects_mask Mask, (OpNode (extractelt _.VT:$src2, (iPTR 0))), ZeroFP))), - (!cast<Instruction>("V"#OpcPrefix#r_Intkz) + (!cast<Instruction>("V"#OpcPrefix#rkz_Int) OutMask, _.VT:$src2, _.VT:$src1)>; } } @@ -12174,7 +12182,7 @@ multiclass AVX512_scalar_math_fp_patterns<SDPatternOperator Op, SDNode MaskedOp, (extractelt (_.VT VR128X:$src1), (iPTR 0))), _.FRC:$src2), _.FRC:$src0))), - (!cast<Instruction>("V"#OpcPrefix#"Zrr_Intk") + (!cast<Instruction>("V"#OpcPrefix#"Zrrk_Int") (_.VT (COPY_TO_REGCLASS _.FRC:$src0, VR128X)), VK1WM:$mask, _.VT:$src1, (_.VT (COPY_TO_REGCLASS _.FRC:$src2, VR128X)))>; @@ -12185,7 +12193,7 @@ multiclass AVX512_scalar_math_fp_patterns<SDPatternOperator Op, SDNode MaskedOp, (extractelt (_.VT VR128X:$src1), (iPTR 0))), (_.ScalarLdFrag addr:$src2)), _.FRC:$src0))), - (!cast<Instruction>("V"#OpcPrefix#"Zrm_Intk") + (!cast<Instruction>("V"#OpcPrefix#"Zrmk_Int") (_.VT (COPY_TO_REGCLASS _.FRC:$src0, VR128X)), VK1WM:$mask, _.VT:$src1, addr:$src2)>; @@ -12196,7 +12204,7 @@ multiclass AVX512_scalar_math_fp_patterns<SDPatternOperator Op, SDNode MaskedOp, (MaskedOp (_.EltVT (extractelt (_.VT VR128X:$src1), (iPTR 0))), _.FRC:$src2), (_.EltVT ZeroFP)))), - (!cast<I>("V"#OpcPrefix#"Zrr_Intkz") + (!cast<I>("V"#OpcPrefix#"Zrrkz_Int") VK1WM:$mask, _.VT:$src1, (_.VT (COPY_TO_REGCLASS _.FRC:$src2, VR128X)))>; def : Pat<(MoveNode (_.VT VR128X:$src1), @@ -12205,7 +12213,7 @@ multiclass AVX512_scalar_math_fp_patterns<SDPatternOperator Op, SDNode MaskedOp, (MaskedOp (_.EltVT (extractelt (_.VT VR128X:$src1), (iPTR 0))), (_.ScalarLdFrag addr:$src2)), (_.EltVT ZeroFP)))), - (!cast<I>("V"#OpcPrefix#"Zrm_Intkz") VK1WM:$mask, _.VT:$src1, addr:$src2)>; + (!cast<I>("V"#OpcPrefix#"Zrmkz_Int") VK1WM:$mask, _.VT:$src1, addr:$src2)>; } } @@ -12569,10 +12577,10 @@ multiclass GF2P8AFFINE_avx512_common<bits<8> Op, string OpStr, SDNode OpNode, defm VGF2P8AFFINEINVQB : GF2P8AFFINE_avx512_common<0xCF, "vgf2p8affineinvqb", X86GF2P8affineinvqb, SchedWriteVecIMul>, - EVEX, VVVV, EVEX_CD8<8, CD8VF>, REX_W, AVX512AIi8Base; + EVEX, VVVV, EVEX_CD8<64, CD8VF>, REX_W, AVX512AIi8Base; defm VGF2P8AFFINEQB : GF2P8AFFINE_avx512_common<0xCE, "vgf2p8affineqb", X86GF2P8affineqb, SchedWriteVecIMul>, - EVEX, VVVV, EVEX_CD8<8, CD8VF>, REX_W, AVX512AIi8Base; + EVEX, VVVV, EVEX_CD8<64, CD8VF>, REX_W, AVX512AIi8Base; //===----------------------------------------------------------------------===// diff --git a/llvm/lib/Target/X86/X86InstrFMA3Info.cpp b/llvm/lib/Target/X86/X86InstrFMA3Info.cpp index 090ec687d28c..0da4857d6674 100644 --- a/llvm/lib/Target/X86/X86InstrFMA3Info.cpp +++ b/llvm/lib/Target/X86/X86InstrFMA3Info.cpp @@ -27,6 +27,11 @@ using namespace llvm; FMA3GROUP(Name, Suf##k, Attrs | X86InstrFMA3Group::KMergeMasked) \ FMA3GROUP(Name, Suf##kz, Attrs | X86InstrFMA3Group::KZeroMasked) +#define FMA3GROUP_MASKED_INT(Name, Suf, Attrs) \ + FMA3GROUP(Name, Suf##_Int, Attrs) \ + FMA3GROUP(Name, Suf##k_Int, Attrs | X86InstrFMA3Group::KMergeMasked) \ + FMA3GROUP(Name, Suf##kz_Int, Attrs | X86InstrFMA3Group::KZeroMasked) + #define FMA3GROUP_PACKED_WIDTHS_Z(Name, Suf, Attrs) \ FMA3GROUP_MASKED(Name, Suf##Z128m, Attrs) \ FMA3GROUP_MASKED(Name, Suf##Z128r, Attrs) \ @@ -52,9 +57,9 @@ using namespace llvm; #define FMA3GROUP_SCALAR_WIDTHS_Z(Name, Suf, Attrs) \ FMA3GROUP(Name, Suf##Zm, Attrs) \ - FMA3GROUP_MASKED(Name, Suf##Zm_Int, Attrs | X86InstrFMA3Group::Intrinsic) \ + FMA3GROUP_MASKED_INT(Name, Suf##Zm, Attrs | X86InstrFMA3Group::Intrinsic) \ FMA3GROUP(Name, Suf##Zr, Attrs) \ - FMA3GROUP_MASKED(Name, Suf##Zr_Int, Attrs | X86InstrFMA3Group::Intrinsic) \ + FMA3GROUP_MASKED_INT(Name, Suf##Zr, Attrs | X86InstrFMA3Group::Intrinsic) \ #define FMA3GROUP_SCALAR_WIDTHS_ALL(Name, Suf, Attrs) \ FMA3GROUP_SCALAR_WIDTHS_Z(Name, Suf, Attrs) \ @@ -108,11 +113,11 @@ static const X86InstrFMA3Group Groups[] = { #define FMA3GROUP_SCALAR_AVX512_ROUND(Name, Suf, Attrs) \ FMA3GROUP(Name, SDZ##Suf, Attrs) \ - FMA3GROUP_MASKED(Name, SDZ##Suf##_Int, Attrs) \ + FMA3GROUP_MASKED_INT(Name, SDZ##Suf, Attrs) \ FMA3GROUP(Name, SHZ##Suf, Attrs) \ - FMA3GROUP_MASKED(Name, SHZ##Suf##_Int, Attrs) \ + FMA3GROUP_MASKED_INT(Name, SHZ##Suf, Attrs) \ FMA3GROUP(Name, SSZ##Suf, Attrs) \ - FMA3GROUP_MASKED(Name, SSZ##Suf##_Int, Attrs) + FMA3GROUP_MASKED_INT(Name, SSZ##Suf, Attrs) static const X86InstrFMA3Group BroadcastGroups[] = { FMA3GROUP_PACKED_AVX512_ALL(VFMADD, mb, 0) diff --git a/llvm/lib/Target/X86/X86InstrFragments.td b/llvm/lib/Target/X86/X86InstrFragments.td index ba9779cdc335..ea7af893ce10 100644 --- a/llvm/lib/Target/X86/X86InstrFragments.td +++ b/llvm/lib/Target/X86/X86InstrFragments.td @@ -352,7 +352,8 @@ def X86cmpccxadd : SDNode<"X86ISD::CMPCCXADD", SDTX86Cmpccxadd, SDNPMemOperand]>; // Define X86-specific addressing mode. -def addr : ComplexPattern<iPTR, 5, "selectAddr", [], [SDNPWantParent]>; +let WantsParent = true in +def addr : ComplexPattern<iPTR, 5, "selectAddr">; def lea32addr : ComplexPattern<i32, 5, "selectLEAAddr", [add, sub, mul, X86mul_imm, shl, or, xor, frameindex], []>; @@ -378,7 +379,8 @@ def tls64addr : ComplexPattern<i64, 5, "selectTLSADDRAddr", def tls64baseaddr : ComplexPattern<i64, 5, "selectTLSADDRAddr", [tglobaltlsaddr], []>; -def vectoraddr : ComplexPattern<iPTR, 5, "selectVectorAddr", [],[SDNPWantParent]>; +let WantsParent = true in +def vectoraddr : ComplexPattern<iPTR, 5, "selectVectorAddr">; // A relocatable immediate is an operand that can be relocated by the linker to // an immediate, such as a regular symbol in non-PIC code. diff --git a/llvm/lib/Target/X86/X86InstrInfo.cpp b/llvm/lib/Target/X86/X86InstrInfo.cpp index 5a6ea1182ccb..30a5161bbcc5 100644 --- a/llvm/lib/Target/X86/X86InstrInfo.cpp +++ b/llvm/lib/Target/X86/X86InstrInfo.cpp @@ -7646,8 +7646,8 @@ static bool isNonFoldablePartialRegisterLoad(const MachineInstr &LoadMI, case X86::CVTSS2SDrr_Int: case X86::VCVTSS2SDrr_Int: case X86::VCVTSS2SDZrr_Int: - case X86::VCVTSS2SDZrr_Intk: - case X86::VCVTSS2SDZrr_Intkz: + case X86::VCVTSS2SDZrrk_Int: + case X86::VCVTSS2SDZrrkz_Int: case X86::CVTSS2SIrr_Int: case X86::CVTSS2SI64rr_Int: case X86::VCVTSS2SIrr_Int: @@ -7700,21 +7700,21 @@ static bool isNonFoldablePartialRegisterLoad(const MachineInstr &LoadMI, case X86::SUBSSrr_Int: case X86::VSUBSSrr_Int: case X86::VSUBSSZrr_Int: - case X86::VADDSSZrr_Intk: - case X86::VADDSSZrr_Intkz: - case X86::VCMPSSZrri_Intk: - case X86::VDIVSSZrr_Intk: - case X86::VDIVSSZrr_Intkz: - case X86::VMAXSSZrr_Intk: - case X86::VMAXSSZrr_Intkz: - case X86::VMINSSZrr_Intk: - case X86::VMINSSZrr_Intkz: - case X86::VMULSSZrr_Intk: - case X86::VMULSSZrr_Intkz: - case X86::VSQRTSSZr_Intk: - case X86::VSQRTSSZr_Intkz: - case X86::VSUBSSZrr_Intk: - case X86::VSUBSSZrr_Intkz: + case X86::VADDSSZrrk_Int: + case X86::VADDSSZrrkz_Int: + case X86::VCMPSSZrrik_Int: + case X86::VDIVSSZrrk_Int: + case X86::VDIVSSZrrkz_Int: + case X86::VMAXSSZrrk_Int: + case X86::VMAXSSZrrkz_Int: + case X86::VMINSSZrrk_Int: + case X86::VMINSSZrrkz_Int: + case X86::VMULSSZrrk_Int: + case X86::VMULSSZrrkz_Int: + case X86::VSQRTSSZrk_Int: + case X86::VSQRTSSZrkz_Int: + case X86::VSUBSSZrrk_Int: + case X86::VSUBSSZrrkz_Int: case X86::VFMADDSS4rr_Int: case X86::VFNMADDSS4rr_Int: case X86::VFMSUBSS4rr_Int: @@ -7743,30 +7743,30 @@ static bool isNonFoldablePartialRegisterLoad(const MachineInstr &LoadMI, case X86::VFNMSUB213SSZr_Int: case X86::VFMSUB231SSZr_Int: case X86::VFNMSUB231SSZr_Int: - case X86::VFMADD132SSZr_Intk: - case X86::VFNMADD132SSZr_Intk: - case X86::VFMADD213SSZr_Intk: - case X86::VFNMADD213SSZr_Intk: - case X86::VFMADD231SSZr_Intk: - case X86::VFNMADD231SSZr_Intk: - case X86::VFMSUB132SSZr_Intk: - case X86::VFNMSUB132SSZr_Intk: - case X86::VFMSUB213SSZr_Intk: - case X86::VFNMSUB213SSZr_Intk: - case X86::VFMSUB231SSZr_Intk: - case X86::VFNMSUB231SSZr_Intk: - case X86::VFMADD132SSZr_Intkz: - case X86::VFNMADD132SSZr_Intkz: - case X86::VFMADD213SSZr_Intkz: - case X86::VFNMADD213SSZr_Intkz: - case X86::VFMADD231SSZr_Intkz: - case X86::VFNMADD231SSZr_Intkz: - case X86::VFMSUB132SSZr_Intkz: - case X86::VFNMSUB132SSZr_Intkz: - case X86::VFMSUB213SSZr_Intkz: - case X86::VFNMSUB213SSZr_Intkz: - case X86::VFMSUB231SSZr_Intkz: - case X86::VFNMSUB231SSZr_Intkz: + case X86::VFMADD132SSZrk_Int: + case X86::VFNMADD132SSZrk_Int: + case X86::VFMADD213SSZrk_Int: + case X86::VFNMADD213SSZrk_Int: + case X86::VFMADD231SSZrk_Int: + case X86::VFNMADD231SSZrk_Int: + case X86::VFMSUB132SSZrk_Int: + case X86::VFNMSUB132SSZrk_Int: + case X86::VFMSUB213SSZrk_Int: + case X86::VFNMSUB213SSZrk_Int: + case X86::VFMSUB231SSZrk_Int: + case X86::VFNMSUB231SSZrk_Int: + case X86::VFMADD132SSZrkz_Int: + case X86::VFNMADD132SSZrkz_Int: + case X86::VFMADD213SSZrkz_Int: + case X86::VFNMADD213SSZrkz_Int: + case X86::VFMADD231SSZrkz_Int: + case X86::VFNMADD231SSZrkz_Int: + case X86::VFMSUB132SSZrkz_Int: + case X86::VFNMSUB132SSZrkz_Int: + case X86::VFMSUB213SSZrkz_Int: + case X86::VFNMSUB213SSZrkz_Int: + case X86::VFMSUB231SSZrkz_Int: + case X86::VFNMSUB231SSZrkz_Int: case X86::VFIXUPIMMSSZrri: case X86::VFIXUPIMMSSZrrik: case X86::VFIXUPIMMSSZrrikz: @@ -7791,8 +7791,8 @@ static bool isNonFoldablePartialRegisterLoad(const MachineInstr &LoadMI, case X86::VREDUCESSZrrik: case X86::VREDUCESSZrrikz: case X86::VRNDSCALESSZrri_Int: - case X86::VRNDSCALESSZrri_Intk: - case X86::VRNDSCALESSZrri_Intkz: + case X86::VRNDSCALESSZrrik_Int: + case X86::VRNDSCALESSZrrikz_Int: case X86::VRSQRT14SSZrr: case X86::VRSQRT14SSZrrk: case X86::VRSQRT14SSZrrkz: @@ -7819,8 +7819,8 @@ static bool isNonFoldablePartialRegisterLoad(const MachineInstr &LoadMI, case X86::CVTSD2SSrr_Int: case X86::VCVTSD2SSrr_Int: case X86::VCVTSD2SSZrr_Int: - case X86::VCVTSD2SSZrr_Intk: - case X86::VCVTSD2SSZrr_Intkz: + case X86::VCVTSD2SSZrrk_Int: + case X86::VCVTSD2SSZrrkz_Int: case X86::CVTSD2SIrr_Int: case X86::CVTSD2SI64rr_Int: case X86::VCVTSD2SIrr_Int: @@ -7869,21 +7869,21 @@ static bool isNonFoldablePartialRegisterLoad(const MachineInstr &LoadMI, case X86::SUBSDrr_Int: case X86::VSUBSDrr_Int: case X86::VSUBSDZrr_Int: - case X86::VADDSDZrr_Intk: - case X86::VADDSDZrr_Intkz: - case X86::VCMPSDZrri_Intk: - case X86::VDIVSDZrr_Intk: - case X86::VDIVSDZrr_Intkz: - case X86::VMAXSDZrr_Intk: - case X86::VMAXSDZrr_Intkz: - case X86::VMINSDZrr_Intk: - case X86::VMINSDZrr_Intkz: - case X86::VMULSDZrr_Intk: - case X86::VMULSDZrr_Intkz: - case X86::VSQRTSDZr_Intk: - case X86::VSQRTSDZr_Intkz: - case X86::VSUBSDZrr_Intk: - case X86::VSUBSDZrr_Intkz: + case X86::VADDSDZrrk_Int: + case X86::VADDSDZrrkz_Int: + case X86::VCMPSDZrrik_Int: + case X86::VDIVSDZrrk_Int: + case X86::VDIVSDZrrkz_Int: + case X86::VMAXSDZrrk_Int: + case X86::VMAXSDZrrkz_Int: + case X86::VMINSDZrrk_Int: + case X86::VMINSDZrrkz_Int: + case X86::VMULSDZrrk_Int: + case X86::VMULSDZrrkz_Int: + case X86::VSQRTSDZrk_Int: + case X86::VSQRTSDZrkz_Int: + case X86::VSUBSDZrrk_Int: + case X86::VSUBSDZrrkz_Int: case X86::VFMADDSD4rr_Int: case X86::VFNMADDSD4rr_Int: case X86::VFMSUBSD4rr_Int: @@ -7912,30 +7912,30 @@ static bool isNonFoldablePartialRegisterLoad(const MachineInstr &LoadMI, case X86::VFNMSUB213SDZr_Int: case X86::VFMSUB231SDZr_Int: case X86::VFNMSUB231SDZr_Int: - case X86::VFMADD132SDZr_Intk: - case X86::VFNMADD132SDZr_Intk: - case X86::VFMADD213SDZr_Intk: - case X86::VFNMADD213SDZr_Intk: - case X86::VFMADD231SDZr_Intk: - case X86::VFNMADD231SDZr_Intk: - case X86::VFMSUB132SDZr_Intk: - case X86::VFNMSUB132SDZr_Intk: - case X86::VFMSUB213SDZr_Intk: - case X86::VFNMSUB213SDZr_Intk: - case X86::VFMSUB231SDZr_Intk: - case X86::VFNMSUB231SDZr_Intk: - case X86::VFMADD132SDZr_Intkz: - case X86::VFNMADD132SDZr_Intkz: - case X86::VFMADD213SDZr_Intkz: - case X86::VFNMADD213SDZr_Intkz: - case X86::VFMADD231SDZr_Intkz: - case X86::VFNMADD231SDZr_Intkz: - case X86::VFMSUB132SDZr_Intkz: - case X86::VFNMSUB132SDZr_Intkz: - case X86::VFMSUB213SDZr_Intkz: - case X86::VFNMSUB213SDZr_Intkz: - case X86::VFMSUB231SDZr_Intkz: - case X86::VFNMSUB231SDZr_Intkz: + case X86::VFMADD132SDZrk_Int: + case X86::VFNMADD132SDZrk_Int: + case X86::VFMADD213SDZrk_Int: + case X86::VFNMADD213SDZrk_Int: + case X86::VFMADD231SDZrk_Int: + case X86::VFNMADD231SDZrk_Int: + case X86::VFMSUB132SDZrk_Int: + case X86::VFNMSUB132SDZrk_Int: + case X86::VFMSUB213SDZrk_Int: + case X86::VFNMSUB213SDZrk_Int: + case X86::VFMSUB231SDZrk_Int: + case X86::VFNMSUB231SDZrk_Int: + case X86::VFMADD132SDZrkz_Int: + case X86::VFNMADD132SDZrkz_Int: + case X86::VFMADD213SDZrkz_Int: + case X86::VFNMADD213SDZrkz_Int: + case X86::VFMADD231SDZrkz_Int: + case X86::VFNMADD231SDZrkz_Int: + case X86::VFMSUB132SDZrkz_Int: + case X86::VFNMSUB132SDZrkz_Int: + case X86::VFMSUB213SDZrkz_Int: + case X86::VFNMSUB213SDZrkz_Int: + case X86::VFMSUB231SDZrkz_Int: + case X86::VFNMSUB231SDZrkz_Int: case X86::VFIXUPIMMSDZrri: case X86::VFIXUPIMMSDZrrik: case X86::VFIXUPIMMSDZrrikz: @@ -7960,8 +7960,8 @@ static bool isNonFoldablePartialRegisterLoad(const MachineInstr &LoadMI, case X86::VREDUCESDZrrik: case X86::VREDUCESDZrrikz: case X86::VRNDSCALESDZrri_Int: - case X86::VRNDSCALESDZrri_Intk: - case X86::VRNDSCALESDZrri_Intkz: + case X86::VRNDSCALESDZrrik_Int: + case X86::VRNDSCALESDZrrikz_Int: case X86::VRSQRT14SDZrr: case X86::VRSQRT14SDZrrk: case X86::VRSQRT14SDZrrkz: @@ -7989,19 +7989,19 @@ static bool isNonFoldablePartialRegisterLoad(const MachineInstr &LoadMI, case X86::VMINSHZrr_Int: case X86::VMULSHZrr_Int: case X86::VSUBSHZrr_Int: - case X86::VADDSHZrr_Intk: - case X86::VADDSHZrr_Intkz: - case X86::VCMPSHZrri_Intk: - case X86::VDIVSHZrr_Intk: - case X86::VDIVSHZrr_Intkz: - case X86::VMAXSHZrr_Intk: - case X86::VMAXSHZrr_Intkz: - case X86::VMINSHZrr_Intk: - case X86::VMINSHZrr_Intkz: - case X86::VMULSHZrr_Intk: - case X86::VMULSHZrr_Intkz: - case X86::VSUBSHZrr_Intk: - case X86::VSUBSHZrr_Intkz: + case X86::VADDSHZrrk_Int: + case X86::VADDSHZrrkz_Int: + case X86::VCMPSHZrrik_Int: + case X86::VDIVSHZrrk_Int: + case X86::VDIVSHZrrkz_Int: + case X86::VMAXSHZrrk_Int: + case X86::VMAXSHZrrkz_Int: + case X86::VMINSHZrrk_Int: + case X86::VMINSHZrrkz_Int: + case X86::VMULSHZrrk_Int: + case X86::VMULSHZrrkz_Int: + case X86::VSUBSHZrrk_Int: + case X86::VSUBSHZrrkz_Int: case X86::VFMADD132SHZr_Int: case X86::VFNMADD132SHZr_Int: case X86::VFMADD213SHZr_Int: @@ -8014,30 +8014,30 @@ static bool isNonFoldablePartialRegisterLoad(const MachineInstr &LoadMI, case X86::VFNMSUB213SHZr_Int: case X86::VFMSUB231SHZr_Int: case X86::VFNMSUB231SHZr_Int: - case X86::VFMADD132SHZr_Intk: - case X86::VFNMADD132SHZr_Intk: - case X86::VFMADD213SHZr_Intk: - case X86::VFNMADD213SHZr_Intk: - case X86::VFMADD231SHZr_Intk: - case X86::VFNMADD231SHZr_Intk: - case X86::VFMSUB132SHZr_Intk: - case X86::VFNMSUB132SHZr_Intk: - case X86::VFMSUB213SHZr_Intk: - case X86::VFNMSUB213SHZr_Intk: - case X86::VFMSUB231SHZr_Intk: - case X86::VFNMSUB231SHZr_Intk: - case X86::VFMADD132SHZr_Intkz: - case X86::VFNMADD132SHZr_Intkz: - case X86::VFMADD213SHZr_Intkz: - case X86::VFNMADD213SHZr_Intkz: - case X86::VFMADD231SHZr_Intkz: - case X86::VFNMADD231SHZr_Intkz: - case X86::VFMSUB132SHZr_Intkz: - case X86::VFNMSUB132SHZr_Intkz: - case X86::VFMSUB213SHZr_Intkz: - case X86::VFNMSUB213SHZr_Intkz: - case X86::VFMSUB231SHZr_Intkz: - case X86::VFNMSUB231SHZr_Intkz: + case X86::VFMADD132SHZrk_Int: + case X86::VFNMADD132SHZrk_Int: + case X86::VFMADD213SHZrk_Int: + case X86::VFNMADD213SHZrk_Int: + case X86::VFMADD231SHZrk_Int: + case X86::VFNMADD231SHZrk_Int: + case X86::VFMSUB132SHZrk_Int: + case X86::VFNMSUB132SHZrk_Int: + case X86::VFMSUB213SHZrk_Int: + case X86::VFNMSUB213SHZrk_Int: + case X86::VFMSUB231SHZrk_Int: + case X86::VFNMSUB231SHZrk_Int: + case X86::VFMADD132SHZrkz_Int: + case X86::VFNMADD132SHZrkz_Int: + case X86::VFMADD213SHZrkz_Int: + case X86::VFNMADD213SHZrkz_Int: + case X86::VFMADD231SHZrkz_Int: + case X86::VFNMADD231SHZrkz_Int: + case X86::VFMSUB132SHZrkz_Int: + case X86::VFNMSUB132SHZrkz_Int: + case X86::VFMSUB213SHZrkz_Int: + case X86::VFNMSUB213SHZrkz_Int: + case X86::VFMSUB231SHZrkz_Int: + case X86::VFNMSUB231SHZrkz_Int: return false; default: return true; @@ -9489,25 +9489,25 @@ bool X86InstrInfo::isHighLatencyDef(int opc) const { case X86::VDIVSDZrm: case X86::VDIVSDZrr: case X86::VDIVSDZrm_Int: - case X86::VDIVSDZrm_Intk: - case X86::VDIVSDZrm_Intkz: + case X86::VDIVSDZrmk_Int: + case X86::VDIVSDZrmkz_Int: case X86::VDIVSDZrr_Int: - case X86::VDIVSDZrr_Intk: - case X86::VDIVSDZrr_Intkz: + case X86::VDIVSDZrrk_Int: + case X86::VDIVSDZrrkz_Int: case X86::VDIVSDZrrb_Int: - case X86::VDIVSDZrrb_Intk: - case X86::VDIVSDZrrb_Intkz: + case X86::VDIVSDZrrbk_Int: + case X86::VDIVSDZrrbkz_Int: case X86::VDIVSSZrm: case X86::VDIVSSZrr: case X86::VDIVSSZrm_Int: - case X86::VDIVSSZrm_Intk: - case X86::VDIVSSZrm_Intkz: + case X86::VDIVSSZrmk_Int: + case X86::VDIVSSZrmkz_Int: case X86::VDIVSSZrr_Int: - case X86::VDIVSSZrr_Intk: - case X86::VDIVSSZrr_Intkz: + case X86::VDIVSSZrrk_Int: + case X86::VDIVSSZrrkz_Int: case X86::VDIVSSZrrb_Int: - case X86::VDIVSSZrrb_Intk: - case X86::VDIVSSZrrb_Intkz: + case X86::VDIVSSZrrbk_Int: + case X86::VDIVSSZrrbkz_Int: case X86::VSQRTPDZ128m: case X86::VSQRTPDZ128mb: case X86::VSQRTPDZ128mbk: @@ -9570,26 +9570,26 @@ bool X86InstrInfo::isHighLatencyDef(int opc) const { case X86::VSQRTPSZrkz: case X86::VSQRTSDZm: case X86::VSQRTSDZm_Int: - case X86::VSQRTSDZm_Intk: - case X86::VSQRTSDZm_Intkz: + case X86::VSQRTSDZmk_Int: + case X86::VSQRTSDZmkz_Int: case X86::VSQRTSDZr: case X86::VSQRTSDZr_Int: - case X86::VSQRTSDZr_Intk: - case X86::VSQRTSDZr_Intkz: + case X86::VSQRTSDZrk_Int: + case X86::VSQRTSDZrkz_Int: case X86::VSQRTSDZrb_Int: - case X86::VSQRTSDZrb_Intk: - case X86::VSQRTSDZrb_Intkz: + case X86::VSQRTSDZrbk_Int: + case X86::VSQRTSDZrbkz_Int: case X86::VSQRTSSZm: case X86::VSQRTSSZm_Int: - case X86::VSQRTSSZm_Intk: - case X86::VSQRTSSZm_Intkz: + case X86::VSQRTSSZmk_Int: + case X86::VSQRTSSZmkz_Int: case X86::VSQRTSSZr: case X86::VSQRTSSZr_Int: - case X86::VSQRTSSZr_Intk: - case X86::VSQRTSSZr_Intkz: + case X86::VSQRTSSZrk_Int: + case X86::VSQRTSSZrkz_Int: case X86::VSQRTSSZrb_Int: - case X86::VSQRTSSZrb_Intk: - case X86::VSQRTSSZrb_Intkz: + case X86::VSQRTSSZrbk_Int: + case X86::VSQRTSSZrbkz_Int: case X86::VGATHERDPDYrm: case X86::VGATHERDPDZ128rm: diff --git a/llvm/lib/Target/X86/X86LoadValueInjectionRetHardening.cpp b/llvm/lib/Target/X86/X86LoadValueInjectionRetHardening.cpp index 3b370d8c3eb1..64728a20bd07 100644 --- a/llvm/lib/Target/X86/X86LoadValueInjectionRetHardening.cpp +++ b/llvm/lib/Target/X86/X86LoadValueInjectionRetHardening.cpp @@ -57,8 +57,6 @@ char X86LoadValueInjectionRetHardeningPass::ID = 0; bool X86LoadValueInjectionRetHardeningPass::runOnMachineFunction( MachineFunction &MF) { - LLVM_DEBUG(dbgs() << "***** " << getPassName() << " : " << MF.getName() - << " *****\n"); const X86Subtarget *Subtarget = &MF.getSubtarget<X86Subtarget>(); if (!Subtarget->useLVIControlFlowIntegrity() || !Subtarget->is64Bit()) return false; // FIXME: support 32-bit @@ -68,6 +66,8 @@ bool X86LoadValueInjectionRetHardeningPass::runOnMachineFunction( if (!F.hasOptNone() && skipFunction(F)) return false; + LLVM_DEBUG(dbgs() << "***** " << getPassName() << " : " << MF.getName() + << " *****\n"); ++NumFunctionsConsidered; const X86RegisterInfo *TRI = Subtarget->getRegisterInfo(); const X86InstrInfo *TII = Subtarget->getInstrInfo(); diff --git a/llvm/lib/Target/X86/X86LowerAMXType.cpp b/llvm/lib/Target/X86/X86LowerAMXType.cpp index b484b9334e9b..53091e7bb4db 100644 --- a/llvm/lib/Target/X86/X86LowerAMXType.cpp +++ b/llvm/lib/Target/X86/X86LowerAMXType.cpp @@ -1407,9 +1407,7 @@ class X86LowerAMXTypeLegacyPass : public FunctionPass { public: static char ID; - X86LowerAMXTypeLegacyPass() : FunctionPass(ID) { - initializeX86LowerAMXTypeLegacyPassPass(*PassRegistry::getPassRegistry()); - } + X86LowerAMXTypeLegacyPass() : FunctionPass(ID) {} bool runOnFunction(Function &F) override { // Performance optimization: most code doesn't use AMX, so return early if diff --git a/llvm/lib/Target/X86/X86RegisterInfo.cpp b/llvm/lib/Target/X86/X86RegisterInfo.cpp index 50db211c99d8..164d42059551 100644 --- a/llvm/lib/Target/X86/X86RegisterInfo.cpp +++ b/llvm/lib/Target/X86/X86RegisterInfo.cpp @@ -123,7 +123,8 @@ X86RegisterInfo::getLargestLegalSuperClass(const TargetRegisterClass *RC, const X86Subtarget &Subtarget = MF.getSubtarget<X86Subtarget>(); const TargetRegisterClass *Super = RC; - TargetRegisterClass::sc_iterator I = RC->getSuperClasses(); + auto I = RC->superclasses().begin(); + auto E = RC->superclasses().end(); do { switch (Super->getID()) { case X86::FR32RegClassID: @@ -172,7 +173,12 @@ X86RegisterInfo::getLargestLegalSuperClass(const TargetRegisterClass *RC, if (getRegSizeInBits(*Super) == getRegSizeInBits(*RC)) return Super; } - Super = *I++; + if (I != E) { + Super = getRegClass(*I); + ++I; + } else { + Super = nullptr; + } } while (Super); return RC; } diff --git a/llvm/lib/Target/X86/X86RegisterInfo.td b/llvm/lib/Target/X86/X86RegisterInfo.td index f93f920b6aec..48459b3aca50 100644 --- a/llvm/lib/Target/X86/X86RegisterInfo.td +++ b/llvm/lib/Target/X86/X86RegisterInfo.td @@ -541,8 +541,8 @@ def SSP : X86Reg<"ssp", 0>; // R12, R13, R14, and R15 for X86-64) are callee-save registers. // In 64-mode, there are 12 additional i8 registers, SIL, DIL, BPL, SPL, and // R8B, ... R15B. -// Allocate R12 and R13 last, as these require an extra byte when -// encoded in x86_64 instructions. +// Allocate R12, R13, R20, R21, R28 and R29 last, as these require an extra byte +// when encoded in x86_64 instructions. // FIXME: Allow AH, CH, DH, BH to be used as general-purpose registers in // 64-bit mode. The main complication is that they cannot be encoded in an // instruction requiring a REX prefix, while SIL, DIL, BPL, R8D, etc. @@ -550,9 +550,9 @@ def SSP : X86Reg<"ssp", 0>; // cannot be encoded. def GR8 : RegisterClass<"X86", [i8], 8, (add AL, CL, DL, AH, CH, DH, BL, BH, SIL, DIL, BPL, SPL, - R8B, R9B, R10B, R11B, R16B, R17B, R18B, R19B, R20B, - R21B, R22B, R23B, R24B, R25B, R26B, R27B, R28B, R29B, - R30B, R31B, R14B, R15B, R12B, R13B)> { + R8B, R9B, R10B, R11B, R16B, R17B, R18B, R19B, R22B, + R23B, R24B, R25B, R26B, R27B, R30B, R31B, R14B, + R15B, R12B, R13B, R20B, R21B, R28B, R29B)> { let AltOrders = [(sub GR8, AH, BH, CH, DH)]; let AltOrderSelect = [{ return MF.getSubtarget<X86Subtarget>().is64Bit(); @@ -567,9 +567,9 @@ def GRH8 : RegisterClass<"X86", [i8], 8, R26BH, R27BH, R28BH, R29BH, R30BH, R31BH)>; def GR16 : RegisterClass<"X86", [i16], 16, (add AX, CX, DX, SI, DI, BX, BP, SP, R8W, R9W, R10W, - R11W, R16W, R17W, R18W, R19W, R20W, R21W, R22W, R23W, - R24W, R25W, R26W, R27W, R28W, R29W, R30W, R31W, R14W, - R15W, R12W, R13W)>; + R11W, R16W, R17W, R18W, R19W, R22W, R23W, R24W, + R25W, R26W, R27W, R30W, R31W, R14W, R15W, R12W, + R13W, R20W, R21W, R28W, R29W)>; let isAllocatable = 0 in def GRH16 : RegisterClass<"X86", [i16], 16, @@ -579,9 +579,9 @@ def GRH16 : RegisterClass<"X86", [i16], 16, R25WH, R26WH, R27WH, R28WH, R29WH, R30WH, R31WH)>; def GR32 : RegisterClass<"X86", [i32], 32, (add EAX, ECX, EDX, ESI, EDI, EBX, EBP, ESP, R8D, R9D, - R10D, R11D, R16D, R17D, R18D, R19D, R20D, R21D, R22D, - R23D, R24D, R25D, R26D, R27D, R28D, R29D, R30D, R31D, - R14D, R15D, R12D, R13D)>; + R10D, R11D, R16D, R17D, R18D, R19D, R22D, R23D, + R24D, R25D, R26D, R27D, R30D, R31D, R14D, R15D, + R12D, R13D, R20D, R21D, R28D, R29D)>; // GR64 - 64-bit GPRs. This oddly includes RIP, which isn't accurate, since // RIP isn't really a register and it can't be used anywhere except in an @@ -590,8 +590,8 @@ def GR32 : RegisterClass<"X86", [i32], 32, // tests because of the inclusion of RIP in this register class. def GR64 : RegisterClass<"X86", [i64], 64, (add RAX, RCX, RDX, RSI, RDI, R8, R9, R10, R11, R16, R17, - R18, R19, R20, R21, R22, R23, R24, R25, R26, R27, R28, R29, - R30, R31, RBX, R14, R15, R12, R13, RBP, RSP, RIP)>; + R18, R19, R22, R23, R24, R25, R26, R27, R30, R31, RBX, + R14, R15, R12, R13, R20, R21, R28, R29, RBP, RSP, RIP)>; // GR64PLTSafe - 64-bit GPRs without R10, R11, RSP and RIP. Could be used when // emitting code for intrinsics, which use implict input registers. diff --git a/llvm/lib/Target/X86/X86SchedIceLake.td b/llvm/lib/Target/X86/X86SchedIceLake.td index 60e14ae593f8..1c57093c8979 100644 --- a/llvm/lib/Target/X86/X86SchedIceLake.td +++ b/llvm/lib/Target/X86/X86SchedIceLake.td @@ -429,16 +429,14 @@ defm : ICXWriteResPair<WriteVarVecShiftY, [ICXPort01], 1, [1], 1, 7>; defm : ICXWriteResPair<WriteVarVecShiftZ, [ICXPort0], 1, [1], 1, 7>; // Vector insert/extract operations. -def : WriteRes<WriteVecInsert, [ICXPort5]> { +def : WriteRes<WriteVecInsert, [ICXPort15,ICXPort5]> { let Latency = 2; let NumMicroOps = 2; - let ReleaseAtCycles = [2]; } -def : WriteRes<WriteVecInsertLd, [ICXPort5,ICXPort23]> { +def : WriteRes<WriteVecInsertLd, [ICXPort15,ICXPort23]> { let Latency = 6; let NumMicroOps = 2; } -def: InstRW<[WriteVecInsertLd], (instregex "(V?)MOV(H|L)(PD|PS)rm")>; def : WriteRes<WriteVecExtract, [ICXPort0,ICXPort15]> { let Latency = 3; @@ -1297,7 +1295,8 @@ def ICXWriteResGroup92 : SchedWriteRes<[ICXPort5,ICXPort23]> { let ReleaseAtCycles = [1,1]; } def: InstRW<[ICXWriteResGroup92], (instregex "VMOV(SD|SS)Zrm(b?)", - "VPBROADCAST(B|W)(Z128)?rm", + "VPBROADCAST(B|W)(Z128)?rm", + "(V?)MOV(H|L)(PD|PS)rm", "(V?)INSERTPS(Z?)rm", "(V?)PALIGNR(Z128)?rmi", "(V?)PERMIL(PD|PS)(Z128)?m(b?)i", diff --git a/llvm/lib/Target/X86/X86SchedSapphireRapids.td b/llvm/lib/Target/X86/X86SchedSapphireRapids.td index e04ff68d278b..4f0d3669a311 100644 --- a/llvm/lib/Target/X86/X86SchedSapphireRapids.td +++ b/llvm/lib/Target/X86/X86SchedSapphireRapids.td @@ -669,7 +669,7 @@ def : InstRW<[SPRWriteResGroup12], (instregex "^ADD_F(P?)rST0$", "^VALIGN(D|Q)Z256rri((k|kz)?)$", "^VCMPP(D|H|S)Z(128|256)rri(k?)$", "^VCMPS(D|H|S)Zrri$", - "^VCMPS(D|H|S)Zrr(b?)i_Int(k?)$", + "^VCMPS(D|H|S)Zrr(b?)i(k?)_Int$", "^VFPCLASSP(D|H|S)Z(128|256)ri(k?)$", "^VFPCLASSS(D|H|S)Zri(k?)$", "^VPACK(S|U)S(DW|WB)Yrr$", @@ -977,7 +977,7 @@ def SPRWriteResGroup49 : SchedWriteRes<[SPRPort00, SPRPort02_03_10]> { let NumMicroOps = 2; } def : InstRW<[SPRWriteResGroup49], (instregex "^DIV_F(32|64)m$")>; -def : InstRW<[SPRWriteResGroup49, ReadAfterVecLd], (instregex "^VSQRTSHZm_Int((k|kz)?)$")>; +def : InstRW<[SPRWriteResGroup49, ReadAfterVecLd], (instregex "^VSQRTSHZm((k|kz)?)_Int$")>; def : InstRW<[SPRWriteResGroup49, ReadAfterVecLd], (instrs VSQRTSHZm)>; def SPRWriteResGroup50 : SchedWriteRes<[SPRPort00, SPRPort02_03_10, SPRPort05]> { @@ -1166,11 +1166,11 @@ def : InstRW<[SPRWriteResGroup73, ReadAfterVecXLd], (instregex "^(V?)GF2P8AFFINE def : InstRW<[SPRWriteResGroup73, ReadAfterVecXLd], (instrs VGETEXPPHZ128mbkz, VGF2P8MULBZ128rm)>; def : InstRW<[SPRWriteResGroup73, ReadAfterVecLd], (instregex "^V(ADD|SUB)SHZrm$", - "^V(ADD|SUB)SHZrm_Int((k|kz)?)$", + "^V(ADD|SUB)SHZrm((k|kz)?)_Int$", "^VCVTSH2SSZrm((_Int)?)$", "^VM(AX|IN)CSHZrm$", "^VM(AX|IN|UL)SHZrm$", - "^VM(AX|IN|UL)SHZrm_Int((k|kz)?)$")>; + "^VM(AX|IN|UL)SHZrm((k|kz)?)_Int$")>; def : InstRW<[SPRWriteResGroup73, ReadAfterVecYLd], (instregex "^VGF2P8AFFINE((INV)?)QBYrmi$", "^VGF2P8AFFINE((INV)?)QBZ256rm(b?)i$", "^VGF2P8MULB(Y|Z256)rm$")>; @@ -1181,7 +1181,7 @@ def : InstRW<[SPRWriteResGroup73, ReadAfterVecXLd, ReadAfterVecXLd], (instregex "^VFMSUBADD(132|213|231)PHZ128m((b|k|bk|kz)?)$", "^VFMSUBADD(132|213|231)PHZ128mbkz$")>; def : InstRW<[SPRWriteResGroup73, ReadAfterVecLd, ReadAfterVecLd], (instregex "^VF(N?)M(ADD|SUB)(132|213|231)SHZm$", - "^VF(N?)M(ADD|SUB)(132|213|231)SHZm_Int((k|kz)?)$")>; + "^VF(N?)M(ADD|SUB)(132|213|231)SHZm((k|kz)?)_Int$")>; def : InstRW<[SPRWriteResGroup73, ReadAfterVecYLd, ReadAfterVecYLd], (instregex "^VPMADD52(H|L)UQZ256m((b|k|bk|kz)?)$", "^VPMADD52(H|L)UQZ256mbkz$")>; @@ -2301,7 +2301,7 @@ def : InstRW<[SPRWriteResGroup218, ReadAfterVecXLd], (instregex "^(V?)ROUNDS(D|S "^VRNDSCALEP(D|S)Z128rmbik(z?)$", "^VRNDSCALEP(D|S)Z128rmi((kz)?)$", "^VRNDSCALES(D|S)Zrmi$", - "^VRNDSCALES(D|S)Zrmi_Int((k|kz)?)$")>; + "^VRNDSCALES(D|S)Zrmi((k|kz)?)_Int$")>; def SPRWriteResGroup219 : SchedWriteRes<[SPRPort00_01]> { let ReleaseAtCycles = [2]; @@ -2313,7 +2313,7 @@ def : InstRW<[SPRWriteResGroup219], (instregex "^(V?)ROUND(PD|SS)ri$", "^(V?)ROUNDS(D|S)ri_Int$", "^VRNDSCALEP(D|S)Z(128|256)rri((k|kz)?)$", "^VRNDSCALES(D|S)Zrri$", - "^VRNDSCALES(D|S)Zrri(b?)_Int((k|kz)?)$", + "^VRNDSCALES(D|S)Zrri(b?)((k|kz)?)_Int$", "^VROUNDP(D|S)Yri$")>; def SPRWriteResGroup220 : SchedWriteRes<[SPRPort00_06]> { @@ -2530,7 +2530,7 @@ def SPRWriteResGroup249 : SchedWriteRes<[SPRPort01_05]> { let Latency = 4; } def : InstRW<[SPRWriteResGroup249], (instregex "^V(ADD|SUB)P(D|S)Z(128|256)rrkz$", - "^V(ADD|SUB)S(D|S)Zrr(b?)_Intkz$")>; + "^V(ADD|SUB)S(D|S)Zrr(b?)kz_Int$")>; def SPRWriteResGroup250 : SchedWriteRes<[SPRPort00_05]> { let Latency = 3; @@ -2545,11 +2545,11 @@ def SPRWriteResGroup251 : SchedWriteRes<[SPRPort00_01]> { let Latency = 6; } def : InstRW<[SPRWriteResGroup251], (instregex "^V(ADD|SUB)PHZ(128|256)rrk(z?)$", - "^V(ADD|SUB)SHZrr(b?)_Intk(z?)$", + "^V(ADD|SUB)SHZrr(b?)k(z?)_Int$", "^VCVT(T?)PH2(U?)WZ(128|256)rrk(z?)$", "^VCVT(U?)W2PHZ(128|256)rrk(z?)$", "^VF(N?)M(ADD|SUB)(132|213|231)PHZ(128|256)rk(z?)$", - "^VF(N?)M(ADD|SUB)(132|213|231)SHZr(b?)_Intk(z?)$", + "^VF(N?)M(ADD|SUB)(132|213|231)SHZr(b?)k(z?)_Int$", "^VFMADDSUB(132|213|231)PHZ(128|256)rk(z?)$", "^VFMSUBADD(132|213|231)PHZ(128|256)rk(z?)$", "^VGETEXPPHZ(128|256)rk(z?)$", @@ -2560,7 +2560,7 @@ def : InstRW<[SPRWriteResGroup251], (instregex "^V(ADD|SUB)PHZ(128|256)rrk(z?)$" "^VGETMANTSHZrri(k|bkz)$", "^VM(AX|IN)CPHZ(128|256)rrk(z?)$", "^VM(AX|IN|UL)PHZ(128|256)rrk(z?)$", - "^VM(AX|IN|UL)SHZrr(b?)_Intk(z?)$")>; + "^VM(AX|IN|UL)SHZrr(b?)k(z?)_Int$")>; def SPRWriteResGroup252 : SchedWriteRes<[SPRPort00]> { let Latency = 5; @@ -2745,7 +2745,7 @@ def : InstRW<[SPRWriteResGroup263, ReadAfterVecYLd], (instregex "^VCMPP(D|H|S)Z( "^VPTEST(N?)M(B|D|Q|W)Z((256)?)rm(k?)$", "^VPTEST(N?)M(D|Q)Z((256)?)rmb(k?)$")>; def : InstRW<[SPRWriteResGroup263, ReadAfterVecLd], (instregex "^VCMPS(D|H|S)Zrmi$", - "^VCMPS(D|H|S)Zrmi_Int(k?)$", + "^VCMPS(D|H|S)Zrmi(k?)_Int$", "^VFPCLASSS(D|H|S)Zmik$")>; def SPRWriteResGroup264 : SchedWriteRes<[SPRPort00, SPRPort02_03_10]> { @@ -3171,7 +3171,7 @@ def : InstRW<[SPRWriteResGroup314], (instregex "^VCVT(T?)PD2(U?)QQZ(128|256)rr(( "^VPLZCNT(D|Q)Z(128|256)rr((k|kz)?)$", "^VPMADD52(H|L)UQZ(128|256)r((k|kz)?)$", "^VSCALEFS(D|S)Zrr((k|kz)?)$", - "^VSCALEFS(D|S)Zrrb_Int((k|kz)?)$")>; + "^VSCALEFS(D|S)Zrrb((k|kz)?)_Int$")>; def : InstRW<[SPRWriteResGroup314, ReadAfterVecLd], (instregex "^VFIXUPIMMS(D|S)Zrrib((k|kz)?)$")>; def SPRWriteResGroup315 : SchedWriteRes<[SPRPort00_01, SPRPort02_03_10, SPRPort05]> { @@ -3300,7 +3300,7 @@ def SPRWriteResGroup331 : SchedWriteRes<[SPRPort00_01, SPRPort02_03_10]> { let NumMicroOps = 2; } def : InstRW<[SPRWriteResGroup331], (instregex "^VCVTPH2PSZ(128|256)rmk(z?)$")>; -def : InstRW<[SPRWriteResGroup331, ReadAfterVecLd], (instregex "^VCVTSH2SSZrm_Intk(z?)$")>; +def : InstRW<[SPRWriteResGroup331, ReadAfterVecLd], (instregex "^VCVTSH2SSZrmk(z?)_Int$")>; def : InstRW<[SPRWriteResGroup331, ReadAfterVecXLd], (instregex "^VPMADDUBSWZ128rmk(z?)$", "^VPMULH((U|RS)?)WZ128rmk(z?)$", "^VPMULLWZ128rmk(z?)$")>; @@ -3460,7 +3460,7 @@ def SPRWriteResGroup353 : SchedWriteRes<[SPRPort00_01, SPRPort00_01_05, SPRPort0 let Latency = 21; let NumMicroOps = 7; } -def : InstRW<[SPRWriteResGroup353, ReadAfterVecLd], (instregex "^VCVTSD2SHZrm_Intk(z?)$")>; +def : InstRW<[SPRWriteResGroup353, ReadAfterVecLd], (instregex "^VCVTSD2SHZrmk(z?)_Int$")>; def SPRWriteResGroup354 : SchedWriteRes<[SPRPort00_01, SPRPort00_01_05, SPRPort05]> { let ReleaseAtCycles = [2, 1, 1]; @@ -3475,7 +3475,7 @@ def SPRWriteResGroup355 : SchedWriteRes<[SPRPort00_01, SPRPort00_01_05, SPRPort0 let Latency = 14; let NumMicroOps = 4; } -def : InstRW<[SPRWriteResGroup355], (instregex "^VCVTSD2SHZrr(b?)_Intk(z?)$")>; +def : InstRW<[SPRWriteResGroup355], (instregex "^VCVTSD2SHZrr(b?)k(z?)_Int$")>; def SPRWriteResGroup356 : SchedWriteRes<[SPRPort00_01, SPRPort02_03_10, SPRPort05]> { let ReleaseAtCycles = [2, 1, 1]; @@ -3489,7 +3489,7 @@ def SPRWriteResGroup357 : SchedWriteRes<[SPRPort00_01, SPRPort02_03_10, SPRPort0 let Latency = 20; let NumMicroOps = 4; } -def : InstRW<[SPRWriteResGroup357, ReadAfterVecLd], (instregex "^VCVTSH2SDZrm_Intk(z?)$")>; +def : InstRW<[SPRWriteResGroup357, ReadAfterVecLd], (instregex "^VCVTSH2SDZrmk(z?)_Int$")>; def SPRWriteResGroup358 : SchedWriteRes<[SPRPort00_01, SPRPort05]> { let ReleaseAtCycles = [2, 1]; @@ -3504,7 +3504,7 @@ def SPRWriteResGroup359 : SchedWriteRes<[SPRPort00_01, SPRPort05]> { let Latency = 13; let NumMicroOps = 3; } -def : InstRW<[SPRWriteResGroup359], (instregex "^VCVTSH2SDZrr(b?)_Intk(z?)$")>; +def : InstRW<[SPRWriteResGroup359], (instregex "^VCVTSH2SDZrr(b?)k(z?)_Int$")>; def SPRWriteResGroup360 : SchedWriteRes<[SPRPort00, SPRPort00_01, SPRPort02_03_10]> { let Latency = 13; @@ -3523,7 +3523,7 @@ def : InstRW<[SPRWriteResGroup361], (instregex "^VCVT(T?)SH2(U?)SI((64)?)Zrr(b?) def SPRWriteResGroup362 : SchedWriteRes<[SPRPort00_01]> { let Latency = 8; } -def : InstRW<[SPRWriteResGroup362], (instregex "^VCVTSH2SSZrr(b?)_Intk(z?)$")>; +def : InstRW<[SPRWriteResGroup362], (instregex "^VCVTSH2SSZrr(b?)k(z?)_Int$")>; def SPRWriteResGroup363 : SchedWriteRes<[SPRPort00_01, SPRPort00_01_05, SPRPort02_03_10]> { let Latency = 14; @@ -3536,7 +3536,7 @@ def SPRWriteResGroup364 : SchedWriteRes<[SPRPort00_01, SPRPort00_01_05, SPRPort0 let Latency = 16; let NumMicroOps = 3; } -def : InstRW<[SPRWriteResGroup364, ReadAfterVecLd], (instregex "^VCVTSS2SHZrm_Intk(z?)$")>; +def : InstRW<[SPRWriteResGroup364, ReadAfterVecLd], (instregex "^VCVTSS2SHZrmk(z?)_Int$")>; def SPRWriteResGroup365 : SchedWriteRes<[SPRPort00_01, SPRPort00_01_05]> { let Latency = 6; @@ -3549,7 +3549,7 @@ def SPRWriteResGroup366 : SchedWriteRes<[SPRPort00_01, SPRPort00_01_05]> { let Latency = 9; let NumMicroOps = 2; } -def : InstRW<[SPRWriteResGroup366], (instregex "^VCVTSS2SHZrr(b?)_Intk(z?)$")>; +def : InstRW<[SPRWriteResGroup366], (instregex "^VCVTSS2SHZrr(b?)k(z?)_Int$")>; def SPRWriteResGroup367 : SchedWriteRes<[SPRPort05]> { let Latency = 5; @@ -3667,7 +3667,7 @@ def SPRWriteResGroup380 : SchedWriteRes<[SPRPort00, SPRPort02_03_10]> { let Latency = 21; let NumMicroOps = 2; } -def : InstRW<[SPRWriteResGroup380, ReadAfterVecLd], (instregex "^VDIVSHZrm_Int((k|kz)?)$")>; +def : InstRW<[SPRWriteResGroup380, ReadAfterVecLd], (instregex "^VDIVSHZrm((k|kz)?)_Int$")>; def : InstRW<[SPRWriteResGroup380, ReadAfterVecLd], (instrs VDIVSHZrm)>; def SPRWriteResGroup381 : SchedWriteRes<[SPRPort00]> { @@ -4884,7 +4884,7 @@ def SPRWriteResGroup534 : SchedWriteRes<[SPRPort00_01, SPRPort02_03_10]> { let NumMicroOps = 3; } def : InstRW<[SPRWriteResGroup534, ReadAfterVecXLd], (instregex "^VRNDSCALEPHZ128rm(b?)ik(z?)$", - "^VRNDSCALESHZrmi_Intk(z?)$", + "^VRNDSCALESHZrmik(z?)_Int$", "^VSCALEFPHZ128rm(bk|kz)$", "^VSCALEFPHZ128rm(k|bkz)$")>; def : InstRW<[SPRWriteResGroup534, ReadAfterVecYLd], (instregex "^VRNDSCALEPHZ256rm(b?)ik(z?)$", @@ -4898,9 +4898,9 @@ def SPRWriteResGroup535 : SchedWriteRes<[SPRPort00_01]> { let NumMicroOps = 2; } def : InstRW<[SPRWriteResGroup535], (instregex "^VRNDSCALEPHZ(128|256)rrik(z?)$", - "^VRNDSCALESHZrri(b?)_Intk(z?)$", + "^VRNDSCALESHZrri(b?)k(z?)_Int$", "^VSCALEFPHZ(128|256)rrk(z?)$", - "^VSCALEFSHZrrb_Intk(z?)$", + "^VSCALEFSHZrrbk(z?)_Int$", "^VSCALEFSHZrrk(z?)$")>; def SPRWriteResGroup536 : SchedWriteRes<[SPRPort00, SPRPort02_03_10]> { @@ -4944,7 +4944,7 @@ def SPRWriteResGroup540 : SchedWriteRes<[SPRPort00, SPRPort02_03_10]> { } def : InstRW<[SPRWriteResGroup540, ReadAfterVecXLd], (instregex "^VSQRTPDZ128m(bk|kz)$", "^VSQRTPDZ128m(k|bkz)$")>; -def : InstRW<[SPRWriteResGroup540, ReadAfterVecLd], (instregex "^VSQRTSDZm_Intk(z?)$")>; +def : InstRW<[SPRWriteResGroup540, ReadAfterVecLd], (instregex "^VSQRTSDZmk(z?)_Int$")>; def SPRWriteResGroup541 : SchedWriteRes<[SPRPort00, SPRPort00_05, SPRPort02_03_10]> { let ReleaseAtCycles = [2, 1, 1]; diff --git a/llvm/lib/Target/X86/X86ScheduleZnver4.td b/llvm/lib/Target/X86/X86ScheduleZnver4.td index 38f9b5ef1d80..c5478dd9fc13 100644 --- a/llvm/lib/Target/X86/X86ScheduleZnver4.td +++ b/llvm/lib/Target/X86/X86ScheduleZnver4.td @@ -1545,7 +1545,7 @@ def Zn4WriteSCALErr: SchedWriteRes<[Zn4FPFMisc23]> { let NumMicroOps = 2; } def : InstRW<[Zn4WriteSCALErr], (instregex - "V(SCALEF|REDUCE)(S|P)(S|D)(Z?|Z128?|Z256?)(rr|rrb|rrkz|rrik|rrikz|rri)(_Int?|_Intkz?)", + "V(SCALEF|REDUCE)(S|P)(S|D)(Z?|Z128?|Z256?)(rr|rrb|rrkz|rrik|rrikz|rri)(_Int?)", "(V?)REDUCE(PD|PS|SD|SS)(Z?|Z128?)(rri|rrikz|rrib)" )>; @@ -1585,7 +1585,7 @@ def : InstRW<[Zn4WriteSHIFTrr], (instregex "(V?)P(ROL|ROR)(D|Q|VD|VQ)(Z?|Z128?|Z256?)(rr|rrk|rrkz)", "(V?)P(ROL|ROR)(D|Q|VD|VQ)(Z256?)(ri|rik|rikz)", "(V?)P(ROL|ROR)(D|Q)(Z?|Z128?)(ri|rik|rikz)", - "VPSHUFBITQMBZ128rr", "VFMSUB231SSZr_Intkz" + "VPSHUFBITQMBZ128rr", "VFMSUB231SSZrkz_Int" )>; def Zn4WriteSHIFTri: SchedWriteRes<[Zn4FPFMisc01]> { diff --git a/llvm/lib/Target/X86/X86SelectionDAGInfo.cpp b/llvm/lib/Target/X86/X86SelectionDAGInfo.cpp index 3f88bcf9ce5e..aba62c36546f 100644 --- a/llvm/lib/Target/X86/X86SelectionDAGInfo.cpp +++ b/llvm/lib/Target/X86/X86SelectionDAGInfo.cpp @@ -27,6 +27,16 @@ static cl::opt<bool> UseFSRMForMemcpy("x86-use-fsrm-for-memcpy", cl::Hidden, cl::init(false), cl::desc("Use fast short rep mov in memcpy lowering")); +bool X86SelectionDAGInfo::isTargetMemoryOpcode(unsigned Opcode) const { + return Opcode >= X86ISD::FIRST_MEMORY_OPCODE && + Opcode <= X86ISD::LAST_MEMORY_OPCODE; +} + +bool X86SelectionDAGInfo::isTargetStrictFPOpcode(unsigned Opcode) const { + return Opcode >= X86ISD::FIRST_STRICTFP_OPCODE && + Opcode <= X86ISD::LAST_STRICTFP_OPCODE; +} + /// Returns the best type to use with repmovs/repstos depending on alignment. static MVT getOptimalRepType(const X86Subtarget &Subtarget, Align Alignment) { uint64_t Align = Alignment.value(); diff --git a/llvm/lib/Target/X86/X86SelectionDAGInfo.h b/llvm/lib/Target/X86/X86SelectionDAGInfo.h index 19136ca4f6f5..e77e16bab830 100644 --- a/llvm/lib/Target/X86/X86SelectionDAGInfo.h +++ b/llvm/lib/Target/X86/X86SelectionDAGInfo.h @@ -26,6 +26,10 @@ class X86SelectionDAGInfo : public SelectionDAGTargetInfo { public: explicit X86SelectionDAGInfo() = default; + bool isTargetMemoryOpcode(unsigned Opcode) const override; + + bool isTargetStrictFPOpcode(unsigned Opcode) const override; + SDValue EmitTargetCodeForMemset(SelectionDAG &DAG, const SDLoc &dl, SDValue Chain, SDValue Dst, SDValue Src, SDValue Size, Align Alignment, diff --git a/llvm/lib/Target/X86/X86TargetTransformInfo.cpp b/llvm/lib/Target/X86/X86TargetTransformInfo.cpp index cae72c421035..808f48eb92a6 100644 --- a/llvm/lib/Target/X86/X86TargetTransformInfo.cpp +++ b/llvm/lib/Target/X86/X86TargetTransformInfo.cpp @@ -1532,6 +1532,11 @@ InstructionCost X86TTIImpl::getShuffleCost( Kind = improveShuffleKindFromMask(Kind, Mask, BaseTp, Index, SubTp); + // If all args are constant than this will be constant folded away. + if (!Args.empty() && + all_of(Args, [](const Value *Arg) { return isa<Constant>(Arg); })) + return TTI::TCC_Free; + // Recognize a basic concat_vector shuffle. if (Kind == TTI::SK_PermuteTwoSrc && Mask.size() == (2 * BaseTp->getElementCount().getKnownMinValue()) && @@ -1584,7 +1589,7 @@ InstructionCost X86TTIImpl::getShuffleCost( if (Kind == TTI::SK_ExtractSubvector && LT.second.isVector()) { int NumElts = LT.second.getVectorNumElements(); if ((Index % NumElts) == 0) - return 0; + return TTI::TCC_Free; std::pair<InstructionCost, MVT> SubLT = getTypeLegalizationCost(SubTp); if (SubLT.second.isVector()) { int NumSubElts = SubLT.second.getVectorNumElements(); @@ -1630,14 +1635,19 @@ InstructionCost X86TTIImpl::getShuffleCost( // Subvector insertions are cheap if the subvectors are aligned. // Note that in general, the insertion starting at the beginning of a vector - // isn't free, because we need to preserve the rest of the wide vector. + // isn't free, because we need to preserve the rest of the wide vector, + // but if the destination vector legalizes to the same width as the subvector + // then the insertion will simplify to a (free) register copy. if (Kind == TTI::SK_InsertSubvector && LT.second.isVector()) { int NumElts = LT.second.getVectorNumElements(); std::pair<InstructionCost, MVT> SubLT = getTypeLegalizationCost(SubTp); if (SubLT.second.isVector()) { int NumSubElts = SubLT.second.getVectorNumElements(); + bool MatchingTypes = + NumElts == NumSubElts && + (SubTp->getElementCount().getKnownMinValue() % NumSubElts) == 0; if ((Index % NumSubElts) == 0 && (NumElts % NumSubElts) == 0) - return SubLT.first; + return MatchingTypes ? TTI::TCC_Free : SubLT.first; } // If the insertion isn't aligned, treat it like a 2-op shuffle. @@ -1688,7 +1698,8 @@ InstructionCost X86TTIImpl::getShuffleCost( // We are going to permute multiple sources and the result will be in multiple // destinations. Providing an accurate cost only for splits where the element // type remains the same. - if (Kind == TTI::SK_PermuteSingleSrc && LT.first != 1) { + if ((Kind == TTI::SK_PermuteSingleSrc || Kind == TTI::SK_PermuteTwoSrc) && + LT.first != 1) { MVT LegalVT = LT.second; if (LegalVT.isVector() && LegalVT.getVectorElementType().getSizeInBits() == @@ -1718,7 +1729,7 @@ InstructionCost X86TTIImpl::getShuffleCost( // destination register is just a copy of the source register or the // copy of the previous destination register (the cost is // TTI::TCC_Basic). If the source register is just reused, the cost for - // this operation is 0. + // this operation is TTI::TCC_Free. NumOfDests = getTypeLegalizationCost( FixedVectorType::get(BaseTp->getElementType(), Mask.size())) @@ -1752,7 +1763,7 @@ InstructionCost X86TTIImpl::getShuffleCost( if (SrcReg != DestReg && any_of(RegMask, [](int I) { return I != PoisonMaskElem; })) { // Just a copy of the source register. - Cost += TTI::TCC_Basic; + Cost += TTI::TCC_Free; } PrevSrcReg = SrcReg; PrevRegMask = RegMask; @@ -1774,14 +1785,6 @@ InstructionCost X86TTIImpl::getShuffleCost( return BaseT::getShuffleCost(Kind, BaseTp, Mask, CostKind, Index, SubTp); } - // For 2-input shuffles, we must account for splitting the 2 inputs into many. - if (Kind == TTI::SK_PermuteTwoSrc && !IsInLaneShuffle && LT.first != 1) { - // We assume that source and destination have the same vector type. - InstructionCost NumOfDests = LT.first; - InstructionCost NumOfShufflesPerDest = LT.first * 2 - 1; - LT.first = NumOfDests * NumOfShufflesPerDest; - } - static const CostTblEntry AVX512VBMIShuffleTbl[] = { {TTI::SK_Reverse, MVT::v64i8, 1}, // vpermb {TTI::SK_Reverse, MVT::v32i8, 1}, // vpermb @@ -4763,7 +4766,7 @@ InstructionCost X86TTIImpl::getVectorInstrCost(unsigned Opcode, Type *Val, // This type is legalized to a scalar type. if (!LT.second.isVector()) - return 0; + return TTI::TCC_Free; // The type may be split. Normalize the index to the new type. unsigned SizeInBits = LT.second.getSizeInBits(); @@ -6030,9 +6033,9 @@ InstructionCost X86TTIImpl::getCFInstrCost(unsigned Opcode, TTI::TargetCostKind CostKind, const Instruction *I) { if (CostKind != TTI::TCK_RecipThroughput) - return Opcode == Instruction::PHI ? 0 : 1; + return Opcode == Instruction::PHI ? TTI::TCC_Free : TTI::TCC_Basic; // Branches are assumed to be predicted. - return 0; + return TTI::TCC_Free; } int X86TTIImpl::getGatherOverhead() const { diff --git a/llvm/lib/Target/X86/X86TileConfig.cpp b/llvm/lib/Target/X86/X86TileConfig.cpp index 75f002f418ba..c054b2672785 100644 --- a/llvm/lib/Target/X86/X86TileConfig.cpp +++ b/llvm/lib/Target/X86/X86TileConfig.cpp @@ -103,7 +103,7 @@ static void collectVirtRegShapes(MachineRegisterInfo *MRI, VirtRegMap &VRM, unsigned Index = PhysReg - X86::TMM0; if (!Phys2Shapes[Index].isValid()) { ShapeT Shape = VRM.getShape(VirtReg); - Phys2Shapes[Index] = Shape; + Phys2Shapes[Index] = std::move(Shape); return; } } @@ -118,12 +118,12 @@ static void collectVirtRegShapes(MachineRegisterInfo *MRI, VirtRegMap &VRM, if (!Phys2Shapes[Index0].isValid()) { ShapeT Shape0(Shape.getRow(0), Shape.getCol(0), MRI); - Phys2Shapes[Index0] = Shape0; + Phys2Shapes[Index0] = std::move(Shape0); } if (!Phys2Shapes[Index1].isValid()) { ShapeT Shape1(Shape.getRow(1), Shape.getCol(1), MRI); - Phys2Shapes[Index1] = Shape1; + Phys2Shapes[Index1] = std::move(Shape1); } } } diff --git a/llvm/lib/Target/X86/X86WinEHState.cpp b/llvm/lib/Target/X86/X86WinEHState.cpp index b3e4c70eb57f..35b7d7f508b0 100644 --- a/llvm/lib/Target/X86/X86WinEHState.cpp +++ b/llvm/lib/Target/X86/X86WinEHState.cpp @@ -363,6 +363,11 @@ void WinEHStatePass::emitExceptionRegistrationRecord(Function *F) { Instruction *T = BB.getTerminator(); if (!isa<ReturnInst>(T)) continue; + + // If there is a musttail call, that's the de-facto terminator. + if (CallInst *CI = BB.getTerminatingMustTailCall()) + T = CI; + Builder.SetInsertPoint(T); unlinkExceptionRegistration(Builder); } |