3 files changed, 74 insertions, 13 deletions

diff --git a/llvm/lib/CodeGen/CodeGenPrepare.cpp b/llvm/lib/CodeGen/CodeGenPrepare.cpp
index 52263026d6ce..422916a28669 100644
--- a/llvm/lib/CodeGen/CodeGenPrepare.cpp
+++ b/llvm/lib/CodeGen/CodeGenPrepare.cpp
@@ -7739,9 +7739,14 @@ bool CodeGenPrepare::tryToSinkFreeOperands(Instruction *I) {
 
   for (Use *U : reverse(OpsToSink)) {
     auto *UI = cast<Instruction>(U->get());
-    if (isa<PHINode>(UI))
-      continue;
-    if (UI->getParent() == TargetBB) {
+    if (auto *PN = dyn_cast<PHINode>(UI)) {
+      auto *I0 = dyn_cast<Instruction>(PN->hasIdenticalValue());
+      if (!I0)
+        continue;
+      if (I0->getParent() == TargetBB &&
+          InstOrdering[I0] < InstOrdering[InsertPoint])
+        InsertPoint = I0;
+    } else if (UI->getParent() == TargetBB) {
       if (InstOrdering[UI] < InstOrdering[InsertPoint])
         InsertPoint = UI;
       continue;
@@ -7753,7 +7758,11 @@ bool CodeGenPrepare::tryToSinkFreeOperands(Instruction *I) {
   DenseMap<Instruction *, Instruction *> NewInstructions;
   for (Use *U : ToReplace) {
     auto *UI = cast<Instruction>(U->get());
-    Instruction *NI = UI->clone();
+    Instruction *NI;
+    if (auto *PN = dyn_cast<PHINode>(UI))
+      NI = cast<Instruction>(PN->hasIdenticalValue())->clone();
+    else
+      NI = UI->clone();
 
     if (IsHugeFunc) {
       // Now we clone an instruction, its operands' defs may sink to this BB
diff --git a/llvm/lib/IR/Instructions.cpp b/llvm/lib/IR/Instructions.cpp
index f404e11b9c0f..9c5937821399 100644
--- a/llvm/lib/IR/Instructions.cpp
+++ b/llvm/lib/IR/Instructions.cpp
@@ -48,6 +48,7 @@
 #include <cassert>
 #include <cstdint>
 #include <optional>
+#include <set>
 #include <vector>
 
 using namespace llvm;
@@ -239,6 +240,40 @@ bool PHINode::hasConstantOrUndefValue() const {
   return true;
 }
 
+/// If the specified PHI node (possibly via other PHI nodes) merges together the
+/// same or identical (i.e. Instruction::isIdenticalTo() returns true) values,
+/// return one of the values, otherwise return null.
+Value *PHINode::hasIdenticalValue() {
+  std::vector<PHINode *> Worklist;
+  std::set<PHINode *> Seen;
+  Value *Result = nullptr;
+  Worklist.push_back(this);
+  while (!Worklist.empty()) {
+    PHINode *PN = Worklist.back();
+    Worklist.pop_back();
+    if (!Seen.insert(PN).second)
+      continue;
+    for (Value *V : PN->incoming_values()) {
+      if (auto *PN = dyn_cast<PHINode>(V)) {
+        Worklist.push_back(PN);
+        continue;
+      }
+      if (!Result) {
+        Result = V;
+        continue;
+      }
+      if (V == Result)
+        continue;
+      if (auto *I = dyn_cast<Instruction>(V))
+        if (auto *ResultI = dyn_cast<Instruction>(Result))
+          if (I->isIdenticalTo(ResultI))
+            continue;
+      return nullptr;
+    }
+  }
+  return Result;
+}
+
 //===----------------------------------------------------------------------===//
 //                       LandingPadInst Implementation
 //===----------------------------------------------------------------------===//
diff --git a/llvm/lib/Target/X86/X86TargetTransformInfo.cpp b/llvm/lib/Target/X86/X86TargetTransformInfo.cpp
index 9864adc4dcc9..5baf5e3001ba 100644
--- a/llvm/lib/Target/X86/X86TargetTransformInfo.cpp
+++ b/llvm/lib/Target/X86/X86TargetTransformInfo.cpp
@@ -7132,10 +7132,8 @@ bool X86TTIImpl::isProfitableToSinkOperands(Instruction *I,
   using namespace llvm::PatternMatch;
 
   FixedVectorType *VTy = dyn_cast<FixedVectorType>(I->getType());
-  if (!VTy)
-    return false;
 
-  if (I->getOpcode() == Instruction::Mul &&
+  if (VTy && I->getOpcode() == Instruction::Mul &&
       VTy->getElementType()->isIntegerTy(64)) {
     for (auto &Op : I->operands()) {
       // Make sure we are not already sinking this operand
@@ -7159,9 +7157,6 @@ bool X86TTIImpl::isProfitableToSinkOperands(Instruction *I,
     return !Ops.empty();
   }
 
-  // A uniform shift amount in a vector shift or funnel shift may be much
-  // cheaper than a generic variable vector shift, so make that pattern visible
-  // to SDAG by sinking the shuffle instruction next to the shift.
   int ShiftAmountOpNum = -1;
   if (I->isShift())
     ShiftAmountOpNum = 1;
@@ -7170,16 +7165,38 @@ bool X86TTIImpl::isProfitableToSinkOperands(Instruction *I,
         II->getIntrinsicID() == Intrinsic::fshr)
       ShiftAmountOpNum = 2;
   }
-
   if (ShiftAmountOpNum == -1)
     return false;
+  auto *ShiftAmountUse = &I->getOperandUse(ShiftAmountOpNum);
+
+  Value *ShiftAmount = ShiftAmountUse->get();
+  if (auto *PN = dyn_cast<PHINode>(ShiftAmount)) {
+    ShiftAmount = PN->hasIdenticalValue();
+    if (!ShiftAmount)
+      return false;
+  }
 
-  auto *Shuf = dyn_cast<ShuffleVectorInst>(I->getOperand(ShiftAmountOpNum));
+  // A uniform shift amount in a vector shift or funnel shift may be much
+  // cheaper than a generic variable vector shift, so make that pattern visible
+  // to SDAG by sinking the shuffle instruction next to the shift.
+  auto *Shuf = dyn_cast<ShuffleVectorInst>(ShiftAmount);
   if (Shuf && getSplatIndex(Shuf->getShuffleMask()) >= 0 &&
       isVectorShiftByScalarCheap(I->getType())) {
-    Ops.push_back(&I->getOperandUse(ShiftAmountOpNum));
+    Ops.push_back(ShiftAmountUse);
     return true;
   }
 
+  // Casts taking a constant expression (generally derived from a global
+  // variable address) as an operand are profitable to sink because they appear
+  // as subexpressions in the instruction sequence generated by the
+  // LowerTypeTests pass which is expected to pattern match to the rotate
+  // instruction's immediate operand.
+  if (auto *CI = dyn_cast<CastInst>(ShiftAmount)) {
+    if (isa<ConstantExpr>(CI->getOperand(0))) {
+      Ops.push_back(ShiftAmountUse);
+      return true;
+    }
+  }
+
   return false;
 }