[𝘀𝗽𝗿] changes introduced through rebaseusers/vitalybuka/spr/main.analysis-exclude-llvmallowruntimeubsancheck-from-aliassettracker

Created using spr 1.3.4

[skip ci]

1 files changed, 2295 insertions, 0 deletions

diff --git a/llvm/utils/TableGen/Common/CodeGenSchedule.cpp b/llvm/utils/TableGen/Common/CodeGenSchedule.cpp
new file mode 100644
index 000000000000..0e81623a6aa3
--- /dev/null
+++ b/llvm/utils/TableGen/Common/CodeGenSchedule.cpp
@@ -0,0 +1,2295 @@
+//===- CodeGenSchedule.cpp - Scheduling MachineModels ---------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This file defines structures to encapsulate the machine model as described in
+// the target description.
+//
+//===----------------------------------------------------------------------===//
+
+#include "CodeGenSchedule.h"
+#include "CodeGenInstruction.h"
+#include "CodeGenTarget.h"
+#include "llvm/ADT/MapVector.h"
+#include "llvm/ADT/STLExtras.h"
+#include "llvm/ADT/SmallPtrSet.h"
+#include "llvm/ADT/SmallVector.h"
+#include "llvm/Support/Casting.h"
+#include "llvm/Support/Debug.h"
+#include "llvm/Support/Regex.h"
+#include "llvm/Support/raw_ostream.h"
+#include "llvm/TableGen/Error.h"
+#include <algorithm>
+#include <iterator>
+#include <utility>
+
+using namespace llvm;
+
+#define DEBUG_TYPE "subtarget-emitter"
+
+#ifndef NDEBUG
+static void dumpIdxVec(ArrayRef<unsigned> V) {
+  for (unsigned Idx : V)
+    dbgs() << Idx << ", ";
+}
+#endif
+
+namespace {
+
+// (instrs a, b, ...) Evaluate and union all arguments. Identical to AddOp.
+struct InstrsOp : public SetTheory::Operator {
+  void apply(SetTheory &ST, DagInit *Expr, SetTheory::RecSet &Elts,
+             ArrayRef<SMLoc> Loc) override {
+    ST.evaluate(Expr->arg_begin(), Expr->arg_end(), Elts, Loc);
+  }
+};
+
+// (instregex "OpcPat",...) Find all instructions matching an opcode pattern.
+struct InstRegexOp : public SetTheory::Operator {
+  const CodeGenTarget &Target;
+  InstRegexOp(const CodeGenTarget &t) : Target(t) {}
+
+  /// Remove any text inside of parentheses from S.
+  static std::string removeParens(llvm::StringRef S) {
+    std::string Result;
+    unsigned Paren = 0;
+    // NB: We don't care about escaped parens here.
+    for (char C : S) {
+      switch (C) {
+      case '(':
+        ++Paren;
+        break;
+      case ')':
+        --Paren;
+        break;
+      default:
+        if (Paren == 0)
+          Result += C;
+      }
+    }
+    return Result;
+  }
+
+  void apply(SetTheory &ST, DagInit *Expr, SetTheory::RecSet &Elts,
+             ArrayRef<SMLoc> Loc) override {
+    ArrayRef<const CodeGenInstruction *> Instructions =
+        Target.getInstructionsByEnumValue();
+
+    unsigned NumGeneric = Target.getNumFixedInstructions();
+    unsigned NumPseudos = Target.getNumPseudoInstructions();
+    auto Generics = Instructions.slice(0, NumGeneric);
+    auto Pseudos = Instructions.slice(NumGeneric, NumPseudos);
+    auto NonPseudos = Instructions.slice(NumGeneric + NumPseudos);
+
+    for (Init *Arg : Expr->getArgs()) {
+      StringInit *SI = dyn_cast<StringInit>(Arg);
+      if (!SI)
+        PrintFatalError(Loc, "instregex requires pattern string: " +
+                                 Expr->getAsString());
+      StringRef Original = SI->getValue();
+      // Drop an explicit ^ anchor to not interfere with prefix search.
+      bool HadAnchor = Original.consume_front("^");
+
+      // Extract a prefix that we can binary search on.
+      static const char RegexMetachars[] = "()^$|*+?.[]\\{}";
+      auto FirstMeta = Original.find_first_of(RegexMetachars);
+      if (FirstMeta != StringRef::npos && FirstMeta > 0) {
+        // If we have a regex like ABC* we can only use AB as the prefix, as
+        // the * acts on C.
+        switch (Original[FirstMeta]) {
+        case '+':
+        case '*':
+        case '?':
+          --FirstMeta;
+          break;
+        default:
+          break;
+        }
+      }
+
+      // Look for top-level | or ?. We cannot optimize them to binary search.
+      if (removeParens(Original).find_first_of("|?") != std::string::npos)
+        FirstMeta = 0;
+
+      std::optional<Regex> Regexpr;
+      StringRef Prefix = Original.substr(0, FirstMeta);
+      StringRef PatStr = Original.substr(FirstMeta);
+      if (!PatStr.empty()) {
+        // For the rest use a python-style prefix match.
+        std::string pat = std::string(PatStr);
+        // Add ^ anchor. If we had one originally, don't need the group.
+        if (HadAnchor) {
+          pat.insert(0, "^");
+        } else {
+          pat.insert(0, "^(");
+          pat.insert(pat.end(), ')');
+        }
+        Regexpr = Regex(pat);
+      }
+
+      int NumMatches = 0;
+
+      // The generic opcodes are unsorted, handle them manually.
+      for (auto *Inst : Generics) {
+        StringRef InstName = Inst->TheDef->getName();
+        if (InstName.starts_with(Prefix) &&
+            (!Regexpr || Regexpr->match(InstName.substr(Prefix.size())))) {
+          Elts.insert(Inst->TheDef);
+          NumMatches++;
+        }
+      }
+
+      // Target instructions are split into two ranges: pseudo instructions
+      // first, than non-pseudos. Each range is in lexicographical order
+      // sorted by name. Find the sub-ranges that start with our prefix.
+      struct Comp {
+        bool operator()(const CodeGenInstruction *LHS, StringRef RHS) {
+          return LHS->TheDef->getName() < RHS;
+        }
+        bool operator()(StringRef LHS, const CodeGenInstruction *RHS) {
+          return LHS < RHS->TheDef->getName() &&
+                 !RHS->TheDef->getName().starts_with(LHS);
+        }
+      };
+      auto Range1 =
+          std::equal_range(Pseudos.begin(), Pseudos.end(), Prefix, Comp());
+      auto Range2 = std::equal_range(NonPseudos.begin(), NonPseudos.end(),
+                                     Prefix, Comp());
+
+      // For these ranges we know that instruction names start with the prefix.
+      // Check if there's a regex that needs to be checked.
+      const auto HandleNonGeneric = [&](const CodeGenInstruction *Inst) {
+        StringRef InstName = Inst->TheDef->getName();
+        if (!Regexpr || Regexpr->match(InstName.substr(Prefix.size()))) {
+          Elts.insert(Inst->TheDef);
+          NumMatches++;
+        }
+      };
+      std::for_each(Range1.first, Range1.second, HandleNonGeneric);
+      std::for_each(Range2.first, Range2.second, HandleNonGeneric);
+
+      if (0 == NumMatches)
+        PrintFatalError(Loc, "instregex has no matches: " + Original);
+    }
+  }
+};
+
+} // end anonymous namespace
+
+/// CodeGenModels ctor interprets machine model records and populates maps.
+CodeGenSchedModels::CodeGenSchedModels(RecordKeeper &RK,
+                                       const CodeGenTarget &TGT)
+    : Records(RK), Target(TGT) {
+
+  Sets.addFieldExpander("InstRW", "Instrs");
+
+  // Allow Set evaluation to recognize the dags used in InstRW records:
+  // (instrs Op1, Op1...)
+  Sets.addOperator("instrs", std::make_unique<InstrsOp>());
+  Sets.addOperator("instregex", std::make_unique<InstRegexOp>(Target));
+
+  // Instantiate a CodeGenProcModel for each SchedMachineModel with the values
+  // that are explicitly referenced in tablegen records. Resources associated
+  // with each processor will be derived later. Populate ProcModelMap with the
+  // CodeGenProcModel instances.
+  collectProcModels();
+
+  // Instantiate a CodeGenSchedRW for each SchedReadWrite record explicitly
+  // defined, and populate SchedReads and SchedWrites vectors. Implicit
+  // SchedReadWrites that represent sequences derived from expanded variant will
+  // be inferred later.
+  collectSchedRW();
+
+  // Instantiate a CodeGenSchedClass for each unique SchedRW signature directly
+  // required by an instruction definition, and populate SchedClassIdxMap. Set
+  // NumItineraryClasses to the number of explicit itinerary classes referenced
+  // by instructions. Set NumInstrSchedClasses to the number of itinerary
+  // classes plus any classes implied by instructions that derive from class
+  // Sched and provide SchedRW list. This does not infer any new classes from
+  // SchedVariant.
+  collectSchedClasses();
+
+  // Find instruction itineraries for each processor. Sort and populate
+  // CodeGenProcModel::ItinDefList. (Cycle-to-cycle itineraries). This requires
+  // all itinerary classes to be discovered.
+  collectProcItins();
+
+  // Find ItinRW records for each processor and itinerary class.
+  // (For per-operand resources mapped to itinerary classes).
+  collectProcItinRW();
+
+  // Find UnsupportedFeatures records for each processor.
+  // (For per-operand resources mapped to itinerary classes).
+  collectProcUnsupportedFeatures();
+
+  // Infer new SchedClasses from SchedVariant.
+  inferSchedClasses();
+
+  // Populate each CodeGenProcModel's WriteResDefs, ReadAdvanceDefs, and
+  // ProcResourceDefs.
+  LLVM_DEBUG(
+      dbgs() << "\n+++ RESOURCE DEFINITIONS (collectProcResources) +++\n");
+  collectProcResources();
+
+  // Collect optional processor description.
+  collectOptionalProcessorInfo();
+
+  // Check MCInstPredicate definitions.
+  checkMCInstPredicates();
+
+  // Check STIPredicate definitions.
+  checkSTIPredicates();
+
+  // Find STIPredicate definitions for each processor model, and construct
+  // STIPredicateFunction objects.
+  collectSTIPredicates();
+
+  checkCompleteness();
+}
+
+void CodeGenSchedModels::checkSTIPredicates() const {
+  DenseMap<StringRef, const Record *> Declarations;
+
+  // There cannot be multiple declarations with the same name.
+  const RecVec Decls = Records.getAllDerivedDefinitions("STIPredicateDecl");
+  for (const Record *R : Decls) {
+    StringRef Name = R->getValueAsString("Name");
+    const auto It = Declarations.find(Name);
+    if (It == Declarations.end()) {
+      Declarations[Name] = R;
+      continue;
+    }
+
+    PrintError(R->getLoc(), "STIPredicate " + Name + " multiply declared.");
+    PrintFatalNote(It->second->getLoc(), "Previous declaration was here.");
+  }
+
+  // Disallow InstructionEquivalenceClasses with an empty instruction list.
+  const RecVec Defs =
+      Records.getAllDerivedDefinitions("InstructionEquivalenceClass");
+  for (const Record *R : Defs) {
+    RecVec Opcodes = R->getValueAsListOfDefs("Opcodes");
+    if (Opcodes.empty()) {
+      PrintFatalError(R->getLoc(), "Invalid InstructionEquivalenceClass "
+                                   "defined with an empty opcode list.");
+    }
+  }
+}
+
+// Used by function `processSTIPredicate` to construct a mask of machine
+// instruction operands.
+static APInt constructOperandMask(ArrayRef<int64_t> Indices) {
+  APInt OperandMask;
+  if (Indices.empty())
+    return OperandMask;
+
+  int64_t MaxIndex = *llvm::max_element(Indices);
+  assert(MaxIndex >= 0 && "Invalid negative indices in input!");
+  OperandMask = OperandMask.zext(MaxIndex + 1);
+  for (const int64_t Index : Indices) {
+    assert(Index >= 0 && "Invalid negative indices!");
+    OperandMask.setBit(Index);
+  }
+
+  return OperandMask;
+}
+
+static void processSTIPredicate(STIPredicateFunction &Fn,
+                                const ProcModelMapTy &ProcModelMap) {
+  DenseMap<const Record *, unsigned> Opcode2Index;
+  using OpcodeMapPair = std::pair<const Record *, OpcodeInfo>;
+  std::vector<OpcodeMapPair> OpcodeMappings;
+  std::vector<std::pair<APInt, APInt>> OpcodeMasks;
+
+  DenseMap<const Record *, unsigned> Predicate2Index;
+  unsigned NumUniquePredicates = 0;
+
+  // Number unique predicates and opcodes used by InstructionEquivalenceClass
+  // definitions. Each unique opcode will be associated with an OpcodeInfo
+  // object.
+  for (const Record *Def : Fn.getDefinitions()) {
+    RecVec Classes = Def->getValueAsListOfDefs("Classes");
+    for (const Record *EC : Classes) {
+      const Record *Pred = EC->getValueAsDef("Predicate");
+      if (!Predicate2Index.contains(Pred))
+        Predicate2Index[Pred] = NumUniquePredicates++;
+
+      RecVec Opcodes = EC->getValueAsListOfDefs("Opcodes");
+      for (const Record *Opcode : Opcodes) {
+        if (!Opcode2Index.contains(Opcode)) {
+          Opcode2Index[Opcode] = OpcodeMappings.size();
+          OpcodeMappings.emplace_back(Opcode, OpcodeInfo());
+        }
+      }
+    }
+  }
+
+  // Initialize vector `OpcodeMasks` with default values.  We want to keep track
+  // of which processors "use" which opcodes.  We also want to be able to
+  // identify predicates that are used by different processors for a same
+  // opcode.
+  // This information is used later on by this algorithm to sort OpcodeMapping
+  // elements based on their processor and predicate sets.
+  OpcodeMasks.resize(OpcodeMappings.size());
+  APInt DefaultProcMask(ProcModelMap.size(), 0);
+  APInt DefaultPredMask(NumUniquePredicates, 0);
+  for (std::pair<APInt, APInt> &MaskPair : OpcodeMasks)
+    MaskPair = std::pair(DefaultProcMask, DefaultPredMask);
+
+  // Construct a OpcodeInfo object for every unique opcode declared by an
+  // InstructionEquivalenceClass definition.
+  for (const Record *Def : Fn.getDefinitions()) {
+    RecVec Classes = Def->getValueAsListOfDefs("Classes");
+    const Record *SchedModel = Def->getValueAsDef("SchedModel");
+    unsigned ProcIndex = ProcModelMap.find(SchedModel)->second;
+    APInt ProcMask(ProcModelMap.size(), 0);
+    ProcMask.setBit(ProcIndex);
+
+    for (const Record *EC : Classes) {
+      RecVec Opcodes = EC->getValueAsListOfDefs("Opcodes");
+
+      std::vector<int64_t> OpIndices =
+          EC->getValueAsListOfInts("OperandIndices");
+      APInt OperandMask = constructOperandMask(OpIndices);
+
+      const Record *Pred = EC->getValueAsDef("Predicate");
+      APInt PredMask(NumUniquePredicates, 0);
+      PredMask.setBit(Predicate2Index[Pred]);
+
+      for (const Record *Opcode : Opcodes) {
+        unsigned OpcodeIdx = Opcode2Index[Opcode];
+        if (OpcodeMasks[OpcodeIdx].first[ProcIndex]) {
+          std::string Message =
+              "Opcode " + Opcode->getName().str() +
+              " used by multiple InstructionEquivalenceClass definitions.";
+          PrintFatalError(EC->getLoc(), Message);
+        }
+        OpcodeMasks[OpcodeIdx].first |= ProcMask;
+        OpcodeMasks[OpcodeIdx].second |= PredMask;
+        OpcodeInfo &OI = OpcodeMappings[OpcodeIdx].second;
+
+        OI.addPredicateForProcModel(ProcMask, OperandMask, Pred);
+      }
+    }
+  }
+
+  // Sort OpcodeMappings elements based on their CPU and predicate masks.
+  // As a last resort, order elements by opcode identifier.
+  llvm::sort(
+      OpcodeMappings, [&](const OpcodeMapPair &Lhs, const OpcodeMapPair &Rhs) {
+        unsigned LhsIdx = Opcode2Index[Lhs.first];
+        unsigned RhsIdx = Opcode2Index[Rhs.first];
+        const std::pair<APInt, APInt> &LhsMasks = OpcodeMasks[LhsIdx];
+        const std::pair<APInt, APInt> &RhsMasks = OpcodeMasks[RhsIdx];
+
+        auto PopulationCountAndLeftBit =
+            [](const APInt &Other) -> std::pair<int, int> {
+          return std::pair<int, int>(Other.popcount(), -Other.countl_zero());
+        };
+        auto lhsmask_first = PopulationCountAndLeftBit(LhsMasks.first);
+        auto rhsmask_first = PopulationCountAndLeftBit(RhsMasks.first);
+        if (lhsmask_first != rhsmask_first)
+          return lhsmask_first < rhsmask_first;
+
+        auto lhsmask_second = PopulationCountAndLeftBit(LhsMasks.second);
+        auto rhsmask_second = PopulationCountAndLeftBit(RhsMasks.second);
+        if (lhsmask_second != rhsmask_second)
+          return lhsmask_second < rhsmask_second;
+
+        return LhsIdx < RhsIdx;
+      });
+
+  // Now construct opcode groups. Groups are used by the SubtargetEmitter when
+  // expanding the body of a STIPredicate function. In particular, each opcode
+  // group is expanded into a sequence of labels in a switch statement.
+  // It identifies opcodes for which different processors define same predicates
+  // and same opcode masks.
+  for (OpcodeMapPair &Info : OpcodeMappings)
+    Fn.addOpcode(Info.first, std::move(Info.second));
+}
+
+void CodeGenSchedModels::collectSTIPredicates() {
+  // Map STIPredicateDecl records to elements of vector
+  // CodeGenSchedModels::STIPredicates.
+  DenseMap<const Record *, unsigned> Decl2Index;
+
+  RecVec RV = Records.getAllDerivedDefinitions("STIPredicate");
+  for (const Record *R : RV) {
+    const Record *Decl = R->getValueAsDef("Declaration");
+
+    const auto It = Decl2Index.find(Decl);
+    if (It == Decl2Index.end()) {
+      Decl2Index[Decl] = STIPredicates.size();
+      STIPredicateFunction Predicate(Decl);
+      Predicate.addDefinition(R);
+      STIPredicates.emplace_back(std::move(Predicate));
+      continue;
+    }
+
+    STIPredicateFunction &PreviousDef = STIPredicates[It->second];
+    PreviousDef.addDefinition(R);
+  }
+
+  for (STIPredicateFunction &Fn : STIPredicates)
+    processSTIPredicate(Fn, ProcModelMap);
+}
+
+void OpcodeInfo::addPredicateForProcModel(const llvm::APInt &CpuMask,
+                                          const llvm::APInt &OperandMask,
+                                          const Record *Predicate) {
+  auto It = llvm::find_if(
+      Predicates, [&OperandMask, &Predicate](const PredicateInfo &P) {
+        return P.Predicate == Predicate && P.OperandMask == OperandMask;
+      });
+  if (It == Predicates.end()) {
+    Predicates.emplace_back(CpuMask, OperandMask, Predicate);
+    return;
+  }
+  It->ProcModelMask |= CpuMask;
+}
+
+void CodeGenSchedModels::checkMCInstPredicates() const {
+  RecVec MCPredicates = Records.getAllDerivedDefinitions("TIIPredicate");
+  if (MCPredicates.empty())
+    return;
+
+  // A target cannot have multiple TIIPredicate definitions with a same name.
+  llvm::StringMap<const Record *> TIIPredicates(MCPredicates.size());
+  for (const Record *TIIPred : MCPredicates) {
+    StringRef Name = TIIPred->getValueAsString("FunctionName");
+    StringMap<const Record *>::const_iterator It = TIIPredicates.find(Name);
+    if (It == TIIPredicates.end()) {
+      TIIPredicates[Name] = TIIPred;
+      continue;
+    }
+
+    PrintError(TIIPred->getLoc(),
+               "TIIPredicate " + Name + " is multiply defined.");
+    PrintFatalNote(It->second->getLoc(),
+                   " Previous definition of " + Name + " was here.");
+  }
+}
+
+void CodeGenSchedModels::collectRetireControlUnits() {
+  RecVec Units = Records.getAllDerivedDefinitions("RetireControlUnit");
+
+  for (Record *RCU : Units) {
+    CodeGenProcModel &PM = getProcModel(RCU->getValueAsDef("SchedModel"));
+    if (PM.RetireControlUnit) {
+      PrintError(RCU->getLoc(),
+                 "Expected a single RetireControlUnit definition");
+      PrintNote(PM.RetireControlUnit->getLoc(),
+                "Previous definition of RetireControlUnit was here");
+    }
+    PM.RetireControlUnit = RCU;
+  }
+}
+
+void CodeGenSchedModels::collectLoadStoreQueueInfo() {
+  RecVec Queues = Records.getAllDerivedDefinitions("MemoryQueue");
+
+  for (Record *Queue : Queues) {
+    CodeGenProcModel &PM = getProcModel(Queue->getValueAsDef("SchedModel"));
+    if (Queue->isSubClassOf("LoadQueue")) {
+      if (PM.LoadQueue) {
+        PrintError(Queue->getLoc(), "Expected a single LoadQueue definition");
+        PrintNote(PM.LoadQueue->getLoc(),
+                  "Previous definition of LoadQueue was here");
+      }
+
+      PM.LoadQueue = Queue;
+    }
+
+    if (Queue->isSubClassOf("StoreQueue")) {
+      if (PM.StoreQueue) {
+        PrintError(Queue->getLoc(), "Expected a single StoreQueue definition");
+        PrintNote(PM.StoreQueue->getLoc(),
+                  "Previous definition of StoreQueue was here");
+      }
+
+      PM.StoreQueue = Queue;
+    }
+  }
+}
+
+/// Collect optional processor information.
+void CodeGenSchedModels::collectOptionalProcessorInfo() {
+  // Find register file definitions for each processor.
+  collectRegisterFiles();
+
+  // Collect processor RetireControlUnit descriptors if available.
+  collectRetireControlUnits();
+
+  // Collect information about load/store queues.
+  collectLoadStoreQueueInfo();
+
+  checkCompleteness();
+}
+
+/// Gather all processor models.
+void CodeGenSchedModels::collectProcModels() {
+  RecVec ProcRecords = Records.getAllDerivedDefinitions("Processor");
+  llvm::sort(ProcRecords, LessRecordFieldName());
+
+  // Check for duplicated names.
+  auto I = std::adjacent_find(ProcRecords.begin(), ProcRecords.end(),
+                              [](const Record *Rec1, const Record *Rec2) {
+                                return Rec1->getValueAsString("Name") ==
+                                       Rec2->getValueAsString("Name");
+                              });
+  if (I != ProcRecords.end())
+    PrintFatalError((*I)->getLoc(), "Duplicate processor name " +
+                                        (*I)->getValueAsString("Name"));
+
+  // Reserve space because we can. Reallocation would be ok.
+  ProcModels.reserve(ProcRecords.size() + 1);
+
+  // Use idx=0 for NoModel/NoItineraries.
+  Record *NoModelDef = Records.getDef("NoSchedModel");
+  Record *NoItinsDef = Records.getDef("NoItineraries");
+  ProcModels.emplace_back(0, "NoSchedModel", NoModelDef, NoItinsDef);
+  ProcModelMap[NoModelDef] = 0;
+
+  // For each processor, find a unique machine model.
+  LLVM_DEBUG(dbgs() << "+++ PROCESSOR MODELs (addProcModel) +++\n");
+  for (Record *ProcRecord : ProcRecords)
+    addProcModel(ProcRecord);
+}
+
+/// Get a unique processor model based on the defined MachineModel and
+/// ProcessorItineraries.
+void CodeGenSchedModels::addProcModel(Record *ProcDef) {
+  Record *ModelKey = getModelOrItinDef(ProcDef);
+  if (!ProcModelMap.insert(std::pair(ModelKey, ProcModels.size())).second)
+    return;
+
+  std::string Name = std::string(ModelKey->getName());
+  if (ModelKey->isSubClassOf("SchedMachineModel")) {
+    Record *ItinsDef = ModelKey->getValueAsDef("Itineraries");
+    ProcModels.emplace_back(ProcModels.size(), Name, ModelKey, ItinsDef);
+  } else {
+    // An itinerary is defined without a machine model. Infer a new model.
+    if (!ModelKey->getValueAsListOfDefs("IID").empty())
+      Name = Name + "Model";
+    ProcModels.emplace_back(ProcModels.size(), Name,
+                            ProcDef->getValueAsDef("SchedModel"), ModelKey);
+  }
+  LLVM_DEBUG(ProcModels.back().dump());
+}
+
+// Recursively find all reachable SchedReadWrite records.
+static void scanSchedRW(Record *RWDef, RecVec &RWDefs,
+                        SmallPtrSet<Record *, 16> &RWSet) {
+  if (!RWSet.insert(RWDef).second)
+    return;
+  RWDefs.push_back(RWDef);
+  // Reads don't currently have sequence records, but it can be added later.
+  if (RWDef->isSubClassOf("WriteSequence")) {
+    RecVec Seq = RWDef->getValueAsListOfDefs("Writes");
+    for (Record *WSRec : Seq)
+      scanSchedRW(WSRec, RWDefs, RWSet);
+  } else if (RWDef->isSubClassOf("SchedVariant")) {
+    // Visit each variant (guarded by a different predicate).
+    RecVec Vars = RWDef->getValueAsListOfDefs("Variants");
+    for (Record *Variant : Vars) {
+      // Visit each RW in the sequence selected by the current variant.
+      RecVec Selected = Variant->getValueAsListOfDefs("Selected");
+      for (Record *SelDef : Selected)
+        scanSchedRW(SelDef, RWDefs, RWSet);
+    }
+  }
+}
+
+// Collect and sort all SchedReadWrites reachable via tablegen records.
+// More may be inferred later when inferring new SchedClasses from variants.
+void CodeGenSchedModels::collectSchedRW() {
+  // Reserve idx=0 for invalid writes/reads.
+  SchedWrites.resize(1);
+  SchedReads.resize(1);
+
+  SmallPtrSet<Record *, 16> RWSet;
+
+  // Find all SchedReadWrites referenced by instruction defs.
+  RecVec SWDefs, SRDefs;
+  for (const CodeGenInstruction *Inst : Target.getInstructionsByEnumValue()) {
+    Record *SchedDef = Inst->TheDef;
+    if (SchedDef->isValueUnset("SchedRW"))
+      continue;
+    RecVec RWs = SchedDef->getValueAsListOfDefs("SchedRW");
+    for (Record *RW : RWs) {
+      if (RW->isSubClassOf("SchedWrite"))
+        scanSchedRW(RW, SWDefs, RWSet);
+      else {
+        assert(RW->isSubClassOf("SchedRead") && "Unknown SchedReadWrite");
+        scanSchedRW(RW, SRDefs, RWSet);
+      }
+    }
+  }
+  // Find all ReadWrites referenced by InstRW.
+  RecVec InstRWDefs = Records.getAllDerivedDefinitions("InstRW");
+  for (Record *InstRWDef : InstRWDefs) {
+    // For all OperandReadWrites.
+    RecVec RWDefs = InstRWDef->getValueAsListOfDefs("OperandReadWrites");
+    for (Record *RWDef : RWDefs) {
+      if (RWDef->isSubClassOf("SchedWrite"))
+        scanSchedRW(RWDef, SWDefs, RWSet);
+      else {
+        assert(RWDef->isSubClassOf("SchedRead") && "Unknown SchedReadWrite");
+        scanSchedRW(RWDef, SRDefs, RWSet);
+      }
+    }
+  }
+  // Find all ReadWrites referenced by ItinRW.
+  RecVec ItinRWDefs = Records.getAllDerivedDefinitions("ItinRW");
+  for (Record *ItinRWDef : ItinRWDefs) {
+    // For all OperandReadWrites.
+    RecVec RWDefs = ItinRWDef->getValueAsListOfDefs("OperandReadWrites");
+    for (Record *RWDef : RWDefs) {
+      if (RWDef->isSubClassOf("SchedWrite"))
+        scanSchedRW(RWDef, SWDefs, RWSet);
+      else {
+        assert(RWDef->isSubClassOf("SchedRead") && "Unknown SchedReadWrite");
+        scanSchedRW(RWDef, SRDefs, RWSet);
+      }
+    }
+  }
+  // Find all ReadWrites referenced by SchedAlias. AliasDefs needs to be sorted
+  // for the loop below that initializes Alias vectors.
+  RecVec AliasDefs = Records.getAllDerivedDefinitions("SchedAlias");
+  llvm::sort(AliasDefs, LessRecord());
+  for (Record *ADef : AliasDefs) {
+    Record *MatchDef = ADef->getValueAsDef("MatchRW");
+    Record *AliasDef = ADef->getValueAsDef("AliasRW");
+    if (MatchDef->isSubClassOf("SchedWrite")) {
+      if (!AliasDef->isSubClassOf("SchedWrite"))
+        PrintFatalError(ADef->getLoc(), "SchedWrite Alias must be SchedWrite");
+      scanSchedRW(AliasDef, SWDefs, RWSet);
+    } else {
+      assert(MatchDef->isSubClassOf("SchedRead") && "Unknown SchedReadWrite");
+      if (!AliasDef->isSubClassOf("SchedRead"))
+        PrintFatalError(ADef->getLoc(), "SchedRead Alias must be SchedRead");
+      scanSchedRW(AliasDef, SRDefs, RWSet);
+    }
+  }
+  // Sort and add the SchedReadWrites directly referenced by instructions or
+  // itinerary resources. Index reads and writes in separate domains.
+  llvm::sort(SWDefs, LessRecord());
+  for (Record *SWDef : SWDefs) {
+    assert(!getSchedRWIdx(SWDef, /*IsRead=*/false) && "duplicate SchedWrite");
+    SchedWrites.emplace_back(SchedWrites.size(), SWDef);
+  }
+  llvm::sort(SRDefs, LessRecord());
+  for (Record *SRDef : SRDefs) {
+    assert(!getSchedRWIdx(SRDef, /*IsRead-*/ true) && "duplicate SchedWrite");
+    SchedReads.emplace_back(SchedReads.size(), SRDef);
+  }
+  // Initialize WriteSequence vectors.
+  for (CodeGenSchedRW &CGRW : SchedWrites) {
+    if (!CGRW.IsSequence)
+      continue;
+    findRWs(CGRW.TheDef->getValueAsListOfDefs("Writes"), CGRW.Sequence,
+            /*IsRead=*/false);
+  }
+  // Initialize Aliases vectors.
+  for (Record *ADef : AliasDefs) {
+    Record *AliasDef = ADef->getValueAsDef("AliasRW");
+    getSchedRW(AliasDef).IsAlias = true;
+    Record *MatchDef = ADef->getValueAsDef("MatchRW");
+    CodeGenSchedRW &RW = getSchedRW(MatchDef);
+    if (RW.IsAlias)
+      PrintFatalError(ADef->getLoc(), "Cannot Alias an Alias");
+    RW.Aliases.push_back(ADef);
+  }
+  LLVM_DEBUG(
+      dbgs() << "\n+++ SCHED READS and WRITES (collectSchedRW) +++\n";
+      for (unsigned WIdx = 0, WEnd = SchedWrites.size(); WIdx != WEnd; ++WIdx) {
+        dbgs() << WIdx << ": ";
+        SchedWrites[WIdx].dump();
+        dbgs() << '\n';
+      } for (unsigned RIdx = 0, REnd = SchedReads.size(); RIdx != REnd;
+             ++RIdx) {
+        dbgs() << RIdx << ": ";
+        SchedReads[RIdx].dump();
+        dbgs() << '\n';
+      } RecVec RWDefs = Records.getAllDerivedDefinitions("SchedReadWrite");
+      for (Record *RWDef
+           : RWDefs) {
+        if (!getSchedRWIdx(RWDef, RWDef->isSubClassOf("SchedRead"))) {
+          StringRef Name = RWDef->getName();
+          if (Name != "NoWrite" && Name != "ReadDefault")
+            dbgs() << "Unused SchedReadWrite " << Name << '\n';
+        }
+      });
+}
+
+/// Compute a SchedWrite name from a sequence of writes.
+std::string CodeGenSchedModels::genRWName(ArrayRef<unsigned> Seq, bool IsRead) {
+  std::string Name("(");
+  ListSeparator LS("_");
+  for (unsigned I : Seq) {
+    Name += LS;
+    Name += getSchedRW(I, IsRead).Name;
+  }
+  Name += ')';
+  return Name;
+}
+
+unsigned CodeGenSchedModels::getSchedRWIdx(const Record *Def,
+                                           bool IsRead) const {
+  const std::vector<CodeGenSchedRW> &RWVec = IsRead ? SchedReads : SchedWrites;
+  const auto I = find_if(
+      RWVec, [Def](const CodeGenSchedRW &RW) { return RW.TheDef == Def; });
+  return I == RWVec.end() ? 0 : std::distance(RWVec.begin(), I);
+}
+
+bool CodeGenSchedModels::hasReadOfWrite(Record *WriteDef) const {
+  for (auto &ProcModel : ProcModels) {
+    const RecVec &RADefs = ProcModel.ReadAdvanceDefs;
+    for (auto &RADef : RADefs) {
+      RecVec ValidWrites = RADef->getValueAsListOfDefs("ValidWrites");
+      if (is_contained(ValidWrites, WriteDef))
+        return true;
+    }
+  }
+  return false;
+}
+
+static void splitSchedReadWrites(const RecVec &RWDefs, RecVec &WriteDefs,
+                                 RecVec &ReadDefs) {
+  for (Record *RWDef : RWDefs) {
+    if (RWDef->isSubClassOf("SchedWrite"))
+      WriteDefs.push_back(RWDef);
+    else {
+      assert(RWDef->isSubClassOf("SchedRead") && "unknown SchedReadWrite");
+      ReadDefs.push_back(RWDef);
+    }
+  }
+}
+
+// Split the SchedReadWrites defs and call findRWs for each list.
+void CodeGenSchedModels::findRWs(const RecVec &RWDefs, IdxVec &Writes,
+                                 IdxVec &Reads) const {
+  RecVec WriteDefs;
+  RecVec ReadDefs;
+  splitSchedReadWrites(RWDefs, WriteDefs, ReadDefs);
+  findRWs(WriteDefs, Writes, false);
+  findRWs(ReadDefs, Reads, true);
+}
+
+// Call getSchedRWIdx for all elements in a sequence of SchedRW defs.
+void CodeGenSchedModels::findRWs(const RecVec &RWDefs, IdxVec &RWs,
+                                 bool IsRead) const {
+  for (Record *RWDef : RWDefs) {
+    unsigned Idx = getSchedRWIdx(RWDef, IsRead);
+    assert(Idx && "failed to collect SchedReadWrite");
+    RWs.push_back(Idx);
+  }
+}
+
+void CodeGenSchedModels::expandRWSequence(unsigned RWIdx, IdxVec &RWSeq,
+                                          bool IsRead) const {
+  const CodeGenSchedRW &SchedRW = getSchedRW(RWIdx, IsRead);
+  if (!SchedRW.IsSequence) {
+    RWSeq.push_back(RWIdx);
+    return;
+  }
+  int Repeat = SchedRW.TheDef ? SchedRW.TheDef->getValueAsInt("Repeat") : 1;
+  for (int i = 0; i < Repeat; ++i) {
+    for (unsigned I : SchedRW.Sequence) {
+      expandRWSequence(I, RWSeq, IsRead);
+    }
+  }
+}
+
+// Expand a SchedWrite as a sequence following any aliases that coincide with
+// the given processor model.
+void CodeGenSchedModels::expandRWSeqForProc(
+    unsigned RWIdx, IdxVec &RWSeq, bool IsRead,
+    const CodeGenProcModel &ProcModel) const {
+
+  const CodeGenSchedRW &SchedWrite = getSchedRW(RWIdx, IsRead);
+  Record *AliasDef = nullptr;
+  for (const Record *Rec : SchedWrite.Aliases) {
+    const CodeGenSchedRW &AliasRW = getSchedRW(Rec->getValueAsDef("AliasRW"));
+    if (Rec->getValueInit("SchedModel")->isComplete()) {
+      Record *ModelDef = Rec->getValueAsDef("SchedModel");
+      if (&getProcModel(ModelDef) != &ProcModel)
+        continue;
+    }
+    if (AliasDef)
+      PrintFatalError(AliasRW.TheDef->getLoc(),
+                      "Multiple aliases "
+                      "defined for processor " +
+                          ProcModel.ModelName +
+                          " Ensure only one SchedAlias exists per RW.");
+    AliasDef = AliasRW.TheDef;
+  }
+  if (AliasDef) {
+    expandRWSeqForProc(getSchedRWIdx(AliasDef, IsRead), RWSeq, IsRead,
+                       ProcModel);
+    return;
+  }
+  if (!SchedWrite.IsSequence) {
+    RWSeq.push_back(RWIdx);
+    return;
+  }
+  int Repeat =
+      SchedWrite.TheDef ? SchedWrite.TheDef->getValueAsInt("Repeat") : 1;
+  for (int I = 0, E = Repeat; I < E; ++I) {
+    for (unsigned Idx : SchedWrite.Sequence) {
+      expandRWSeqForProc(Idx, RWSeq, IsRead, ProcModel);
+    }
+  }
+}
+
+// Find the existing SchedWrite that models this sequence of writes.
+unsigned CodeGenSchedModels::findRWForSequence(ArrayRef<unsigned> Seq,
+                                               bool IsRead) {
+  std::vector<CodeGenSchedRW> &RWVec = IsRead ? SchedReads : SchedWrites;
+
+  auto I = find_if(RWVec, [Seq](CodeGenSchedRW &RW) {
+    return ArrayRef(RW.Sequence) == Seq;
+  });
+  // Index zero reserved for invalid RW.
+  return I == RWVec.end() ? 0 : std::distance(RWVec.begin(), I);
+}
+
+/// Add this ReadWrite if it doesn't already exist.
+unsigned CodeGenSchedModels::findOrInsertRW(ArrayRef<unsigned> Seq,
+                                            bool IsRead) {
+  assert(!Seq.empty() && "cannot insert empty sequence");
+  if (Seq.size() == 1)
+    return Seq.back();
+
+  unsigned Idx = findRWForSequence(Seq, IsRead);
+  if (Idx)
+    return Idx;
+
+  std::vector<CodeGenSchedRW> &RWVec = IsRead ? SchedReads : SchedWrites;
+  unsigned RWIdx = RWVec.size();
+  CodeGenSchedRW SchedRW(RWIdx, IsRead, Seq, genRWName(Seq, IsRead));
+  RWVec.push_back(SchedRW);
+  return RWIdx;
+}
+
+/// Visit all the instruction definitions for this target to gather and
+/// enumerate the itinerary classes. These are the explicitly specified
+/// SchedClasses. More SchedClasses may be inferred.
+void CodeGenSchedModels::collectSchedClasses() {
+
+  // NoItinerary is always the first class at Idx=0
+  assert(SchedClasses.empty() && "Expected empty sched class");
+  SchedClasses.emplace_back(0, "NoInstrModel", Records.getDef("NoItinerary"));
+  SchedClasses.back().ProcIndices.push_back(0);
+
+  // Create a SchedClass for each unique combination of itinerary class and
+  // SchedRW list.
+  for (const CodeGenInstruction *Inst : Target.getInstructionsByEnumValue()) {
+    Record *ItinDef = Inst->TheDef->getValueAsDef("Itinerary");
+    IdxVec Writes, Reads;
+    if (!Inst->TheDef->isValueUnset("SchedRW"))
+      findRWs(Inst->TheDef->getValueAsListOfDefs("SchedRW"), Writes, Reads);
+
+    // ProcIdx == 0 indicates the class applies to all processors.
+    unsigned SCIdx = addSchedClass(ItinDef, Writes, Reads, /*ProcIndices*/ {0});
+    InstrClassMap[Inst->TheDef] = SCIdx;
+  }
+  // Create classes for InstRW defs.
+  RecVec InstRWDefs = Records.getAllDerivedDefinitions("InstRW");
+  llvm::sort(InstRWDefs, LessRecord());
+  LLVM_DEBUG(dbgs() << "\n+++ SCHED CLASSES (createInstRWClass) +++\n");
+  for (Record *RWDef : InstRWDefs)
+    createInstRWClass(RWDef);
+
+  NumInstrSchedClasses = SchedClasses.size();
+
+  bool EnableDump = false;
+  LLVM_DEBUG(EnableDump = true);
+  if (!EnableDump)
+    return;
+
+  LLVM_DEBUG(
+      dbgs()
+      << "\n+++ ITINERARIES and/or MACHINE MODELS (collectSchedClasses) +++\n");
+  for (const CodeGenInstruction *Inst : Target.getInstructionsByEnumValue()) {
+    StringRef InstName = Inst->TheDef->getName();
+    unsigned SCIdx = getSchedClassIdx(*Inst);
+    if (!SCIdx) {
+      LLVM_DEBUG({
+        if (!Inst->hasNoSchedulingInfo)
+          dbgs() << "No machine model for " << Inst->TheDef->getName() << '\n';
+      });
+      continue;
+    }
+    CodeGenSchedClass &SC = getSchedClass(SCIdx);
+    if (SC.ProcIndices[0] != 0)
+      PrintFatalError(Inst->TheDef->getLoc(),
+                      "Instruction's sched class "
+                      "must not be subtarget specific.");
+
+    IdxVec ProcIndices;
+    if (SC.ItinClassDef->getName() != "NoItinerary") {
+      ProcIndices.push_back(0);
+      dbgs() << "Itinerary for " << InstName << ": "
+             << SC.ItinClassDef->getName() << '\n';
+    }
+    if (!SC.Writes.empty()) {
+      ProcIndices.push_back(0);
+      LLVM_DEBUG({
+        dbgs() << "SchedRW machine model for " << InstName;
+        for (unsigned int Write : SC.Writes)
+          dbgs() << " " << SchedWrites[Write].Name;
+        for (unsigned int Read : SC.Reads)
+          dbgs() << " " << SchedReads[Read].Name;
+        dbgs() << '\n';
+      });
+    }
+    const RecVec &RWDefs = SchedClasses[SCIdx].InstRWs;
+    for (Record *RWDef : RWDefs) {
+      const CodeGenProcModel &ProcModel =
+          getProcModel(RWDef->getValueAsDef("SchedModel"));
+      ProcIndices.push_back(ProcModel.Index);
+      LLVM_DEBUG(dbgs() << "InstRW on " << ProcModel.ModelName << " for "
+                        << InstName);
+      IdxVec Writes;
+      IdxVec Reads;
+      findRWs(RWDef->getValueAsListOfDefs("OperandReadWrites"), Writes, Reads);
+      LLVM_DEBUG({
+        for (unsigned WIdx : Writes)
+          dbgs() << " " << SchedWrites[WIdx].Name;
+        for (unsigned RIdx : Reads)
+          dbgs() << " " << SchedReads[RIdx].Name;
+        dbgs() << '\n';
+      });
+    }
+    // If ProcIndices contains zero, the class applies to all processors.
+    LLVM_DEBUG({
+      if (!llvm::is_contained(ProcIndices, 0)) {
+        for (const CodeGenProcModel &PM : ProcModels) {
+          if (!llvm::is_contained(ProcIndices, PM.Index))
+            dbgs() << "No machine model for " << Inst->TheDef->getName()
+                   << " on processor " << PM.ModelName << '\n';
+        }
+      }
+    });
+  }
+}
+
+// Get the SchedClass index for an instruction.
+unsigned
+CodeGenSchedModels::getSchedClassIdx(const CodeGenInstruction &Inst) const {
+  return InstrClassMap.lookup(Inst.TheDef);
+}
+
+std::string
+CodeGenSchedModels::createSchedClassName(Record *ItinClassDef,
+                                         ArrayRef<unsigned> OperWrites,
+                                         ArrayRef<unsigned> OperReads) {
+
+  std::string Name;
+  if (ItinClassDef && ItinClassDef->getName() != "NoItinerary")
+    Name = std::string(ItinClassDef->getName());
+  for (unsigned Idx : OperWrites) {
+    if (!Name.empty())
+      Name += '_';
+    Name += SchedWrites[Idx].Name;
+  }
+  for (unsigned Idx : OperReads) {
+    Name += '_';
+    Name += SchedReads[Idx].Name;
+  }
+  return Name;
+}
+
+std::string CodeGenSchedModels::createSchedClassName(const RecVec &InstDefs) {
+
+  std::string Name;
+  ListSeparator LS("_");
+  for (const Record *InstDef : InstDefs) {
+    Name += LS;
+    Name += InstDef->getName();
+  }
+  return Name;
+}
+
+/// Add an inferred sched class from an itinerary class and per-operand list of
+/// SchedWrites and SchedReads. ProcIndices contains the set of IDs of
+/// processors that may utilize this class.
+unsigned CodeGenSchedModels::addSchedClass(Record *ItinClassDef,
+                                           ArrayRef<unsigned> OperWrites,
+                                           ArrayRef<unsigned> OperReads,
+                                           ArrayRef<unsigned> ProcIndices) {
+  assert(!ProcIndices.empty() && "expect at least one ProcIdx");
+
+  auto IsKeyEqual = [=](const CodeGenSchedClass &SC) {
+    return SC.isKeyEqual(ItinClassDef, OperWrites, OperReads);
+  };
+
+  auto I = find_if(make_range(schedClassBegin(), schedClassEnd()), IsKeyEqual);
+  unsigned Idx = I == schedClassEnd() ? 0 : std::distance(schedClassBegin(), I);
+  if (Idx || SchedClasses[0].isKeyEqual(ItinClassDef, OperWrites, OperReads)) {
+    IdxVec PI;
+    std::set_union(SchedClasses[Idx].ProcIndices.begin(),
+                   SchedClasses[Idx].ProcIndices.end(), ProcIndices.begin(),
+                   ProcIndices.end(), std::back_inserter(PI));
+    SchedClasses[Idx].ProcIndices = std::move(PI);
+    return Idx;
+  }
+  Idx = SchedClasses.size();
+  SchedClasses.emplace_back(
+      Idx, createSchedClassName(ItinClassDef, OperWrites, OperReads),
+      ItinClassDef);
+  CodeGenSchedClass &SC = SchedClasses.back();
+  SC.Writes = OperWrites;
+  SC.Reads = OperReads;
+  SC.ProcIndices = ProcIndices;
+
+  return Idx;
+}
+
+// Create classes for each set of opcodes that are in the same InstReadWrite
+// definition across all processors.
+void CodeGenSchedModels::createInstRWClass(Record *InstRWDef) {
+  // ClassInstrs will hold an entry for each subset of Instrs in InstRWDef that
+  // intersects with an existing class via a previous InstRWDef. Instrs that do
+  // not intersect with an existing class refer back to their former class as
+  // determined from ItinDef or SchedRW.
+  SmallMapVector<unsigned, SmallVector<Record *, 8>, 4> ClassInstrs;
+  // Sort Instrs into sets.
+  const RecVec *InstDefs = Sets.expand(InstRWDef);
+  if (InstDefs->empty())
+    PrintFatalError(InstRWDef->getLoc(), "No matching instruction opcodes");
+
+  for (Record *InstDef : *InstDefs) {
+    InstClassMapTy::const_iterator Pos = InstrClassMap.find(InstDef);
+    if (Pos == InstrClassMap.end())
+      PrintFatalError(InstDef->getLoc(), "No sched class for instruction.");
+    unsigned SCIdx = Pos->second;
+    ClassInstrs[SCIdx].push_back(InstDef);
+  }
+  // For each set of Instrs, create a new class if necessary, and map or remap
+  // the Instrs to it.
+  for (auto &Entry : ClassInstrs) {
+    unsigned OldSCIdx = Entry.first;
+    ArrayRef<Record *> InstDefs = Entry.second;
+    // If the all instrs in the current class are accounted for, then leave
+    // them mapped to their old class.
+    if (OldSCIdx) {
+      const RecVec &RWDefs = SchedClasses[OldSCIdx].InstRWs;
+      if (!RWDefs.empty()) {
+        const RecVec *OrigInstDefs = Sets.expand(RWDefs[0]);
+        unsigned OrigNumInstrs = count_if(*OrigInstDefs, [&](Record *OIDef) {
+          return InstrClassMap[OIDef] == OldSCIdx;
+        });
+        if (OrigNumInstrs == InstDefs.size()) {
+          assert(SchedClasses[OldSCIdx].ProcIndices[0] == 0 &&
+                 "expected a generic SchedClass");
+          Record *RWModelDef = InstRWDef->getValueAsDef("SchedModel");
+          // Make sure we didn't already have a InstRW containing this
+          // instruction on this model.
+          for (Record *RWD : RWDefs) {
+            if (RWD->getValueAsDef("SchedModel") == RWModelDef &&
+                RWModelDef->getValueAsBit("FullInstRWOverlapCheck")) {
+              assert(!InstDefs.empty()); // Checked at function start.
+              PrintError(
+                  InstRWDef->getLoc(),
+                  "Overlapping InstRW definition for \"" +
+                      InstDefs.front()->getName() +
+                      "\" also matches previous \"" +
+                      RWD->getValue("Instrs")->getValue()->getAsString() +
+                      "\".");
+              PrintFatalNote(RWD->getLoc(), "Previous match was here.");
+            }
+          }
+          LLVM_DEBUG(dbgs() << "InstRW: Reuse SC " << OldSCIdx << ":"
+                            << SchedClasses[OldSCIdx].Name << " on "
+                            << RWModelDef->getName() << "\n");
+          SchedClasses[OldSCIdx].InstRWs.push_back(InstRWDef);
+          continue;
+        }
+      }
+    }
+    unsigned SCIdx = SchedClasses.size();
+    SchedClasses.emplace_back(SCIdx, createSchedClassName(InstDefs), nullptr);
+    CodeGenSchedClass &SC = SchedClasses.back();
+    LLVM_DEBUG(dbgs() << "InstRW: New SC " << SCIdx << ":" << SC.Name << " on "
+                      << InstRWDef->getValueAsDef("SchedModel")->getName()
+                      << "\n");
+
+    // Preserve ItinDef and Writes/Reads for processors without an InstRW entry.
+    SC.ItinClassDef = SchedClasses[OldSCIdx].ItinClassDef;
+    SC.Writes = SchedClasses[OldSCIdx].Writes;
+    SC.Reads = SchedClasses[OldSCIdx].Reads;
+    SC.ProcIndices.push_back(0);
+    // If we had an old class, copy it's InstRWs to this new class.
+    if (OldSCIdx) {
+      Record *RWModelDef = InstRWDef->getValueAsDef("SchedModel");
+      for (Record *OldRWDef : SchedClasses[OldSCIdx].InstRWs) {
+        if (OldRWDef->getValueAsDef("SchedModel") == RWModelDef) {
+          assert(!InstDefs.empty()); // Checked at function start.
+          PrintError(
+              InstRWDef->getLoc(),
+              "Overlapping InstRW definition for \"" +
+                  InstDefs.front()->getName() + "\" also matches previous \"" +
+                  OldRWDef->getValue("Instrs")->getValue()->getAsString() +
+                  "\".");
+          PrintFatalNote(OldRWDef->getLoc(), "Previous match was here.");
+        }
+        assert(OldRWDef != InstRWDef && "SchedClass has duplicate InstRW def");
+        SC.InstRWs.push_back(OldRWDef);
+      }
+    }
+    // Map each Instr to this new class.
+    for (Record *InstDef : InstDefs)
+      InstrClassMap[InstDef] = SCIdx;
+    SC.InstRWs.push_back(InstRWDef);
+  }
+}
+
+// True if collectProcItins found anything.
+bool CodeGenSchedModels::hasItineraries() const {
+  for (const CodeGenProcModel &PM :
+       make_range(procModelBegin(), procModelEnd()))
+    if (PM.hasItineraries())
+      return true;
+  return false;
+}
+
+// Gather the processor itineraries.
+void CodeGenSchedModels::collectProcItins() {
+  LLVM_DEBUG(dbgs() << "\n+++ PROBLEM ITINERARIES (collectProcItins) +++\n");
+  for (CodeGenProcModel &ProcModel : ProcModels) {
+    if (!ProcModel.hasItineraries())
+      continue;
+
+    RecVec ItinRecords = ProcModel.ItinsDef->getValueAsListOfDefs("IID");
+    assert(!ItinRecords.empty() && "ProcModel.hasItineraries is incorrect");
+
+    // Populate ItinDefList with Itinerary records.
+    ProcModel.ItinDefList.resize(NumInstrSchedClasses);
+
+    // Insert each itinerary data record in the correct position within
+    // the processor model's ItinDefList.
+    for (Record *ItinData : ItinRecords) {
+      const Record *ItinDef = ItinData->getValueAsDef("TheClass");
+      bool FoundClass = false;
+
+      for (const CodeGenSchedClass &SC :
+           make_range(schedClassBegin(), schedClassEnd())) {
+        // Multiple SchedClasses may share an itinerary. Update all of them.
+        if (SC.ItinClassDef == ItinDef) {
+          ProcModel.ItinDefList[SC.Index] = ItinData;
+          FoundClass = true;
+        }
+      }
+      if (!FoundClass) {
+        LLVM_DEBUG(dbgs() << ProcModel.ItinsDef->getName()
+                          << " missing class for itinerary "
+                          << ItinDef->getName() << '\n');
+      }
+    }
+    // Check for missing itinerary entries.
+    assert(!ProcModel.ItinDefList[0] && "NoItinerary class can't have rec");
+    LLVM_DEBUG(
+        for (unsigned i = 1, N = ProcModel.ItinDefList.size(); i < N; ++i) {
+          if (!ProcModel.ItinDefList[i])
+            dbgs() << ProcModel.ItinsDef->getName()
+                   << " missing itinerary for class " << SchedClasses[i].Name
+                   << '\n';
+        });
+  }
+}
+
+// Gather the read/write types for each itinerary class.
+void CodeGenSchedModels::collectProcItinRW() {
+  RecVec ItinRWDefs = Records.getAllDerivedDefinitions("ItinRW");
+  llvm::sort(ItinRWDefs, LessRecord());
+  for (Record *RWDef : ItinRWDefs) {
+    if (!RWDef->getValueInit("SchedModel")->isComplete())
+      PrintFatalError(RWDef->getLoc(), "SchedModel is undefined");
+    Record *ModelDef = RWDef->getValueAsDef("SchedModel");
+    ProcModelMapTy::const_iterator I = ProcModelMap.find(ModelDef);
+    if (I == ProcModelMap.end()) {
+      PrintFatalError(RWDef->getLoc(),
+                      "Undefined SchedMachineModel " + ModelDef->getName());
+    }
+    ProcModels[I->second].ItinRWDefs.push_back(RWDef);
+  }
+}
+
+// Gather the unsupported features for processor models.
+void CodeGenSchedModels::collectProcUnsupportedFeatures() {
+  for (CodeGenProcModel &ProcModel : ProcModels)
+    append_range(
+        ProcModel.UnsupportedFeaturesDefs,
+        ProcModel.ModelDef->getValueAsListOfDefs("UnsupportedFeatures"));
+}
+
+/// Infer new classes from existing classes. In the process, this may create new
+/// SchedWrites from sequences of existing SchedWrites.
+void CodeGenSchedModels::inferSchedClasses() {
+  LLVM_DEBUG(
+      dbgs() << "\n+++ INFERRING SCHED CLASSES (inferSchedClasses) +++\n");
+  LLVM_DEBUG(dbgs() << NumInstrSchedClasses << " instr sched classes.\n");
+
+  // Visit all existing classes and newly created classes.
+  for (unsigned Idx = 0; Idx != SchedClasses.size(); ++Idx) {
+    assert(SchedClasses[Idx].Index == Idx && "bad SCIdx");
+
+    if (SchedClasses[Idx].ItinClassDef)
+      inferFromItinClass(SchedClasses[Idx].ItinClassDef, Idx);
+    if (!SchedClasses[Idx].InstRWs.empty())
+      inferFromInstRWs(Idx);
+    if (!SchedClasses[Idx].Writes.empty()) {
+      inferFromRW(SchedClasses[Idx].Writes, SchedClasses[Idx].Reads, Idx,
+                  SchedClasses[Idx].ProcIndices);
+    }
+    assert(SchedClasses.size() < (NumInstrSchedClasses * 6) &&
+           "too many SchedVariants");
+  }
+}
+
+/// Infer classes from per-processor itinerary resources.
+void CodeGenSchedModels::inferFromItinClass(Record *ItinClassDef,
+                                            unsigned FromClassIdx) {
+  for (unsigned PIdx = 0, PEnd = ProcModels.size(); PIdx != PEnd; ++PIdx) {
+    const CodeGenProcModel &PM = ProcModels[PIdx];
+    // For all ItinRW entries.
+    bool HasMatch = false;
+    for (const Record *Rec : PM.ItinRWDefs) {
+      RecVec Matched = Rec->getValueAsListOfDefs("MatchedItinClasses");
+      if (!llvm::is_contained(Matched, ItinClassDef))
+        continue;
+      if (HasMatch)
+        PrintFatalError(Rec->getLoc(),
+                        "Duplicate itinerary class " + ItinClassDef->getName() +
+                            " in ItinResources for " + PM.ModelName);
+      HasMatch = true;
+      IdxVec Writes, Reads;
+      findRWs(Rec->getValueAsListOfDefs("OperandReadWrites"), Writes, Reads);
+      inferFromRW(Writes, Reads, FromClassIdx, PIdx);
+    }
+  }
+}
+
+/// Infer classes from per-processor InstReadWrite definitions.
+void CodeGenSchedModels::inferFromInstRWs(unsigned SCIdx) {
+  for (unsigned I = 0, E = SchedClasses[SCIdx].InstRWs.size(); I != E; ++I) {
+    assert(SchedClasses[SCIdx].InstRWs.size() == E && "InstrRWs was mutated!");
+    Record *Rec = SchedClasses[SCIdx].InstRWs[I];
+    const RecVec *InstDefs = Sets.expand(Rec);
+    RecIter II = InstDefs->begin(), IE = InstDefs->end();
+    for (; II != IE; ++II) {
+      if (InstrClassMap[*II] == SCIdx)
+        break;
+    }
+    // If this class no longer has any instructions mapped to it, it has become
+    // irrelevant.
+    if (II == IE)
+      continue;
+    IdxVec Writes, Reads;
+    findRWs(Rec->getValueAsListOfDefs("OperandReadWrites"), Writes, Reads);
+    unsigned PIdx = getProcModel(Rec->getValueAsDef("SchedModel")).Index;
+    inferFromRW(Writes, Reads, SCIdx, PIdx); // May mutate SchedClasses.
+    SchedClasses[SCIdx].InstRWProcIndices.insert(PIdx);
+  }
+}
+
+namespace {
+
+// Helper for substituteVariantOperand.
+struct TransVariant {
+  Record *VarOrSeqDef;  // Variant or sequence.
+  unsigned RWIdx;       // Index of this variant or sequence's matched type.
+  unsigned ProcIdx;     // Processor model index or zero for any.
+  unsigned TransVecIdx; // Index into PredTransitions::TransVec.
+
+  TransVariant(Record *def, unsigned rwi, unsigned pi, unsigned ti)
+      : VarOrSeqDef(def), RWIdx(rwi), ProcIdx(pi), TransVecIdx(ti) {}
+};
+
+// Associate a predicate with the SchedReadWrite that it guards.
+// RWIdx is the index of the read/write variant.
+struct PredCheck {
+  bool IsRead;
+  unsigned RWIdx;
+  Record *Predicate;
+
+  PredCheck(bool r, unsigned w, Record *p)
+      : IsRead(r), RWIdx(w), Predicate(p) {}
+};
+
+// A Predicate transition is a list of RW sequences guarded by a PredTerm.
+struct PredTransition {
+  // A predicate term is a conjunction of PredChecks.
+  SmallVector<PredCheck, 4> PredTerm;
+  SmallVector<SmallVector<unsigned, 4>, 16> WriteSequences;
+  SmallVector<SmallVector<unsigned, 4>, 16> ReadSequences;
+  unsigned ProcIndex = 0;
+
+  PredTransition() = default;
+  PredTransition(ArrayRef<PredCheck> PT, unsigned ProcId) {
+    PredTerm.assign(PT.begin(), PT.end());
+    ProcIndex = ProcId;
+  }
+};
+
+// Encapsulate a set of partially constructed transitions.
+// The results are built by repeated calls to substituteVariants.
+class PredTransitions {
+  CodeGenSchedModels &SchedModels;
+
+public:
+  std::vector<PredTransition> TransVec;
+
+  PredTransitions(CodeGenSchedModels &sm) : SchedModels(sm) {}
+
+  bool substituteVariantOperand(const SmallVectorImpl<unsigned> &RWSeq,
+                                bool IsRead, unsigned StartIdx);
+
+  bool substituteVariants(const PredTransition &Trans);
+
+#ifndef NDEBUG
+  void dump() const;
+#endif
+
+private:
+  bool mutuallyExclusive(Record *PredDef, ArrayRef<Record *> Preds,
+                         ArrayRef<PredCheck> Term);
+  void getIntersectingVariants(const CodeGenSchedRW &SchedRW, unsigned TransIdx,
+                               std::vector<TransVariant> &IntersectingVariants);
+  void pushVariant(const TransVariant &VInfo, bool IsRead);
+};
+
+} // end anonymous namespace
+
+// Return true if this predicate is mutually exclusive with a PredTerm. This
+// degenerates into checking if the predicate is mutually exclusive with any
+// predicate in the Term's conjunction.
+//
+// All predicates associated with a given SchedRW are considered mutually
+// exclusive. This should work even if the conditions expressed by the
+// predicates are not exclusive because the predicates for a given SchedWrite
+// are always checked in the order they are defined in the .td file. Later
+// conditions implicitly negate any prior condition.
+bool PredTransitions::mutuallyExclusive(Record *PredDef,
+                                        ArrayRef<Record *> Preds,
+                                        ArrayRef<PredCheck> Term) {
+  for (const PredCheck &PC : Term) {
+    if (PC.Predicate == PredDef)
+      return false;
+
+    const CodeGenSchedRW &SchedRW = SchedModels.getSchedRW(PC.RWIdx, PC.IsRead);
+    assert(SchedRW.HasVariants && "PredCheck must refer to a SchedVariant");
+    RecVec Variants = SchedRW.TheDef->getValueAsListOfDefs("Variants");
+    if (any_of(Variants, [PredDef](const Record *R) {
+          return R->getValueAsDef("Predicate") == PredDef;
+        })) {
+      // To check if PredDef is mutually exclusive with PC we also need to
+      // check that PC.Predicate is exclusive with all predicates from variant
+      // we're expanding. Consider following RW sequence with two variants
+      // (1 & 2), where A, B and C are predicates from corresponding SchedVars:
+      //
+      // 1:A/B - 2:C/B
+      //
+      // Here C is not mutually exclusive with variant (1), because A doesn't
+      // exist in variant (2). This means we have possible transitions from A
+      // to C and from A to B, and fully expanded sequence would look like:
+      //
+      // if (A & C) return ...;
+      // if (A & B) return ...;
+      // if (B) return ...;
+      //
+      // Now let's consider another sequence:
+      //
+      // 1:A/B - 2:A/B
+      //
+      // Here A in variant (2) is mutually exclusive with variant (1), because
+      // A also exists in (2). This means A->B transition is impossible and
+      // expanded sequence would look like:
+      //
+      // if (A) return ...;
+      // if (B) return ...;
+      if (!llvm::is_contained(Preds, PC.Predicate))
+        continue;
+      return true;
+    }
+  }
+  return false;
+}
+
+static std::vector<Record *> getAllPredicates(ArrayRef<TransVariant> Variants,
+                                              unsigned ProcId) {
+  std::vector<Record *> Preds;
+  for (auto &Variant : Variants) {
+    if (!Variant.VarOrSeqDef->isSubClassOf("SchedVar"))
+      continue;
+    Preds.push_back(Variant.VarOrSeqDef->getValueAsDef("Predicate"));
+  }
+  return Preds;
+}
+
+// Populate IntersectingVariants with any variants or aliased sequences of the
+// given SchedRW whose processor indices and predicates are not mutually
+// exclusive with the given transition.
+void PredTransitions::getIntersectingVariants(
+    const CodeGenSchedRW &SchedRW, unsigned TransIdx,
+    std::vector<TransVariant> &IntersectingVariants) {
+
+  bool GenericRW = false;
+
+  std::vector<TransVariant> Variants;
+  if (SchedRW.HasVariants) {
+    unsigned VarProcIdx = 0;
+    if (SchedRW.TheDef->getValueInit("SchedModel")->isComplete()) {
+      Record *ModelDef = SchedRW.TheDef->getValueAsDef("SchedModel");
+      VarProcIdx = SchedModels.getProcModel(ModelDef).Index;
+    }
+    if (VarProcIdx == 0 || VarProcIdx == TransVec[TransIdx].ProcIndex) {
+      // Push each variant. Assign TransVecIdx later.
+      const RecVec VarDefs = SchedRW.TheDef->getValueAsListOfDefs("Variants");
+      for (Record *VarDef : VarDefs)
+        Variants.emplace_back(VarDef, SchedRW.Index, VarProcIdx, 0);
+      if (VarProcIdx == 0)
+        GenericRW = true;
+    }
+  }
+  for (RecIter AI = SchedRW.Aliases.begin(), AE = SchedRW.Aliases.end();
+       AI != AE; ++AI) {
+    // If either the SchedAlias itself or the SchedReadWrite that it aliases
+    // to is defined within a processor model, constrain all variants to
+    // that processor.
+    unsigned AliasProcIdx = 0;
+    if ((*AI)->getValueInit("SchedModel")->isComplete()) {
+      Record *ModelDef = (*AI)->getValueAsDef("SchedModel");
+      AliasProcIdx = SchedModels.getProcModel(ModelDef).Index;
+    }
+    if (AliasProcIdx && AliasProcIdx != TransVec[TransIdx].ProcIndex)
+      continue;
+    if (!Variants.empty()) {
+      const CodeGenProcModel &PM =
+          *(SchedModels.procModelBegin() + AliasProcIdx);
+      PrintFatalError((*AI)->getLoc(),
+                      "Multiple variants defined for processor " +
+                          PM.ModelName +
+                          " Ensure only one SchedAlias exists per RW.");
+    }
+
+    const CodeGenSchedRW &AliasRW =
+        SchedModels.getSchedRW((*AI)->getValueAsDef("AliasRW"));
+
+    if (AliasRW.HasVariants) {
+      const RecVec VarDefs = AliasRW.TheDef->getValueAsListOfDefs("Variants");
+      for (Record *VD : VarDefs)
+        Variants.emplace_back(VD, AliasRW.Index, AliasProcIdx, 0);
+    }
+    if (AliasRW.IsSequence)
+      Variants.emplace_back(AliasRW.TheDef, SchedRW.Index, AliasProcIdx, 0);
+    if (AliasProcIdx == 0)
+      GenericRW = true;
+  }
+  std::vector<Record *> AllPreds =
+      getAllPredicates(Variants, TransVec[TransIdx].ProcIndex);
+  for (TransVariant &Variant : Variants) {
+    // Don't expand variants if the processor models don't intersect.
+    // A zero processor index means any processor.
+    if (Variant.VarOrSeqDef->isSubClassOf("SchedVar")) {
+      Record *PredDef = Variant.VarOrSeqDef->getValueAsDef("Predicate");
+      if (mutuallyExclusive(PredDef, AllPreds, TransVec[TransIdx].PredTerm))
+        continue;
+    }
+
+    if (IntersectingVariants.empty()) {
+      // The first variant builds on the existing transition.
+      Variant.TransVecIdx = TransIdx;
+      IntersectingVariants.push_back(Variant);
+    } else {
+      // Push another copy of the current transition for more variants.
+      Variant.TransVecIdx = TransVec.size();
+      IntersectingVariants.push_back(Variant);
+      TransVec.push_back(TransVec[TransIdx]);
+    }
+  }
+  if (GenericRW && IntersectingVariants.empty()) {
+    PrintFatalError(SchedRW.TheDef->getLoc(),
+                    "No variant of this type has "
+                    "a matching predicate on any processor");
+  }
+}
+
+// Push the Reads/Writes selected by this variant onto the PredTransition
+// specified by VInfo.
+void PredTransitions::pushVariant(const TransVariant &VInfo, bool IsRead) {
+  PredTransition &Trans = TransVec[VInfo.TransVecIdx];
+
+  // If this operand transition is reached through a processor-specific alias,
+  // then the whole transition is specific to this processor.
+  IdxVec SelectedRWs;
+  if (VInfo.VarOrSeqDef->isSubClassOf("SchedVar")) {
+    Record *PredDef = VInfo.VarOrSeqDef->getValueAsDef("Predicate");
+    Trans.PredTerm.emplace_back(IsRead, VInfo.RWIdx, PredDef);
+    RecVec SelectedDefs = VInfo.VarOrSeqDef->getValueAsListOfDefs("Selected");
+    SchedModels.findRWs(SelectedDefs, SelectedRWs, IsRead);
+  } else {
+    assert(VInfo.VarOrSeqDef->isSubClassOf("WriteSequence") &&
+           "variant must be a SchedVariant or aliased WriteSequence");
+    SelectedRWs.push_back(SchedModels.getSchedRWIdx(VInfo.VarOrSeqDef, IsRead));
+  }
+
+  const CodeGenSchedRW &SchedRW = SchedModels.getSchedRW(VInfo.RWIdx, IsRead);
+
+  SmallVectorImpl<SmallVector<unsigned, 4>> &RWSequences =
+      IsRead ? Trans.ReadSequences : Trans.WriteSequences;
+  if (SchedRW.IsVariadic) {
+    unsigned OperIdx = RWSequences.size() - 1;
+    // Make N-1 copies of this transition's last sequence.
+    RWSequences.reserve(RWSequences.size() + SelectedRWs.size() - 1);
+    RWSequences.insert(RWSequences.end(), SelectedRWs.size() - 1,
+                       RWSequences[OperIdx]);
+    // Push each of the N elements of the SelectedRWs onto a copy of the last
+    // sequence (split the current operand into N operands).
+    // Note that write sequences should be expanded within this loop--the entire
+    // sequence belongs to a single operand.
+    for (IdxIter RWI = SelectedRWs.begin(), RWE = SelectedRWs.end(); RWI != RWE;
+         ++RWI, ++OperIdx) {
+      IdxVec ExpandedRWs;
+      if (IsRead)
+        ExpandedRWs.push_back(*RWI);
+      else
+        SchedModels.expandRWSequence(*RWI, ExpandedRWs, IsRead);
+      llvm::append_range(RWSequences[OperIdx], ExpandedRWs);
+    }
+    assert(OperIdx == RWSequences.size() && "missed a sequence");
+  } else {
+    // Push this transition's expanded sequence onto this transition's last
+    // sequence (add to the current operand's sequence).
+    SmallVectorImpl<unsigned> &Seq = RWSequences.back();
+    IdxVec ExpandedRWs;
+    for (unsigned int SelectedRW : SelectedRWs) {
+      if (IsRead)
+        ExpandedRWs.push_back(SelectedRW);
+      else
+        SchedModels.expandRWSequence(SelectedRW, ExpandedRWs, IsRead);
+    }
+    llvm::append_range(Seq, ExpandedRWs);
+  }
+}
+
+// RWSeq is a sequence of all Reads or all Writes for the next read or write
+// operand. StartIdx is an index into TransVec where partial results
+// starts. RWSeq must be applied to all transitions between StartIdx and the end
+// of TransVec.
+bool PredTransitions::substituteVariantOperand(
+    const SmallVectorImpl<unsigned> &RWSeq, bool IsRead, unsigned StartIdx) {
+  bool Subst = false;
+  // Visit each original RW within the current sequence.
+  for (unsigned int RWI : RWSeq) {
+    const CodeGenSchedRW &SchedRW = SchedModels.getSchedRW(RWI, IsRead);
+    // Push this RW on all partial PredTransitions or distribute variants.
+    // New PredTransitions may be pushed within this loop which should not be
+    // revisited (TransEnd must be loop invariant).
+    for (unsigned TransIdx = StartIdx, TransEnd = TransVec.size();
+         TransIdx != TransEnd; ++TransIdx) {
+      // Distribute this partial PredTransition across intersecting variants.
+      // This will push a copies of TransVec[TransIdx] on the back of TransVec.
+      std::vector<TransVariant> IntersectingVariants;
+      getIntersectingVariants(SchedRW, TransIdx, IntersectingVariants);
+      // Now expand each variant on top of its copy of the transition.
+      for (const TransVariant &IV : IntersectingVariants)
+        pushVariant(IV, IsRead);
+      if (IntersectingVariants.empty()) {
+        if (IsRead)
+          TransVec[TransIdx].ReadSequences.back().push_back(RWI);
+        else
+          TransVec[TransIdx].WriteSequences.back().push_back(RWI);
+        continue;
+      } else {
+        Subst = true;
+      }
+    }
+  }
+  return Subst;
+}
+
+// For each variant of a Read/Write in Trans, substitute the sequence of
+// Read/Writes guarded by the variant. This is exponential in the number of
+// variant Read/Writes, but in practice detection of mutually exclusive
+// predicates should result in linear growth in the total number variants.
+//
+// This is one step in a breadth-first search of nested variants.
+bool PredTransitions::substituteVariants(const PredTransition &Trans) {
+  // Build up a set of partial results starting at the back of
+  // PredTransitions. Remember the first new transition.
+  unsigned StartIdx = TransVec.size();
+  bool Subst = false;
+  assert(Trans.ProcIndex != 0);
+  TransVec.emplace_back(Trans.PredTerm, Trans.ProcIndex);
+
+  // Visit each original write sequence.
+  for (const auto &WriteSequence : Trans.WriteSequences) {
+    // Push a new (empty) write sequence onto all partial Transitions.
+    for (std::vector<PredTransition>::iterator I = TransVec.begin() + StartIdx,
+                                               E = TransVec.end();
+         I != E; ++I) {
+      I->WriteSequences.emplace_back();
+    }
+    Subst |=
+        substituteVariantOperand(WriteSequence, /*IsRead=*/false, StartIdx);
+  }
+  // Visit each original read sequence.
+  for (const auto &ReadSequence : Trans.ReadSequences) {
+    // Push a new (empty) read sequence onto all partial Transitions.
+    for (std::vector<PredTransition>::iterator I = TransVec.begin() + StartIdx,
+                                               E = TransVec.end();
+         I != E; ++I) {
+      I->ReadSequences.emplace_back();
+    }
+    Subst |= substituteVariantOperand(ReadSequence, /*IsRead=*/true, StartIdx);
+  }
+  return Subst;
+}
+
+static void addSequences(CodeGenSchedModels &SchedModels,
+                         const SmallVectorImpl<SmallVector<unsigned, 4>> &Seqs,
+                         IdxVec &Result, bool IsRead) {
+  for (const auto &S : Seqs)
+    if (!S.empty())
+      Result.push_back(SchedModels.findOrInsertRW(S, IsRead));
+}
+
+#ifndef NDEBUG
+static void dumpRecVec(const RecVec &RV) {
+  for (const Record *R : RV)
+    dbgs() << R->getName() << ", ";
+}
+#endif
+
+static void dumpTransition(const CodeGenSchedModels &SchedModels,
+                           const CodeGenSchedClass &FromSC,
+                           const CodeGenSchedTransition &SCTrans,
+                           const RecVec &Preds) {
+  LLVM_DEBUG(dbgs() << "Adding transition from " << FromSC.Name << "("
+                    << FromSC.Index << ") to "
+                    << SchedModels.getSchedClass(SCTrans.ToClassIdx).Name << "("
+                    << SCTrans.ToClassIdx << ") on pred term: (";
+             dumpRecVec(Preds);
+             dbgs() << ") on processor (" << SCTrans.ProcIndex << ")\n");
+}
+// Create a new SchedClass for each variant found by inferFromRW. Pass
+static void inferFromTransitions(ArrayRef<PredTransition> LastTransitions,
+                                 unsigned FromClassIdx,
+                                 CodeGenSchedModels &SchedModels) {
+  // For each PredTransition, create a new CodeGenSchedTransition, which usually
+  // requires creating a new SchedClass.
+  for (const auto &LastTransition : LastTransitions) {
+    // Variant expansion (substituteVariants) may create unconditional
+    // transitions. We don't need to build sched classes for them.
+    if (LastTransition.PredTerm.empty())
+      continue;
+    IdxVec OperWritesVariant, OperReadsVariant;
+    addSequences(SchedModels, LastTransition.WriteSequences, OperWritesVariant,
+                 false);
+    addSequences(SchedModels, LastTransition.ReadSequences, OperReadsVariant,
+                 true);
+    CodeGenSchedTransition SCTrans;
+
+    // Transition should not contain processor indices already assigned to
+    // InstRWs in this scheduling class.
+    const CodeGenSchedClass &FromSC = SchedModels.getSchedClass(FromClassIdx);
+    if (FromSC.InstRWProcIndices.count(LastTransition.ProcIndex))
+      continue;
+    SCTrans.ProcIndex = LastTransition.ProcIndex;
+    SCTrans.ToClassIdx =
+        SchedModels.addSchedClass(/*ItinClassDef=*/nullptr, OperWritesVariant,
+                                  OperReadsVariant, LastTransition.ProcIndex);
+
+    // The final PredTerm is unique set of predicates guarding the transition.
+    RecVec Preds;
+    transform(LastTransition.PredTerm, std::back_inserter(Preds),
+              [](const PredCheck &P) { return P.Predicate; });
+    Preds.erase(std::unique(Preds.begin(), Preds.end()), Preds.end());
+    dumpTransition(SchedModels, FromSC, SCTrans, Preds);
+    SCTrans.PredTerm = std::move(Preds);
+    SchedModels.getSchedClass(FromClassIdx)
+        .Transitions.push_back(std::move(SCTrans));
+  }
+}
+
+std::vector<unsigned> CodeGenSchedModels::getAllProcIndices() const {
+  std::vector<unsigned> ProcIdVec;
+  for (const auto &PM : ProcModelMap)
+    if (PM.second != 0)
+      ProcIdVec.push_back(PM.second);
+  // The order of the keys (Record pointers) of ProcModelMap are not stable.
+  // Sort to stabalize the values.
+  llvm::sort(ProcIdVec);
+  return ProcIdVec;
+}
+
+static std::vector<PredTransition>
+makePerProcessorTransitions(const PredTransition &Trans,
+                            ArrayRef<unsigned> ProcIndices) {
+  std::vector<PredTransition> PerCpuTransVec;
+  for (unsigned ProcId : ProcIndices) {
+    assert(ProcId != 0);
+    PerCpuTransVec.push_back(Trans);
+    PerCpuTransVec.back().ProcIndex = ProcId;
+  }
+  return PerCpuTransVec;
+}
+
+// Create new SchedClasses for the given ReadWrite list. If any of the
+// ReadWrites refers to a SchedVariant, create a new SchedClass for each variant
+// of the ReadWrite list, following Aliases if necessary.
+void CodeGenSchedModels::inferFromRW(ArrayRef<unsigned> OperWrites,
+                                     ArrayRef<unsigned> OperReads,
+                                     unsigned FromClassIdx,
+                                     ArrayRef<unsigned> ProcIndices) {
+  LLVM_DEBUG(dbgs() << "INFER RW proc("; dumpIdxVec(ProcIndices);
+             dbgs() << ") ");
+  // Create a seed transition with an empty PredTerm and the expanded sequences
+  // of SchedWrites for the current SchedClass.
+  std::vector<PredTransition> LastTransitions;
+  LastTransitions.emplace_back();
+
+  for (unsigned WriteIdx : OperWrites) {
+    IdxVec WriteSeq;
+    expandRWSequence(WriteIdx, WriteSeq, /*IsRead=*/false);
+    LastTransitions[0].WriteSequences.emplace_back();
+    SmallVectorImpl<unsigned> &Seq = LastTransitions[0].WriteSequences.back();
+    Seq.append(WriteSeq.begin(), WriteSeq.end());
+    LLVM_DEBUG(dbgs() << "("; dumpIdxVec(Seq); dbgs() << ") ");
+  }
+  LLVM_DEBUG(dbgs() << " Reads: ");
+  for (unsigned ReadIdx : OperReads) {
+    IdxVec ReadSeq;
+    expandRWSequence(ReadIdx, ReadSeq, /*IsRead=*/true);
+    LastTransitions[0].ReadSequences.emplace_back();
+    SmallVectorImpl<unsigned> &Seq = LastTransitions[0].ReadSequences.back();
+    Seq.append(ReadSeq.begin(), ReadSeq.end());
+    LLVM_DEBUG(dbgs() << "("; dumpIdxVec(Seq); dbgs() << ") ");
+  }
+  LLVM_DEBUG(dbgs() << '\n');
+
+  LastTransitions = makePerProcessorTransitions(
+      LastTransitions[0], llvm::is_contained(ProcIndices, 0)
+                              ? ArrayRef<unsigned>(getAllProcIndices())
+                              : ProcIndices);
+  // Collect all PredTransitions for individual operands.
+  // Iterate until no variant writes remain.
+  bool SubstitutedAny;
+  do {
+    SubstitutedAny = false;
+    PredTransitions Transitions(*this);
+    for (const PredTransition &Trans : LastTransitions)
+      SubstitutedAny |= Transitions.substituteVariants(Trans);
+    LLVM_DEBUG(Transitions.dump());
+    LastTransitions = std::move(Transitions.TransVec);
+  } while (SubstitutedAny);
+
+  // WARNING: We are about to mutate the SchedClasses vector. Do not refer to
+  // OperWrites, OperReads, or ProcIndices after calling inferFromTransitions.
+  inferFromTransitions(LastTransitions, FromClassIdx, *this);
+}
+
+// Check if any processor resource group contains all resource records in
+// SubUnits.
+bool CodeGenSchedModels::hasSuperGroup(RecVec &SubUnits, CodeGenProcModel &PM) {
+  for (Record *ProcResourceDef : PM.ProcResourceDefs) {
+    if (!ProcResourceDef->isSubClassOf("ProcResGroup"))
+      continue;
+    RecVec SuperUnits = ProcResourceDef->getValueAsListOfDefs("Resources");
+    RecIter RI = SubUnits.begin(), RE = SubUnits.end();
+    for (; RI != RE; ++RI) {
+      if (!is_contained(SuperUnits, *RI)) {
+        break;
+      }
+    }
+    if (RI == RE)
+      return true;
+  }
+  return false;
+}
+
+// Verify that overlapping groups have a common supergroup.
+void CodeGenSchedModels::verifyProcResourceGroups(CodeGenProcModel &PM) {
+  for (unsigned i = 0, e = PM.ProcResourceDefs.size(); i < e; ++i) {
+    if (!PM.ProcResourceDefs[i]->isSubClassOf("ProcResGroup"))
+      continue;
+    RecVec CheckUnits =
+        PM.ProcResourceDefs[i]->getValueAsListOfDefs("Resources");
+    for (unsigned j = i + 1; j < e; ++j) {
+      if (!PM.ProcResourceDefs[j]->isSubClassOf("ProcResGroup"))
+        continue;
+      RecVec OtherUnits =
+          PM.ProcResourceDefs[j]->getValueAsListOfDefs("Resources");
+      if (std::find_first_of(CheckUnits.begin(), CheckUnits.end(),
+                             OtherUnits.begin(),
+                             OtherUnits.end()) != CheckUnits.end()) {
+        // CheckUnits and OtherUnits overlap
+        llvm::append_range(OtherUnits, CheckUnits);
+        if (!hasSuperGroup(OtherUnits, PM)) {
+          PrintFatalError((PM.ProcResourceDefs[i])->getLoc(),
+                          "proc resource group overlaps with " +
+                              PM.ProcResourceDefs[j]->getName() +
+                              " but no supergroup contains both.");
+        }
+      }
+    }
+  }
+}
+
+// Collect all the RegisterFile definitions available in this target.
+void CodeGenSchedModels::collectRegisterFiles() {
+  RecVec RegisterFileDefs = Records.getAllDerivedDefinitions("RegisterFile");
+
+  // RegisterFiles is the vector of CodeGenRegisterFile.
+  for (Record *RF : RegisterFileDefs) {
+    // For each register file definition, construct a CodeGenRegisterFile object
+    // and add it to the appropriate scheduling model.
+    CodeGenProcModel &PM = getProcModel(RF->getValueAsDef("SchedModel"));
+    PM.RegisterFiles.emplace_back(CodeGenRegisterFile(RF->getName(), RF));
+    CodeGenRegisterFile &CGRF = PM.RegisterFiles.back();
+    CGRF.MaxMovesEliminatedPerCycle =
+        RF->getValueAsInt("MaxMovesEliminatedPerCycle");
+    CGRF.AllowZeroMoveEliminationOnly =
+        RF->getValueAsBit("AllowZeroMoveEliminationOnly");
+
+    // Now set the number of physical registers as well as the cost of registers
+    // in each register class.
+    CGRF.NumPhysRegs = RF->getValueAsInt("NumPhysRegs");
+    if (!CGRF.NumPhysRegs) {
+      PrintFatalError(RF->getLoc(),
+                      "Invalid RegisterFile with zero physical registers");
+    }
+
+    RecVec RegisterClasses = RF->getValueAsListOfDefs("RegClasses");
+    std::vector<int64_t> RegisterCosts = RF->getValueAsListOfInts("RegCosts");
+    ListInit *MoveElimInfo = RF->getValueAsListInit("AllowMoveElimination");
+    for (unsigned I = 0, E = RegisterClasses.size(); I < E; ++I) {
+      int Cost = RegisterCosts.size() > I ? RegisterCosts[I] : 1;
+
+      bool AllowMoveElim = false;
+      if (MoveElimInfo->size() > I) {
+        BitInit *Val = cast<BitInit>(MoveElimInfo->getElement(I));
+        AllowMoveElim = Val->getValue();
+      }
+
+      CGRF.Costs.emplace_back(RegisterClasses[I], Cost, AllowMoveElim);
+    }
+  }
+}
+
+// Collect and sort WriteRes, ReadAdvance, and ProcResources.
+void CodeGenSchedModels::collectProcResources() {
+  ProcResourceDefs = Records.getAllDerivedDefinitions("ProcResourceUnits");
+  ProcResGroups = Records.getAllDerivedDefinitions("ProcResGroup");
+
+  // Add any subtarget-specific SchedReadWrites that are directly associated
+  // with processor resources. Refer to the parent SchedClass's ProcIndices to
+  // determine which processors they apply to.
+  for (const CodeGenSchedClass &SC :
+       make_range(schedClassBegin(), schedClassEnd())) {
+    if (SC.ItinClassDef) {
+      collectItinProcResources(SC.ItinClassDef);
+      continue;
+    }
+
+    // This class may have a default ReadWrite list which can be overriden by
+    // InstRW definitions.
+    for (Record *RW : SC.InstRWs) {
+      Record *RWModelDef = RW->getValueAsDef("SchedModel");
+      unsigned PIdx = getProcModel(RWModelDef).Index;
+      IdxVec Writes, Reads;
+      findRWs(RW->getValueAsListOfDefs("OperandReadWrites"), Writes, Reads);
+      collectRWResources(Writes, Reads, PIdx);
+    }
+
+    collectRWResources(SC.Writes, SC.Reads, SC.ProcIndices);
+  }
+  // Add resources separately defined by each subtarget.
+  RecVec WRDefs = Records.getAllDerivedDefinitions("WriteRes");
+  for (Record *WR : WRDefs) {
+    Record *ModelDef = WR->getValueAsDef("SchedModel");
+    addWriteRes(WR, getProcModel(ModelDef).Index);
+  }
+  RecVec SWRDefs = Records.getAllDerivedDefinitions("SchedWriteRes");
+  for (Record *SWR : SWRDefs) {
+    Record *ModelDef = SWR->getValueAsDef("SchedModel");
+    addWriteRes(SWR, getProcModel(ModelDef).Index);
+  }
+  RecVec RADefs = Records.getAllDerivedDefinitions("ReadAdvance");
+  for (Record *RA : RADefs) {
+    Record *ModelDef = RA->getValueAsDef("SchedModel");
+    addReadAdvance(RA, getProcModel(ModelDef).Index);
+  }
+  RecVec SRADefs = Records.getAllDerivedDefinitions("SchedReadAdvance");
+  for (Record *SRA : SRADefs) {
+    if (SRA->getValueInit("SchedModel")->isComplete()) {
+      Record *ModelDef = SRA->getValueAsDef("SchedModel");
+      addReadAdvance(SRA, getProcModel(ModelDef).Index);
+    }
+  }
+  // Add ProcResGroups that are defined within this processor model, which may
+  // not be directly referenced but may directly specify a buffer size.
+  RecVec ProcResGroups = Records.getAllDerivedDefinitions("ProcResGroup");
+  for (Record *PRG : ProcResGroups) {
+    if (!PRG->getValueInit("SchedModel")->isComplete())
+      continue;
+    CodeGenProcModel &PM = getProcModel(PRG->getValueAsDef("SchedModel"));
+    if (!is_contained(PM.ProcResourceDefs, PRG))
+      PM.ProcResourceDefs.push_back(PRG);
+  }
+  // Add ProcResourceUnits unconditionally.
+  for (Record *PRU : Records.getAllDerivedDefinitions("ProcResourceUnits")) {
+    if (!PRU->getValueInit("SchedModel")->isComplete())
+      continue;
+    CodeGenProcModel &PM = getProcModel(PRU->getValueAsDef("SchedModel"));
+    if (!is_contained(PM.ProcResourceDefs, PRU))
+      PM.ProcResourceDefs.push_back(PRU);
+  }
+  // Finalize each ProcModel by sorting the record arrays.
+  for (CodeGenProcModel &PM : ProcModels) {
+    llvm::sort(PM.WriteResDefs, LessRecord());
+    llvm::sort(PM.ReadAdvanceDefs, LessRecord());
+    llvm::sort(PM.ProcResourceDefs, LessRecord());
+    LLVM_DEBUG(
+        PM.dump(); dbgs() << "WriteResDefs: "; for (auto WriteResDef
+                                                    : PM.WriteResDefs) {
+          if (WriteResDef->isSubClassOf("WriteRes"))
+            dbgs() << WriteResDef->getValueAsDef("WriteType")->getName() << " ";
+          else
+            dbgs() << WriteResDef->getName() << " ";
+        } dbgs() << "\nReadAdvanceDefs: ";
+        for (Record *ReadAdvanceDef
+             : PM.ReadAdvanceDefs) {
+          if (ReadAdvanceDef->isSubClassOf("ReadAdvance"))
+            dbgs() << ReadAdvanceDef->getValueAsDef("ReadType")->getName()
+                   << " ";
+          else
+            dbgs() << ReadAdvanceDef->getName() << " ";
+        } dbgs()
+        << "\nProcResourceDefs: ";
+        for (Record *ProcResourceDef
+             : PM.ProcResourceDefs) {
+          dbgs() << ProcResourceDef->getName() << " ";
+        } dbgs()
+        << '\n');
+    verifyProcResourceGroups(PM);
+  }
+
+  ProcResourceDefs.clear();
+  ProcResGroups.clear();
+}
+
+void CodeGenSchedModels::checkCompleteness() {
+  bool Complete = true;
+  for (const CodeGenProcModel &ProcModel : procModels()) {
+    const bool HasItineraries = ProcModel.hasItineraries();
+    if (!ProcModel.ModelDef->getValueAsBit("CompleteModel"))
+      continue;
+    for (const CodeGenInstruction *Inst : Target.getInstructionsByEnumValue()) {
+      if (Inst->hasNoSchedulingInfo)
+        continue;
+      if (ProcModel.isUnsupported(*Inst))
+        continue;
+      unsigned SCIdx = getSchedClassIdx(*Inst);
+      if (!SCIdx) {
+        if (Inst->TheDef->isValueUnset("SchedRW")) {
+          PrintError(Inst->TheDef->getLoc(),
+                     "No schedule information for instruction '" +
+                         Inst->TheDef->getName() + "' in SchedMachineModel '" +
+                         ProcModel.ModelDef->getName() + "'");
+          Complete = false;
+        }
+        continue;
+      }
+
+      const CodeGenSchedClass &SC = getSchedClass(SCIdx);
+      if (!SC.Writes.empty())
+        continue;
+      if (HasItineraries && SC.ItinClassDef != nullptr &&
+          SC.ItinClassDef->getName() != "NoItinerary")
+        continue;
+
+      const RecVec &InstRWs = SC.InstRWs;
+      auto I = find_if(InstRWs, [&ProcModel](const Record *R) {
+        return R->getValueAsDef("SchedModel") == ProcModel.ModelDef;
+      });
+      if (I == InstRWs.end()) {
+        PrintError(Inst->TheDef->getLoc(), "'" + ProcModel.ModelName +
+                                               "' lacks information for '" +
+                                               Inst->TheDef->getName() + "'");
+        Complete = false;
+      }
+    }
+  }
+  if (!Complete) {
+    errs()
+        << "\n\nIncomplete schedule models found.\n"
+        << "- Consider setting 'CompleteModel = 0' while developing new "
+           "models.\n"
+        << "- Pseudo instructions can be marked with 'hasNoSchedulingInfo = "
+           "1'.\n"
+        << "- Instructions should usually have Sched<[...]> as a superclass, "
+           "you may temporarily use an empty list.\n"
+        << "- Instructions related to unsupported features can be excluded "
+           "with "
+           "list<Predicate> UnsupportedFeatures = [HasA,..,HasY]; in the "
+           "processor model.\n\n";
+    PrintFatalError("Incomplete schedule model");
+  }
+}
+
+// Collect itinerary class resources for each processor.
+void CodeGenSchedModels::collectItinProcResources(Record *ItinClassDef) {
+  for (unsigned PIdx = 0, PEnd = ProcModels.size(); PIdx != PEnd; ++PIdx) {
+    const CodeGenProcModel &PM = ProcModels[PIdx];
+    // For all ItinRW entries.
+    bool HasMatch = false;
+    for (RecIter II = PM.ItinRWDefs.begin(), IE = PM.ItinRWDefs.end(); II != IE;
+         ++II) {
+      RecVec Matched = (*II)->getValueAsListOfDefs("MatchedItinClasses");
+      if (!llvm::is_contained(Matched, ItinClassDef))
+        continue;
+      if (HasMatch)
+        PrintFatalError((*II)->getLoc(),
+                        "Duplicate itinerary class " + ItinClassDef->getName() +
+                            " in ItinResources for " + PM.ModelName);
+      HasMatch = true;
+      IdxVec Writes, Reads;
+      findRWs((*II)->getValueAsListOfDefs("OperandReadWrites"), Writes, Reads);
+      collectRWResources(Writes, Reads, PIdx);
+    }
+  }
+}
+
+void CodeGenSchedModels::collectRWResources(unsigned RWIdx, bool IsRead,
+                                            ArrayRef<unsigned> ProcIndices) {
+  const CodeGenSchedRW &SchedRW = getSchedRW(RWIdx, IsRead);
+  if (SchedRW.TheDef) {
+    if (!IsRead && SchedRW.TheDef->isSubClassOf("SchedWriteRes")) {
+      for (unsigned Idx : ProcIndices)
+        addWriteRes(SchedRW.TheDef, Idx);
+    } else if (IsRead && SchedRW.TheDef->isSubClassOf("SchedReadAdvance")) {
+      for (unsigned Idx : ProcIndices)
+        addReadAdvance(SchedRW.TheDef, Idx);
+    }
+  }
+  for (auto *Alias : SchedRW.Aliases) {
+    IdxVec AliasProcIndices;
+    if (Alias->getValueInit("SchedModel")->isComplete()) {
+      AliasProcIndices.push_back(
+          getProcModel(Alias->getValueAsDef("SchedModel")).Index);
+    } else
+      AliasProcIndices = ProcIndices;
+    const CodeGenSchedRW &AliasRW = getSchedRW(Alias->getValueAsDef("AliasRW"));
+    assert(AliasRW.IsRead == IsRead && "cannot alias reads to writes");
+
+    IdxVec ExpandedRWs;
+    expandRWSequence(AliasRW.Index, ExpandedRWs, IsRead);
+    for (unsigned int ExpandedRW : ExpandedRWs) {
+      collectRWResources(ExpandedRW, IsRead, AliasProcIndices);
+    }
+  }
+}
+
+// Collect resources for a set of read/write types and processor indices.
+void CodeGenSchedModels::collectRWResources(ArrayRef<unsigned> Writes,
+                                            ArrayRef<unsigned> Reads,
+                                            ArrayRef<unsigned> ProcIndices) {
+  for (unsigned Idx : Writes)
+    collectRWResources(Idx, /*IsRead=*/false, ProcIndices);
+
+  for (unsigned Idx : Reads)
+    collectRWResources(Idx, /*IsRead=*/true, ProcIndices);
+}
+
+// Find the processor's resource units for this kind of resource.
+Record *CodeGenSchedModels::findProcResUnits(Record *ProcResKind,
+                                             const CodeGenProcModel &PM,
+                                             ArrayRef<SMLoc> Loc) const {
+  if (ProcResKind->isSubClassOf("ProcResourceUnits"))
+    return ProcResKind;
+
+  Record *ProcUnitDef = nullptr;
+  assert(!ProcResourceDefs.empty());
+  assert(!ProcResGroups.empty());
+
+  for (Record *ProcResDef : ProcResourceDefs) {
+    if (ProcResDef->getValueAsDef("Kind") == ProcResKind &&
+        ProcResDef->getValueAsDef("SchedModel") == PM.ModelDef) {
+      if (ProcUnitDef) {
+        PrintFatalError(Loc,
+                        "Multiple ProcessorResourceUnits associated with " +
+                            ProcResKind->getName());
+      }
+      ProcUnitDef = ProcResDef;
+    }
+  }
+  for (Record *ProcResGroup : ProcResGroups) {
+    if (ProcResGroup == ProcResKind &&
+        ProcResGroup->getValueAsDef("SchedModel") == PM.ModelDef) {
+      if (ProcUnitDef) {
+        PrintFatalError(Loc,
+                        "Multiple ProcessorResourceUnits associated with " +
+                            ProcResKind->getName());
+      }
+      ProcUnitDef = ProcResGroup;
+    }
+  }
+  if (!ProcUnitDef) {
+    PrintFatalError(Loc, "No ProcessorResources associated with " +
+                             ProcResKind->getName());
+  }
+  return ProcUnitDef;
+}
+
+// Iteratively add a resource and its super resources.
+void CodeGenSchedModels::addProcResource(Record *ProcResKind,
+                                         CodeGenProcModel &PM,
+                                         ArrayRef<SMLoc> Loc) {
+  while (true) {
+    Record *ProcResUnits = findProcResUnits(ProcResKind, PM, Loc);
+
+    // See if this ProcResource is already associated with this processor.
+    if (is_contained(PM.ProcResourceDefs, ProcResUnits))
+      return;
+
+    PM.ProcResourceDefs.push_back(ProcResUnits);
+    if (ProcResUnits->isSubClassOf("ProcResGroup"))
+      return;
+
+    if (!ProcResUnits->getValueInit("Super")->isComplete())
+      return;
+
+    ProcResKind = ProcResUnits->getValueAsDef("Super");
+  }
+}
+
+// Add resources for a SchedWrite to this processor if they don't exist.
+void CodeGenSchedModels::addWriteRes(Record *ProcWriteResDef, unsigned PIdx) {
+  assert(PIdx && "don't add resources to an invalid Processor model");
+
+  RecVec &WRDefs = ProcModels[PIdx].WriteResDefs;
+  if (is_contained(WRDefs, ProcWriteResDef))
+    return;
+  WRDefs.push_back(ProcWriteResDef);
+
+  // Visit ProcResourceKinds referenced by the newly discovered WriteRes.
+  RecVec ProcResDefs = ProcWriteResDef->getValueAsListOfDefs("ProcResources");
+  for (auto *ProcResDef : ProcResDefs) {
+    addProcResource(ProcResDef, ProcModels[PIdx], ProcWriteResDef->getLoc());
+  }
+}
+
+// Add resources for a ReadAdvance to this processor if they don't exist.
+void CodeGenSchedModels::addReadAdvance(Record *ProcReadAdvanceDef,
+                                        unsigned PIdx) {
+  for (const Record *ValidWrite :
+       ProcReadAdvanceDef->getValueAsListOfDefs("ValidWrites"))
+    if (getSchedRWIdx(ValidWrite, /*IsRead=*/false) == 0)
+      PrintFatalError(
+          ProcReadAdvanceDef->getLoc(),
+          "ReadAdvance referencing a ValidWrite that is not used by "
+          "any instruction (" +
+              ValidWrite->getName() + ")");
+
+  RecVec &RADefs = ProcModels[PIdx].ReadAdvanceDefs;
+  if (is_contained(RADefs, ProcReadAdvanceDef))
+    return;
+  RADefs.push_back(ProcReadAdvanceDef);
+}
+
+unsigned CodeGenProcModel::getProcResourceIdx(Record *PRDef) const {
+  RecIter PRPos = find(ProcResourceDefs, PRDef);
+  if (PRPos == ProcResourceDefs.end())
+    PrintFatalError(PRDef->getLoc(), "ProcResource def is not included in "
+                                     "the ProcResources list for " +
+                                         ModelName);
+  // Idx=0 is reserved for invalid.
+  return 1 + (PRPos - ProcResourceDefs.begin());
+}
+
+bool CodeGenProcModel::isUnsupported(const CodeGenInstruction &Inst) const {
+  for (const Record *TheDef : UnsupportedFeaturesDefs) {
+    for (const Record *PredDef :
+         Inst.TheDef->getValueAsListOfDefs("Predicates")) {
+      if (TheDef->getName() == PredDef->getName())
+        return true;
+    }
+  }
+  return false;
+}
+
+#ifndef NDEBUG
+void CodeGenProcModel::dump() const {
+  dbgs() << Index << ": " << ModelName << " "
+         << (ModelDef ? ModelDef->getName() : "inferred") << " "
+         << (ItinsDef ? ItinsDef->getName() : "no itinerary") << '\n';
+}
+
+void CodeGenSchedRW::dump() const {
+  dbgs() << Name << (IsVariadic ? " (V) " : " ");
+  if (IsSequence) {
+    dbgs() << "(";
+    dumpIdxVec(Sequence);
+    dbgs() << ")";
+  }
+}
+
+void CodeGenSchedClass::dump(const CodeGenSchedModels *SchedModels) const {
+  dbgs() << "SCHEDCLASS " << Index << ":" << Name << '\n' << "  Writes: ";
+  for (unsigned i = 0, N = Writes.size(); i < N; ++i) {
+    SchedModels->getSchedWrite(Writes[i]).dump();
+    if (i < N - 1) {
+      dbgs() << '\n';
+      dbgs().indent(10);
+    }
+  }
+  dbgs() << "\n  Reads: ";
+  for (unsigned i = 0, N = Reads.size(); i < N; ++i) {
+    SchedModels->getSchedRead(Reads[i]).dump();
+    if (i < N - 1) {
+      dbgs() << '\n';
+      dbgs().indent(10);
+    }
+  }
+  dbgs() << "\n  ProcIdx: ";
+  dumpIdxVec(ProcIndices);
+  if (!Transitions.empty()) {
+    dbgs() << "\n Transitions for Proc ";
+    for (const CodeGenSchedTransition &Transition : Transitions) {
+      dbgs() << Transition.ProcIndex << ", ";
+    }
+  }
+  dbgs() << '\n';
+}
+
+void PredTransitions::dump() const {
+  dbgs() << "Expanded Variants:\n";
+  for (const auto &TI : TransVec) {
+    dbgs() << "{";
+    ListSeparator LS;
+    for (const PredCheck &PC : TI.PredTerm)
+      dbgs() << LS << SchedModels.getSchedRW(PC.RWIdx, PC.IsRead).Name << ":"
+             << PC.Predicate->getName();
+    dbgs() << "},\n  => {";
+    for (SmallVectorImpl<SmallVector<unsigned, 4>>::const_iterator
+             WSI = TI.WriteSequences.begin(),
+             WSE = TI.WriteSequences.end();
+         WSI != WSE; ++WSI) {
+      dbgs() << "(";
+      ListSeparator LS;
+      for (unsigned N : *WSI)
+        dbgs() << LS << SchedModels.getSchedWrite(N).Name;
+      dbgs() << "),";
+    }
+    dbgs() << "}\n";
+  }
+}
+#endif // NDEBUG