| File: | llvm/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp |
| Warning: | line 245, column 7 1st function call argument is an uninitialized value |
Press '?' to see keyboard shortcuts
Keyboard shortcuts:
| 1 | //===- SelectionDAGISel.cpp - Implement the SelectionDAGISel class --------===// | |||
| 2 | // | |||
| 3 | // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. | |||
| 4 | // See https://llvm.org/LICENSE.txt for license information. | |||
| 5 | // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception | |||
| 6 | // | |||
| 7 | //===----------------------------------------------------------------------===// | |||
| 8 | // | |||
| 9 | // This implements the SelectionDAGISel class. | |||
| 10 | // | |||
| 11 | //===----------------------------------------------------------------------===// | |||
| 12 | ||||
| 13 | #include "llvm/CodeGen/SelectionDAGISel.h" | |||
| 14 | #include "ScheduleDAGSDNodes.h" | |||
| 15 | #include "SelectionDAGBuilder.h" | |||
| 16 | #include "llvm/ADT/APInt.h" | |||
| 17 | #include "llvm/ADT/DenseMap.h" | |||
| 18 | #include "llvm/ADT/None.h" | |||
| 19 | #include "llvm/ADT/PostOrderIterator.h" | |||
| 20 | #include "llvm/ADT/STLExtras.h" | |||
| 21 | #include "llvm/ADT/SmallPtrSet.h" | |||
| 22 | #include "llvm/ADT/SmallSet.h" | |||
| 23 | #include "llvm/ADT/SmallVector.h" | |||
| 24 | #include "llvm/ADT/Statistic.h" | |||
| 25 | #include "llvm/ADT/StringRef.h" | |||
| 26 | #include "llvm/Analysis/AliasAnalysis.h" | |||
| 27 | #include "llvm/Analysis/BranchProbabilityInfo.h" | |||
| 28 | #include "llvm/Analysis/CFG.h" | |||
| 29 | #include "llvm/Analysis/EHPersonalities.h" | |||
| 30 | #include "llvm/Analysis/LazyBlockFrequencyInfo.h" | |||
| 31 | #include "llvm/Analysis/LegacyDivergenceAnalysis.h" | |||
| 32 | #include "llvm/Analysis/OptimizationRemarkEmitter.h" | |||
| 33 | #include "llvm/Analysis/ProfileSummaryInfo.h" | |||
| 34 | #include "llvm/Analysis/TargetLibraryInfo.h" | |||
| 35 | #include "llvm/Analysis/TargetTransformInfo.h" | |||
| 36 | #include "llvm/CodeGen/FastISel.h" | |||
| 37 | #include "llvm/CodeGen/FunctionLoweringInfo.h" | |||
| 38 | #include "llvm/CodeGen/GCMetadata.h" | |||
| 39 | #include "llvm/CodeGen/ISDOpcodes.h" | |||
| 40 | #include "llvm/CodeGen/MachineBasicBlock.h" | |||
| 41 | #include "llvm/CodeGen/MachineFrameInfo.h" | |||
| 42 | #include "llvm/CodeGen/MachineFunction.h" | |||
| 43 | #include "llvm/CodeGen/MachineFunctionPass.h" | |||
| 44 | #include "llvm/CodeGen/MachineInstr.h" | |||
| 45 | #include "llvm/CodeGen/MachineInstrBuilder.h" | |||
| 46 | #include "llvm/CodeGen/MachineMemOperand.h" | |||
| 47 | #include "llvm/CodeGen/MachineModuleInfo.h" | |||
| 48 | #include "llvm/CodeGen/MachineOperand.h" | |||
| 49 | #include "llvm/CodeGen/MachinePassRegistry.h" | |||
| 50 | #include "llvm/CodeGen/MachineRegisterInfo.h" | |||
| 51 | #include "llvm/CodeGen/SchedulerRegistry.h" | |||
| 52 | #include "llvm/CodeGen/SelectionDAG.h" | |||
| 53 | #include "llvm/CodeGen/SelectionDAGNodes.h" | |||
| 54 | #include "llvm/CodeGen/StackProtector.h" | |||
| 55 | #include "llvm/CodeGen/SwiftErrorValueTracking.h" | |||
| 56 | #include "llvm/CodeGen/TargetInstrInfo.h" | |||
| 57 | #include "llvm/CodeGen/TargetLowering.h" | |||
| 58 | #include "llvm/CodeGen/TargetRegisterInfo.h" | |||
| 59 | #include "llvm/CodeGen/TargetSubtargetInfo.h" | |||
| 60 | #include "llvm/CodeGen/ValueTypes.h" | |||
| 61 | #include "llvm/IR/BasicBlock.h" | |||
| 62 | #include "llvm/IR/Constants.h" | |||
| 63 | #include "llvm/IR/DataLayout.h" | |||
| 64 | #include "llvm/IR/DebugInfoMetadata.h" | |||
| 65 | #include "llvm/IR/DebugLoc.h" | |||
| 66 | #include "llvm/IR/DiagnosticInfo.h" | |||
| 67 | #include "llvm/IR/Dominators.h" | |||
| 68 | #include "llvm/IR/Function.h" | |||
| 69 | #include "llvm/IR/InlineAsm.h" | |||
| 70 | #include "llvm/IR/InstIterator.h" | |||
| 71 | #include "llvm/IR/InstrTypes.h" | |||
| 72 | #include "llvm/IR/Instruction.h" | |||
| 73 | #include "llvm/IR/Instructions.h" | |||
| 74 | #include "llvm/IR/IntrinsicInst.h" | |||
| 75 | #include "llvm/IR/Intrinsics.h" | |||
| 76 | #include "llvm/IR/IntrinsicsWebAssembly.h" | |||
| 77 | #include "llvm/IR/Metadata.h" | |||
| 78 | #include "llvm/IR/Type.h" | |||
| 79 | #include "llvm/IR/User.h" | |||
| 80 | #include "llvm/IR/Value.h" | |||
| 81 | #include "llvm/InitializePasses.h" | |||
| 82 | #include "llvm/MC/MCInstrDesc.h" | |||
| 83 | #include "llvm/MC/MCRegisterInfo.h" | |||
| 84 | #include "llvm/Pass.h" | |||
| 85 | #include "llvm/Support/BranchProbability.h" | |||
| 86 | #include "llvm/Support/Casting.h" | |||
| 87 | #include "llvm/Support/CodeGen.h" | |||
| 88 | #include "llvm/Support/CommandLine.h" | |||
| 89 | #include "llvm/Support/Compiler.h" | |||
| 90 | #include "llvm/Support/Debug.h" | |||
| 91 | #include "llvm/Support/ErrorHandling.h" | |||
| 92 | #include "llvm/Support/KnownBits.h" | |||
| 93 | #include "llvm/Support/MachineValueType.h" | |||
| 94 | #include "llvm/Support/Timer.h" | |||
| 95 | #include "llvm/Support/raw_ostream.h" | |||
| 96 | #include "llvm/Target/TargetIntrinsicInfo.h" | |||
| 97 | #include "llvm/Target/TargetMachine.h" | |||
| 98 | #include "llvm/Target/TargetOptions.h" | |||
| 99 | #include "llvm/Transforms/Utils/BasicBlockUtils.h" | |||
| 100 | #include <algorithm> | |||
| 101 | #include <cassert> | |||
| 102 | #include <cstdint> | |||
| 103 | #include <iterator> | |||
| 104 | #include <limits> | |||
| 105 | #include <memory> | |||
| 106 | #include <string> | |||
| 107 | #include <utility> | |||
| 108 | #include <vector> | |||
| 109 | ||||
| 110 | using namespace llvm; | |||
| 111 | ||||
| 112 | #define DEBUG_TYPE"isel" "isel" | |||
| 113 | ||||
| 114 | STATISTIC(NumFastIselFailures, "Number of instructions fast isel failed on")static llvm::Statistic NumFastIselFailures = {"isel", "NumFastIselFailures" , "Number of instructions fast isel failed on"}; | |||
| 115 | STATISTIC(NumFastIselSuccess, "Number of instructions fast isel selected")static llvm::Statistic NumFastIselSuccess = {"isel", "NumFastIselSuccess" , "Number of instructions fast isel selected"}; | |||
| 116 | STATISTIC(NumFastIselBlocks, "Number of blocks selected entirely by fast isel")static llvm::Statistic NumFastIselBlocks = {"isel", "NumFastIselBlocks" , "Number of blocks selected entirely by fast isel"}; | |||
| 117 | STATISTIC(NumDAGBlocks, "Number of blocks selected using DAG")static llvm::Statistic NumDAGBlocks = {"isel", "NumDAGBlocks" , "Number of blocks selected using DAG"}; | |||
| 118 | STATISTIC(NumDAGIselRetries,"Number of times dag isel has to try another path")static llvm::Statistic NumDAGIselRetries = {"isel", "NumDAGIselRetries" , "Number of times dag isel has to try another path"}; | |||
| 119 | STATISTIC(NumEntryBlocks, "Number of entry blocks encountered")static llvm::Statistic NumEntryBlocks = {"isel", "NumEntryBlocks" , "Number of entry blocks encountered"}; | |||
| 120 | STATISTIC(NumFastIselFailLowerArguments,static llvm::Statistic NumFastIselFailLowerArguments = {"isel" , "NumFastIselFailLowerArguments", "Number of entry blocks where fast isel failed to lower arguments" } | |||
| 121 | "Number of entry blocks where fast isel failed to lower arguments")static llvm::Statistic NumFastIselFailLowerArguments = {"isel" , "NumFastIselFailLowerArguments", "Number of entry blocks where fast isel failed to lower arguments" }; | |||
| 122 | ||||
| 123 | static cl::opt<int> EnableFastISelAbort( | |||
| 124 | "fast-isel-abort", cl::Hidden, | |||
| 125 | cl::desc("Enable abort calls when \"fast\" instruction selection " | |||
| 126 | "fails to lower an instruction: 0 disable the abort, 1 will " | |||
| 127 | "abort but for args, calls and terminators, 2 will also " | |||
| 128 | "abort for argument lowering, and 3 will never fallback " | |||
| 129 | "to SelectionDAG.")); | |||
| 130 | ||||
| 131 | static cl::opt<bool> EnableFastISelFallbackReport( | |||
| 132 | "fast-isel-report-on-fallback", cl::Hidden, | |||
| 133 | cl::desc("Emit a diagnostic when \"fast\" instruction selection " | |||
| 134 | "falls back to SelectionDAG.")); | |||
| 135 | ||||
| 136 | static cl::opt<bool> | |||
| 137 | UseMBPI("use-mbpi", | |||
| 138 | cl::desc("use Machine Branch Probability Info"), | |||
| 139 | cl::init(true), cl::Hidden); | |||
| 140 | ||||
| 141 | #ifndef NDEBUG | |||
| 142 | static cl::opt<std::string> | |||
| 143 | FilterDAGBasicBlockName("filter-view-dags", cl::Hidden, | |||
| 144 | cl::desc("Only display the basic block whose name " | |||
| 145 | "matches this for all view-*-dags options")); | |||
| 146 | static cl::opt<bool> | |||
| 147 | ViewDAGCombine1("view-dag-combine1-dags", cl::Hidden, | |||
| 148 | cl::desc("Pop up a window to show dags before the first " | |||
| 149 | "dag combine pass")); | |||
| 150 | static cl::opt<bool> | |||
| 151 | ViewLegalizeTypesDAGs("view-legalize-types-dags", cl::Hidden, | |||
| 152 | cl::desc("Pop up a window to show dags before legalize types")); | |||
| 153 | static cl::opt<bool> | |||
| 154 | ViewDAGCombineLT("view-dag-combine-lt-dags", cl::Hidden, | |||
| 155 | cl::desc("Pop up a window to show dags before the post " | |||
| 156 | "legalize types dag combine pass")); | |||
| 157 | static cl::opt<bool> | |||
| 158 | ViewLegalizeDAGs("view-legalize-dags", cl::Hidden, | |||
| 159 | cl::desc("Pop up a window to show dags before legalize")); | |||
| 160 | static cl::opt<bool> | |||
| 161 | ViewDAGCombine2("view-dag-combine2-dags", cl::Hidden, | |||
| 162 | cl::desc("Pop up a window to show dags before the second " | |||
| 163 | "dag combine pass")); | |||
| 164 | static cl::opt<bool> | |||
| 165 | ViewISelDAGs("view-isel-dags", cl::Hidden, | |||
| 166 | cl::desc("Pop up a window to show isel dags as they are selected")); | |||
| 167 | static cl::opt<bool> | |||
| 168 | ViewSchedDAGs("view-sched-dags", cl::Hidden, | |||
| 169 | cl::desc("Pop up a window to show sched dags as they are processed")); | |||
| 170 | static cl::opt<bool> | |||
| 171 | ViewSUnitDAGs("view-sunit-dags", cl::Hidden, | |||
| 172 | cl::desc("Pop up a window to show SUnit dags after they are processed")); | |||
| 173 | #else | |||
| 174 | static const bool ViewDAGCombine1 = false, ViewLegalizeTypesDAGs = false, | |||
| 175 | ViewDAGCombineLT = false, ViewLegalizeDAGs = false, | |||
| 176 | ViewDAGCombine2 = false, ViewISelDAGs = false, | |||
| 177 | ViewSchedDAGs = false, ViewSUnitDAGs = false; | |||
| 178 | #endif | |||
| 179 | ||||
| 180 | //===---------------------------------------------------------------------===// | |||
| 181 | /// | |||
| 182 | /// RegisterScheduler class - Track the registration of instruction schedulers. | |||
| 183 | /// | |||
| 184 | //===---------------------------------------------------------------------===// | |||
| 185 | MachinePassRegistry<RegisterScheduler::FunctionPassCtor> | |||
| 186 | RegisterScheduler::Registry; | |||
| 187 | ||||
| 188 | //===---------------------------------------------------------------------===// | |||
| 189 | /// | |||
| 190 | /// ISHeuristic command line option for instruction schedulers. | |||
| 191 | /// | |||
| 192 | //===---------------------------------------------------------------------===// | |||
| 193 | static cl::opt<RegisterScheduler::FunctionPassCtor, false, | |||
| 194 | RegisterPassParser<RegisterScheduler>> | |||
| 195 | ISHeuristic("pre-RA-sched", | |||
| 196 | cl::init(&createDefaultScheduler), cl::Hidden, | |||
| 197 | cl::desc("Instruction schedulers available (before register" | |||
| 198 | " allocation):")); | |||
| 199 | ||||
| 200 | static RegisterScheduler | |||
| 201 | defaultListDAGScheduler("default", "Best scheduler for the target", | |||
| 202 | createDefaultScheduler); | |||
| 203 | ||||
| 204 | namespace llvm { | |||
| 205 | ||||
| 206 | //===--------------------------------------------------------------------===// | |||
| 207 | /// This class is used by SelectionDAGISel to temporarily override | |||
| 208 | /// the optimization level on a per-function basis. | |||
| 209 | class OptLevelChanger { | |||
| 210 | SelectionDAGISel &IS; | |||
| 211 | CodeGenOpt::Level SavedOptLevel; | |||
| 212 | bool SavedFastISel; | |||
| 213 | ||||
| 214 | public: | |||
| 215 | OptLevelChanger(SelectionDAGISel &ISel, | |||
| 216 | CodeGenOpt::Level NewOptLevel) : IS(ISel) { | |||
| 217 | SavedOptLevel = IS.OptLevel; | |||
| 218 | if (NewOptLevel
| |||
| 219 | return; | |||
| 220 | IS.OptLevel = NewOptLevel; | |||
| 221 | IS.TM.setOptLevel(NewOptLevel); | |||
| 222 | LLVM_DEBUG(dbgs() << "\nChanging optimization level for Function "do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("isel")) { dbgs() << "\nChanging optimization level for Function " << IS.MF->getFunction().getName() << "\n"; } } while (false) | |||
| 223 | << IS.MF->getFunction().getName() << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("isel")) { dbgs() << "\nChanging optimization level for Function " << IS.MF->getFunction().getName() << "\n"; } } while (false); | |||
| 224 | LLVM_DEBUG(dbgs() << "\tBefore: -O" << SavedOptLevel << " ; After: -O"do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("isel")) { dbgs() << "\tBefore: -O" << SavedOptLevel << " ; After: -O" << NewOptLevel << "\n"; } } while (false) | |||
| 225 | << NewOptLevel << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("isel")) { dbgs() << "\tBefore: -O" << SavedOptLevel << " ; After: -O" << NewOptLevel << "\n"; } } while (false); | |||
| 226 | SavedFastISel = IS.TM.Options.EnableFastISel; | |||
| 227 | if (NewOptLevel == CodeGenOpt::None) { | |||
| 228 | IS.TM.setFastISel(IS.TM.getO0WantsFastISel()); | |||
| 229 | LLVM_DEBUG(do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("isel")) { dbgs() << "\tFastISel is " << (IS.TM. Options.EnableFastISel ? "enabled" : "disabled") << "\n" ; } } while (false) | |||
| 230 | dbgs() << "\tFastISel is "do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("isel")) { dbgs() << "\tFastISel is " << (IS.TM. Options.EnableFastISel ? "enabled" : "disabled") << "\n" ; } } while (false) | |||
| 231 | << (IS.TM.Options.EnableFastISel ? "enabled" : "disabled")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("isel")) { dbgs() << "\tFastISel is " << (IS.TM. Options.EnableFastISel ? "enabled" : "disabled") << "\n" ; } } while (false) | |||
| 232 | << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("isel")) { dbgs() << "\tFastISel is " << (IS.TM. Options.EnableFastISel ? "enabled" : "disabled") << "\n" ; } } while (false); | |||
| 233 | } | |||
| 234 | } | |||
| 235 | ||||
| 236 | ~OptLevelChanger() { | |||
| 237 | if (IS.OptLevel == SavedOptLevel) | |||
| 238 | return; | |||
| 239 | LLVM_DEBUG(dbgs() << "\nRestoring optimization level for Function "do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("isel")) { dbgs() << "\nRestoring optimization level for Function " << IS.MF->getFunction().getName() << "\n"; } } while (false) | |||
| 240 | << IS.MF->getFunction().getName() << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("isel")) { dbgs() << "\nRestoring optimization level for Function " << IS.MF->getFunction().getName() << "\n"; } } while (false); | |||
| 241 | LLVM_DEBUG
("isel")) { dbgs() << "\tBefore: -O" << IS.OptLevel << " ; After: -O" << SavedOptLevel << "\n" ; } } while (false) | |||
| 242 | << SavedOptLevel << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("isel")) { dbgs() << "\tBefore: -O" << IS.OptLevel << " ; After: -O" << SavedOptLevel << "\n" ; } } while (false); | |||
| 243 | IS.OptLevel = SavedOptLevel; | |||
| 244 | IS.TM.setOptLevel(SavedOptLevel); | |||
| 245 | IS.TM.setFastISel(SavedFastISel); | |||
| ||||
| 246 | } | |||
| 247 | }; | |||
| 248 | ||||
| 249 | //===--------------------------------------------------------------------===// | |||
| 250 | /// createDefaultScheduler - This creates an instruction scheduler appropriate | |||
| 251 | /// for the target. | |||
| 252 | ScheduleDAGSDNodes* createDefaultScheduler(SelectionDAGISel *IS, | |||
| 253 | CodeGenOpt::Level OptLevel) { | |||
| 254 | const TargetLowering *TLI = IS->TLI; | |||
| 255 | const TargetSubtargetInfo &ST = IS->MF->getSubtarget(); | |||
| 256 | ||||
| 257 | // Try first to see if the Target has its own way of selecting a scheduler | |||
| 258 | if (auto *SchedulerCtor = ST.getDAGScheduler(OptLevel)) { | |||
| 259 | return SchedulerCtor(IS, OptLevel); | |||
| 260 | } | |||
| 261 | ||||
| 262 | if (OptLevel == CodeGenOpt::None || | |||
| 263 | (ST.enableMachineScheduler() && ST.enableMachineSchedDefaultSched()) || | |||
| 264 | TLI->getSchedulingPreference() == Sched::Source) | |||
| 265 | return createSourceListDAGScheduler(IS, OptLevel); | |||
| 266 | if (TLI->getSchedulingPreference() == Sched::RegPressure) | |||
| 267 | return createBURRListDAGScheduler(IS, OptLevel); | |||
| 268 | if (TLI->getSchedulingPreference() == Sched::Hybrid) | |||
| 269 | return createHybridListDAGScheduler(IS, OptLevel); | |||
| 270 | if (TLI->getSchedulingPreference() == Sched::VLIW) | |||
| 271 | return createVLIWDAGScheduler(IS, OptLevel); | |||
| 272 | assert(TLI->getSchedulingPreference() == Sched::ILP &&((TLI->getSchedulingPreference() == Sched::ILP && "Unknown sched type!" ) ? static_cast<void> (0) : __assert_fail ("TLI->getSchedulingPreference() == Sched::ILP && \"Unknown sched type!\"" , "/build/llvm-toolchain-snapshot-10~++20200112100611+7fa5290d5bd/llvm/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp" , 273, __PRETTY_FUNCTION__)) | |||
| 273 | "Unknown sched type!")((TLI->getSchedulingPreference() == Sched::ILP && "Unknown sched type!" ) ? static_cast<void> (0) : __assert_fail ("TLI->getSchedulingPreference() == Sched::ILP && \"Unknown sched type!\"" , "/build/llvm-toolchain-snapshot-10~++20200112100611+7fa5290d5bd/llvm/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp" , 273, __PRETTY_FUNCTION__)); | |||
| 274 | return createILPListDAGScheduler(IS, OptLevel); | |||
| 275 | } | |||
| 276 | ||||
| 277 | } // end namespace llvm | |||
| 278 | ||||
| 279 | // EmitInstrWithCustomInserter - This method should be implemented by targets | |||
| 280 | // that mark instructions with the 'usesCustomInserter' flag. These | |||
| 281 | // instructions are special in various ways, which require special support to | |||
| 282 | // insert. The specified MachineInstr is created but not inserted into any | |||
| 283 | // basic blocks, and this method is called to expand it into a sequence of | |||
| 284 | // instructions, potentially also creating new basic blocks and control flow. | |||
| 285 | // When new basic blocks are inserted and the edges from MBB to its successors | |||
| 286 | // are modified, the method should insert pairs of <OldSucc, NewSucc> into the | |||
| 287 | // DenseMap. | |||
| 288 | MachineBasicBlock * | |||
| 289 | TargetLowering::EmitInstrWithCustomInserter(MachineInstr &MI, | |||
| 290 | MachineBasicBlock *MBB) const { | |||
| 291 | #ifndef NDEBUG | |||
| 292 | dbgs() << "If a target marks an instruction with " | |||
| 293 | "'usesCustomInserter', it must implement " | |||
| 294 | "TargetLowering::EmitInstrWithCustomInserter!"; | |||
| 295 | #endif | |||
| 296 | llvm_unreachable(nullptr)::llvm::llvm_unreachable_internal(nullptr, "/build/llvm-toolchain-snapshot-10~++20200112100611+7fa5290d5bd/llvm/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp" , 296); | |||
| 297 | } | |||
| 298 | ||||
| 299 | void TargetLowering::AdjustInstrPostInstrSelection(MachineInstr &MI, | |||
| 300 | SDNode *Node) const { | |||
| 301 | assert(!MI.hasPostISelHook() &&((!MI.hasPostISelHook() && "If a target marks an instruction with 'hasPostISelHook', " "it must implement TargetLowering::AdjustInstrPostInstrSelection!" ) ? static_cast<void> (0) : __assert_fail ("!MI.hasPostISelHook() && \"If a target marks an instruction with 'hasPostISelHook', \" \"it must implement TargetLowering::AdjustInstrPostInstrSelection!\"" , "/build/llvm-toolchain-snapshot-10~++20200112100611+7fa5290d5bd/llvm/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp" , 303, __PRETTY_FUNCTION__)) | |||
| 302 | "If a target marks an instruction with 'hasPostISelHook', "((!MI.hasPostISelHook() && "If a target marks an instruction with 'hasPostISelHook', " "it must implement TargetLowering::AdjustInstrPostInstrSelection!" ) ? static_cast<void> (0) : __assert_fail ("!MI.hasPostISelHook() && \"If a target marks an instruction with 'hasPostISelHook', \" \"it must implement TargetLowering::AdjustInstrPostInstrSelection!\"" , "/build/llvm-toolchain-snapshot-10~++20200112100611+7fa5290d5bd/llvm/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp" , 303, __PRETTY_FUNCTION__)) | |||
| 303 | "it must implement TargetLowering::AdjustInstrPostInstrSelection!")((!MI.hasPostISelHook() && "If a target marks an instruction with 'hasPostISelHook', " "it must implement TargetLowering::AdjustInstrPostInstrSelection!" ) ? static_cast<void> (0) : __assert_fail ("!MI.hasPostISelHook() && \"If a target marks an instruction with 'hasPostISelHook', \" \"it must implement TargetLowering::AdjustInstrPostInstrSelection!\"" , "/build/llvm-toolchain-snapshot-10~++20200112100611+7fa5290d5bd/llvm/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp" , 303, __PRETTY_FUNCTION__)); | |||
| 304 | } | |||
| 305 | ||||
| 306 | //===----------------------------------------------------------------------===// | |||
| 307 | // SelectionDAGISel code | |||
| 308 | //===----------------------------------------------------------------------===// | |||
| 309 | ||||
| 310 | SelectionDAGISel::SelectionDAGISel(TargetMachine &tm, CodeGenOpt::Level OL) | |||
| 311 | : MachineFunctionPass(ID), TM(tm), FuncInfo(new FunctionLoweringInfo()), | |||
| 312 | SwiftError(new SwiftErrorValueTracking()), | |||
| 313 | CurDAG(new SelectionDAG(tm, OL)), | |||
| 314 | SDB(std::make_unique<SelectionDAGBuilder>(*CurDAG, *FuncInfo, *SwiftError, | |||
| 315 | OL)), | |||
| 316 | AA(), GFI(), OptLevel(OL), DAGSize(0) { | |||
| 317 | initializeGCModuleInfoPass(*PassRegistry::getPassRegistry()); | |||
| 318 | initializeBranchProbabilityInfoWrapperPassPass( | |||
| 319 | *PassRegistry::getPassRegistry()); | |||
| 320 | initializeAAResultsWrapperPassPass(*PassRegistry::getPassRegistry()); | |||
| 321 | initializeTargetLibraryInfoWrapperPassPass(*PassRegistry::getPassRegistry()); | |||
| 322 | } | |||
| 323 | ||||
| 324 | SelectionDAGISel::~SelectionDAGISel() { | |||
| 325 | delete CurDAG; | |||
| 326 | delete SwiftError; | |||
| 327 | } | |||
| 328 | ||||
| 329 | void SelectionDAGISel::getAnalysisUsage(AnalysisUsage &AU) const { | |||
| 330 | if (OptLevel != CodeGenOpt::None) | |||
| 331 | AU.addRequired<AAResultsWrapperPass>(); | |||
| 332 | AU.addRequired<GCModuleInfo>(); | |||
| 333 | AU.addRequired<StackProtector>(); | |||
| 334 | AU.addPreserved<GCModuleInfo>(); | |||
| 335 | AU.addRequired<TargetLibraryInfoWrapperPass>(); | |||
| 336 | AU.addRequired<TargetTransformInfoWrapperPass>(); | |||
| 337 | if (UseMBPI && OptLevel != CodeGenOpt::None) | |||
| 338 | AU.addRequired<BranchProbabilityInfoWrapperPass>(); | |||
| 339 | AU.addRequired<ProfileSummaryInfoWrapperPass>(); | |||
| 340 | LazyBlockFrequencyInfoPass::getLazyBFIAnalysisUsage(AU); | |||
| 341 | MachineFunctionPass::getAnalysisUsage(AU); | |||
| 342 | } | |||
| 343 | ||||
| 344 | /// SplitCriticalSideEffectEdges - Look for critical edges with a PHI value that | |||
| 345 | /// may trap on it. In this case we have to split the edge so that the path | |||
| 346 | /// through the predecessor block that doesn't go to the phi block doesn't | |||
| 347 | /// execute the possibly trapping instruction. If available, we pass domtree | |||
| 348 | /// and loop info to be updated when we split critical edges. This is because | |||
| 349 | /// SelectionDAGISel preserves these analyses. | |||
| 350 | /// This is required for correctness, so it must be done at -O0. | |||
| 351 | /// | |||
| 352 | static void SplitCriticalSideEffectEdges(Function &Fn, DominatorTree *DT, | |||
| 353 | LoopInfo *LI) { | |||
| 354 | // Loop for blocks with phi nodes. | |||
| 355 | for (BasicBlock &BB : Fn) { | |||
| 356 | PHINode *PN = dyn_cast<PHINode>(BB.begin()); | |||
| 357 | if (!PN) continue; | |||
| 358 | ||||
| 359 | ReprocessBlock: | |||
| 360 | // For each block with a PHI node, check to see if any of the input values | |||
| 361 | // are potentially trapping constant expressions. Constant expressions are | |||
| 362 | // the only potentially trapping value that can occur as the argument to a | |||
| 363 | // PHI. | |||
| 364 | for (BasicBlock::iterator I = BB.begin(); (PN = dyn_cast<PHINode>(I)); ++I) | |||
| 365 | for (unsigned i = 0, e = PN->getNumIncomingValues(); i != e; ++i) { | |||
| 366 | ConstantExpr *CE = dyn_cast<ConstantExpr>(PN->getIncomingValue(i)); | |||
| 367 | if (!CE || !CE->canTrap()) continue; | |||
| 368 | ||||
| 369 | // The only case we have to worry about is when the edge is critical. | |||
| 370 | // Since this block has a PHI Node, we assume it has multiple input | |||
| 371 | // edges: check to see if the pred has multiple successors. | |||
| 372 | BasicBlock *Pred = PN->getIncomingBlock(i); | |||
| 373 | if (Pred->getTerminator()->getNumSuccessors() == 1) | |||
| 374 | continue; | |||
| 375 | ||||
| 376 | // Okay, we have to split this edge. | |||
| 377 | SplitCriticalEdge( | |||
| 378 | Pred->getTerminator(), GetSuccessorNumber(Pred, &BB), | |||
| 379 | CriticalEdgeSplittingOptions(DT, LI).setMergeIdenticalEdges()); | |||
| 380 | goto ReprocessBlock; | |||
| 381 | } | |||
| 382 | } | |||
| 383 | } | |||
| 384 | ||||
| 385 | static void computeUsesMSVCFloatingPoint(const Triple &TT, const Function &F, | |||
| 386 | MachineModuleInfo &MMI) { | |||
| 387 | // Only needed for MSVC | |||
| 388 | if (!TT.isWindowsMSVCEnvironment()) | |||
| 389 | return; | |||
| 390 | ||||
| 391 | // If it's already set, nothing to do. | |||
| 392 | if (MMI.usesMSVCFloatingPoint()) | |||
| 393 | return; | |||
| 394 | ||||
| 395 | for (const Instruction &I : instructions(F)) { | |||
| 396 | if (I.getType()->isFPOrFPVectorTy()) { | |||
| 397 | MMI.setUsesMSVCFloatingPoint(true); | |||
| 398 | return; | |||
| 399 | } | |||
| 400 | for (const auto &Op : I.operands()) { | |||
| 401 | if (Op->getType()->isFPOrFPVectorTy()) { | |||
| 402 | MMI.setUsesMSVCFloatingPoint(true); | |||
| 403 | return; | |||
| 404 | } | |||
| 405 | } | |||
| 406 | } | |||
| 407 | } | |||
| 408 | ||||
| 409 | bool SelectionDAGISel::runOnMachineFunction(MachineFunction &mf) { | |||
| 410 | // If we already selected that function, we do not need to run SDISel. | |||
| 411 | if (mf.getProperties().hasProperty( | |||
| ||||
| 412 | MachineFunctionProperties::Property::Selected)) | |||
| 413 | return false; | |||
| 414 | // Do some sanity-checking on the command-line options. | |||
| 415 | assert((!EnableFastISelAbort || TM.Options.EnableFastISel) &&(((!EnableFastISelAbort || TM.Options.EnableFastISel) && "-fast-isel-abort > 0 requires -fast-isel") ? static_cast <void> (0) : __assert_fail ("(!EnableFastISelAbort || TM.Options.EnableFastISel) && \"-fast-isel-abort > 0 requires -fast-isel\"" , "/build/llvm-toolchain-snapshot-10~++20200112100611+7fa5290d5bd/llvm/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp" , 416, __PRETTY_FUNCTION__)) | |||
| 416 | "-fast-isel-abort > 0 requires -fast-isel")(((!EnableFastISelAbort || TM.Options.EnableFastISel) && "-fast-isel-abort > 0 requires -fast-isel") ? static_cast <void> (0) : __assert_fail ("(!EnableFastISelAbort || TM.Options.EnableFastISel) && \"-fast-isel-abort > 0 requires -fast-isel\"" , "/build/llvm-toolchain-snapshot-10~++20200112100611+7fa5290d5bd/llvm/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp" , 416, __PRETTY_FUNCTION__)); | |||
| 417 | ||||
| 418 | const Function &Fn = mf.getFunction(); | |||
| 419 | MF = &mf; | |||
| 420 | ||||
| 421 | // Reset the target options before resetting the optimization | |||
| 422 | // level below. | |||
| 423 | // FIXME: This is a horrible hack and should be processed via | |||
| 424 | // codegen looking at the optimization level explicitly when | |||
| 425 | // it wants to look at it. | |||
| 426 | TM.resetTargetOptions(Fn); | |||
| 427 | // Reset OptLevel to None for optnone functions. | |||
| 428 | CodeGenOpt::Level NewOptLevel = OptLevel; | |||
| 429 | if (OptLevel != CodeGenOpt::None && skipFunction(Fn)) | |||
| 430 | NewOptLevel = CodeGenOpt::None; | |||
| 431 | OptLevelChanger OLC(*this, NewOptLevel); | |||
| 432 | ||||
| 433 | TII = MF->getSubtarget().getInstrInfo(); | |||
| 434 | TLI = MF->getSubtarget().getTargetLowering(); | |||
| 435 | RegInfo = &MF->getRegInfo(); | |||
| 436 | LibInfo = &getAnalysis<TargetLibraryInfoWrapperPass>().getTLI(Fn); | |||
| 437 | GFI = Fn.hasGC() ? &getAnalysis<GCModuleInfo>().getFunctionInfo(Fn) : nullptr; | |||
| 438 | ORE = std::make_unique<OptimizationRemarkEmitter>(&Fn); | |||
| 439 | auto *DTWP = getAnalysisIfAvailable<DominatorTreeWrapperPass>(); | |||
| 440 | DominatorTree *DT = DTWP
| |||
| 441 | auto *LIWP = getAnalysisIfAvailable<LoopInfoWrapperPass>(); | |||
| 442 | LoopInfo *LI = LIWP ? &LIWP->getLoopInfo() : nullptr; | |||
| 443 | auto *PSI = &getAnalysis<ProfileSummaryInfoWrapperPass>().getPSI(); | |||
| 444 | auto *BFI = (PSI
| |||
| 445 | &getAnalysis<LazyBlockFrequencyInfoPass>().getBFI() : | |||
| 446 | nullptr; | |||
| 447 | ||||
| 448 | LLVM_DEBUG(dbgs() << "\n\n\n=== " << Fn.getName() << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("isel")) { dbgs() << "\n\n\n=== " << Fn.getName( ) << "\n"; } } while (false); | |||
| 449 | ||||
| 450 | SplitCriticalSideEffectEdges(const_cast<Function &>(Fn), DT, LI); | |||
| 451 | ||||
| 452 | CurDAG->init(*MF, *ORE, this, LibInfo, | |||
| 453 | getAnalysisIfAvailable<LegacyDivergenceAnalysis>(), PSI, BFI); | |||
| 454 | FuncInfo->set(Fn, *MF, CurDAG); | |||
| 455 | SwiftError->setFunction(*MF); | |||
| 456 | ||||
| 457 | // Now get the optional analyzes if we want to. | |||
| 458 | // This is based on the possibly changed OptLevel (after optnone is taken | |||
| 459 | // into account). That's unfortunate but OK because it just means we won't | |||
| 460 | // ask for passes that have been required anyway. | |||
| 461 | ||||
| 462 | if (UseMBPI && OptLevel != CodeGenOpt::None) | |||
| 463 | FuncInfo->BPI = &getAnalysis<BranchProbabilityInfoWrapperPass>().getBPI(); | |||
| 464 | else | |||
| 465 | FuncInfo->BPI = nullptr; | |||
| 466 | ||||
| 467 | if (OptLevel != CodeGenOpt::None) | |||
| 468 | AA = &getAnalysis<AAResultsWrapperPass>().getAAResults(); | |||
| 469 | else | |||
| 470 | AA = nullptr; | |||
| 471 | ||||
| 472 | SDB->init(GFI, AA, LibInfo); | |||
| 473 | ||||
| 474 | MF->setHasInlineAsm(false); | |||
| 475 | ||||
| 476 | FuncInfo->SplitCSR = false; | |||
| 477 | ||||
| 478 | // We split CSR if the target supports it for the given function | |||
| 479 | // and the function has only return exits. | |||
| 480 | if (OptLevel
| |||
| 481 | FuncInfo->SplitCSR = true; | |||
| 482 | ||||
| 483 | // Collect all the return blocks. | |||
| 484 | for (const BasicBlock &BB : Fn) { | |||
| 485 | if (!succ_empty(&BB)) | |||
| 486 | continue; | |||
| 487 | ||||
| 488 | const Instruction *Term = BB.getTerminator(); | |||
| 489 | if (isa<UnreachableInst>(Term) || isa<ReturnInst>(Term)) | |||
| 490 | continue; | |||
| 491 | ||||
| 492 | // Bail out if the exit block is not Return nor Unreachable. | |||
| 493 | FuncInfo->SplitCSR = false; | |||
| 494 | break; | |||
| 495 | } | |||
| 496 | } | |||
| 497 | ||||
| 498 | MachineBasicBlock *EntryMBB = &MF->front(); | |||
| 499 | if (FuncInfo->SplitCSR
| |||
| 500 | // This performs initialization so lowering for SplitCSR will be correct. | |||
| 501 | TLI->initializeSplitCSR(EntryMBB); | |||
| 502 | ||||
| 503 | SelectAllBasicBlocks(Fn); | |||
| 504 | if (FastISelFailed && EnableFastISelFallbackReport) { | |||
| 505 | DiagnosticInfoISelFallback DiagFallback(Fn); | |||
| 506 | Fn.getContext().diagnose(DiagFallback); | |||
| 507 | } | |||
| 508 | ||||
| 509 | // Replace forward-declared registers with the registers containing | |||
| 510 | // the desired value. | |||
| 511 | // Note: it is important that this happens **before** the call to | |||
| 512 | // EmitLiveInCopies, since implementations can skip copies of unused | |||
| 513 | // registers. If we don't apply the reg fixups before, some registers may | |||
| 514 | // appear as unused and will be skipped, resulting in bad MI. | |||
| 515 | MachineRegisterInfo &MRI = MF->getRegInfo(); | |||
| 516 | for (DenseMap<unsigned, unsigned>::iterator I = FuncInfo->RegFixups.begin(), | |||
| 517 | E = FuncInfo->RegFixups.end(); | |||
| 518 | I != E; ++I) { | |||
| 519 | unsigned From = I->first; | |||
| 520 | unsigned To = I->second; | |||
| 521 | // If To is also scheduled to be replaced, find what its ultimate | |||
| 522 | // replacement is. | |||
| 523 | while (true) { | |||
| 524 | DenseMap<unsigned, unsigned>::iterator J = FuncInfo->RegFixups.find(To); | |||
| 525 | if (J == E) | |||
| 526 | break; | |||
| 527 | To = J->second; | |||
| 528 | } | |||
| 529 | // Make sure the new register has a sufficiently constrained register class. | |||
| 530 | if (Register::isVirtualRegister(From) && Register::isVirtualRegister(To)) | |||
| 531 | MRI.constrainRegClass(To, MRI.getRegClass(From)); | |||
| 532 | // Replace it. | |||
| 533 | ||||
| 534 | // Replacing one register with another won't touch the kill flags. | |||
| 535 | // We need to conservatively clear the kill flags as a kill on the old | |||
| 536 | // register might dominate existing uses of the new register. | |||
| 537 | if (!MRI.use_empty(To)) | |||
| 538 | MRI.clearKillFlags(From); | |||
| 539 | MRI.replaceRegWith(From, To); | |||
| 540 | } | |||
| 541 | ||||
| 542 | // If the first basic block in the function has live ins that need to be | |||
| 543 | // copied into vregs, emit the copies into the top of the block before | |||
| 544 | // emitting the code for the block. | |||
| 545 | const TargetRegisterInfo &TRI = *MF->getSubtarget().getRegisterInfo(); | |||
| 546 | RegInfo->EmitLiveInCopies(EntryMBB, TRI, *TII); | |||
| 547 | ||||
| 548 | // Insert copies in the entry block and the return blocks. | |||
| 549 | if (FuncInfo->SplitCSR) { | |||
| 550 | SmallVector<MachineBasicBlock*, 4> Returns; | |||
| 551 | // Collect all the return blocks. | |||
| 552 | for (MachineBasicBlock &MBB : mf) { | |||
| 553 | if (!MBB.succ_empty()) | |||
| 554 | continue; | |||
| 555 | ||||
| 556 | MachineBasicBlock::iterator Term = MBB.getFirstTerminator(); | |||
| 557 | if (Term != MBB.end() && Term->isReturn()) { | |||
| 558 | Returns.push_back(&MBB); | |||
| 559 | continue; | |||
| 560 | } | |||
| 561 | } | |||
| 562 | TLI->insertCopiesSplitCSR(EntryMBB, Returns); | |||
| 563 | } | |||
| 564 | ||||
| 565 | DenseMap<unsigned, unsigned> LiveInMap; | |||
| 566 | if (!FuncInfo->ArgDbgValues.empty()) | |||
| 567 | for (std::pair<unsigned, unsigned> LI : RegInfo->liveins()) | |||
| 568 | if (LI.second) | |||
| 569 | LiveInMap.insert(LI); | |||
| 570 | ||||
| 571 | // Insert DBG_VALUE instructions for function arguments to the entry block. | |||
| 572 | for (unsigned i = 0, e = FuncInfo->ArgDbgValues.size(); i
| |||
| 573 | MachineInstr *MI = FuncInfo->ArgDbgValues[e-i-1]; | |||
| 574 | bool hasFI = MI->getOperand(0).isFI(); | |||
| 575 | Register Reg = | |||
| 576 | hasFI
| |||
| 577 | if (Register::isPhysicalRegister(Reg)) | |||
| 578 | EntryMBB->insert(EntryMBB->begin(), MI); | |||
| 579 | else { | |||
| 580 | MachineInstr *Def = RegInfo->getVRegDef(Reg); | |||
| 581 | if (Def) { | |||
| 582 | MachineBasicBlock::iterator InsertPos = Def; | |||
| 583 | // FIXME: VR def may not be in entry block. | |||
| 584 | Def->getParent()->insert(std::next(InsertPos), MI); | |||
| 585 | } else | |||
| 586 | LLVM_DEBUG(dbgs() << "Dropping debug info for dead vreg"do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("isel")) { dbgs() << "Dropping debug info for dead vreg" << Register::virtReg2Index(Reg) << "\n"; } } while (false) | |||
| 587 | << Register::virtReg2Index(Reg) << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("isel")) { dbgs() << "Dropping debug info for dead vreg" << Register::virtReg2Index(Reg) << "\n"; } } while (false); | |||
| 588 | } | |||
| 589 | ||||
| 590 | // If Reg is live-in then update debug info to track its copy in a vreg. | |||
| 591 | DenseMap<unsigned, unsigned>::iterator LDI = LiveInMap.find(Reg); | |||
| 592 | if (LDI != LiveInMap.end()) { | |||
| 593 | assert(!hasFI && "There's no handling of frame pointer updating here yet "((!hasFI && "There's no handling of frame pointer updating here yet " "- add if needed") ? static_cast<void> (0) : __assert_fail ("!hasFI && \"There's no handling of frame pointer updating here yet \" \"- add if needed\"" , "/build/llvm-toolchain-snapshot-10~++20200112100611+7fa5290d5bd/llvm/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp" , 594, __PRETTY_FUNCTION__)) | |||
| 594 | "- add if needed")((!hasFI && "There's no handling of frame pointer updating here yet " "- add if needed") ? static_cast<void> (0) : __assert_fail ("!hasFI && \"There's no handling of frame pointer updating here yet \" \"- add if needed\"" , "/build/llvm-toolchain-snapshot-10~++20200112100611+7fa5290d5bd/llvm/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp" , 594, __PRETTY_FUNCTION__)); | |||
| 595 | MachineInstr *Def = RegInfo->getVRegDef(LDI->second); | |||
| 596 | MachineBasicBlock::iterator InsertPos = Def; | |||
| 597 | const MDNode *Variable = MI->getDebugVariable(); | |||
| 598 | const MDNode *Expr = MI->getDebugExpression(); | |||
| 599 | DebugLoc DL = MI->getDebugLoc(); | |||
| 600 | bool IsIndirect = MI->isIndirectDebugValue(); | |||
| 601 | if (IsIndirect) | |||
| 602 | assert(MI->getOperand(1).getImm() == 0 &&((MI->getOperand(1).getImm() == 0 && "DBG_VALUE with nonzero offset" ) ? static_cast<void> (0) : __assert_fail ("MI->getOperand(1).getImm() == 0 && \"DBG_VALUE with nonzero offset\"" , "/build/llvm-toolchain-snapshot-10~++20200112100611+7fa5290d5bd/llvm/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp" , 603, __PRETTY_FUNCTION__)) | |||
| 603 | "DBG_VALUE with nonzero offset")((MI->getOperand(1).getImm() == 0 && "DBG_VALUE with nonzero offset" ) ? static_cast<void> (0) : __assert_fail ("MI->getOperand(1).getImm() == 0 && \"DBG_VALUE with nonzero offset\"" , "/build/llvm-toolchain-snapshot-10~++20200112100611+7fa5290d5bd/llvm/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp" , 603, __PRETTY_FUNCTION__)); | |||
| 604 | assert(cast<DILocalVariable>(Variable)->isValidLocationForIntrinsic(DL) &&((cast<DILocalVariable>(Variable)->isValidLocationForIntrinsic (DL) && "Expected inlined-at fields to agree") ? static_cast <void> (0) : __assert_fail ("cast<DILocalVariable>(Variable)->isValidLocationForIntrinsic(DL) && \"Expected inlined-at fields to agree\"" , "/build/llvm-toolchain-snapshot-10~++20200112100611+7fa5290d5bd/llvm/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp" , 605, __PRETTY_FUNCTION__)) | |||
| 605 | "Expected inlined-at fields to agree")((cast<DILocalVariable>(Variable)->isValidLocationForIntrinsic (DL) && "Expected inlined-at fields to agree") ? static_cast <void> (0) : __assert_fail ("cast<DILocalVariable>(Variable)->isValidLocationForIntrinsic(DL) && \"Expected inlined-at fields to agree\"" , "/build/llvm-toolchain-snapshot-10~++20200112100611+7fa5290d5bd/llvm/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp" , 605, __PRETTY_FUNCTION__)); | |||
| 606 | // Def is never a terminator here, so it is ok to increment InsertPos. | |||
| 607 | BuildMI(*EntryMBB, ++InsertPos, DL, TII->get(TargetOpcode::DBG_VALUE), | |||
| 608 | IsIndirect, LDI->second, Variable, Expr); | |||
| 609 | ||||
| 610 | // If this vreg is directly copied into an exported register then | |||
| 611 | // that COPY instructions also need DBG_VALUE, if it is the only | |||
| 612 | // user of LDI->second. | |||
| 613 | MachineInstr *CopyUseMI = nullptr; | |||
| 614 | for (MachineRegisterInfo::use_instr_iterator | |||
| 615 | UI = RegInfo->use_instr_begin(LDI->second), | |||
| 616 | E = RegInfo->use_instr_end(); UI != E; ) { | |||
| 617 | MachineInstr *UseMI = &*(UI++); | |||
| 618 | if (UseMI->isDebugValue()) continue; | |||
| 619 | if (UseMI->isCopy() && !CopyUseMI && UseMI->getParent() == EntryMBB) { | |||
| 620 | CopyUseMI = UseMI; continue; | |||
| 621 | } | |||
| 622 | // Otherwise this is another use or second copy use. | |||
| 623 | CopyUseMI = nullptr; break; | |||
| 624 | } | |||
| 625 | if (CopyUseMI) { | |||
| 626 | // Use MI's debug location, which describes where Variable was | |||
| 627 | // declared, rather than whatever is attached to CopyUseMI. | |||
| 628 | MachineInstr *NewMI = | |||
| 629 | BuildMI(*MF, DL, TII->get(TargetOpcode::DBG_VALUE), IsIndirect, | |||
| 630 | CopyUseMI->getOperand(0).getReg(), Variable, Expr); | |||
| 631 | MachineBasicBlock::iterator Pos = CopyUseMI; | |||
| 632 | EntryMBB->insertAfter(Pos, NewMI); | |||
| 633 | } | |||
| 634 | } | |||
| 635 | } | |||
| 636 | ||||
| 637 | // Determine if there are any calls in this machine function. | |||
| 638 | MachineFrameInfo &MFI = MF->getFrameInfo(); | |||
| 639 | for (const auto &MBB : *MF) { | |||
| 640 | if (MFI.hasCalls() && MF->hasInlineAsm()) | |||
| 641 | break; | |||
| 642 | ||||
| 643 | for (const auto &MI : MBB) { | |||
| 644 | const MCInstrDesc &MCID = TII->get(MI.getOpcode()); | |||
| 645 | if ((MCID.isCall() && !MCID.isReturn()) || | |||
| 646 | MI.isStackAligningInlineAsm()) { | |||
| 647 | MFI.setHasCalls(true); | |||
| 648 | } | |||
| 649 | if (MI.isInlineAsm()) { | |||
| 650 | MF->setHasInlineAsm(true); | |||
| 651 | } | |||
| 652 | } | |||
| 653 | } | |||
| 654 | ||||
| 655 | // Determine if there is a call to setjmp in the machine function. | |||
| 656 | MF->setExposesReturnsTwice(Fn.callsFunctionThatReturnsTwice()); | |||
| 657 | ||||
| 658 | // Determine if floating point is used for msvc | |||
| 659 | computeUsesMSVCFloatingPoint(TM.getTargetTriple(), Fn, MF->getMMI()); | |||
| 660 | ||||
| 661 | // Replace forward-declared registers with the registers containing | |||
| 662 | // the desired value. | |||
| 663 | for (DenseMap<unsigned, unsigned>::iterator | |||
| 664 | I = FuncInfo->RegFixups.begin(), E = FuncInfo->RegFixups.end(); | |||
| 665 | I != E; ++I) { | |||
| 666 | unsigned From = I->first; | |||
| 667 | unsigned To = I->second; | |||
| 668 | // If To is also scheduled to be replaced, find what its ultimate | |||
| 669 | // replacement is. | |||
| 670 | while (true) { | |||
| 671 | DenseMap<unsigned, unsigned>::iterator J = FuncInfo->RegFixups.find(To); | |||
| 672 | if (J == E) break; | |||
| 673 | To = J->second; | |||
| 674 | } | |||
| 675 | // Make sure the new register has a sufficiently constrained register class. | |||
| 676 | if (Register::isVirtualRegister(From) && Register::isVirtualRegister(To)) | |||
| 677 | MRI.constrainRegClass(To, MRI.getRegClass(From)); | |||
| 678 | // Replace it. | |||
| 679 | ||||
| 680 | ||||
| 681 | // Replacing one register with another won't touch the kill flags. | |||
| 682 | // We need to conservatively clear the kill flags as a kill on the old | |||
| 683 | // register might dominate existing uses of the new register. | |||
| 684 | if (!MRI.use_empty(To)) | |||
| 685 | MRI.clearKillFlags(From); | |||
| 686 | MRI.replaceRegWith(From, To); | |||
| 687 | } | |||
| 688 | ||||
| 689 | TLI->finalizeLowering(*MF); | |||
| 690 | ||||
| 691 | // Release function-specific state. SDB and CurDAG are already cleared | |||
| 692 | // at this point. | |||
| 693 | FuncInfo->clear(); | |||
| 694 | ||||
| 695 | LLVM_DEBUG(dbgs() << "*** MachineFunction at end of ISel ***\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("isel")) { dbgs() << "*** MachineFunction at end of ISel ***\n" ; } } while (false); | |||
| 696 | LLVM_DEBUG
("isel")) { MF->print(dbgs()); } } while (false); | |||
| 697 | ||||
| 698 | return true; | |||
| 699 | } | |||
| 700 | ||||
| 701 | static void reportFastISelFailure(MachineFunction &MF, | |||
| 702 | OptimizationRemarkEmitter &ORE, | |||
| 703 | OptimizationRemarkMissed &R, | |||
| 704 | bool ShouldAbort) { | |||
| 705 | // Print the function name explicitly if we don't have a debug location (which | |||
| 706 | // makes the diagnostic less useful) or if we're going to emit a raw error. | |||
| 707 | if (!R.getLocation().isValid() || ShouldAbort) | |||
| 708 | R << (" (in function: " + MF.getName() + ")").str(); | |||
| 709 | ||||
| 710 | if (ShouldAbort) | |||
| 711 | report_fatal_error(R.getMsg()); | |||
| 712 | ||||
| 713 | ORE.emit(R); | |||
| 714 | } | |||
| 715 | ||||
| 716 | void SelectionDAGISel::SelectBasicBlock(BasicBlock::const_iterator Begin, | |||
| 717 | BasicBlock::const_iterator End, | |||
| 718 | bool &HadTailCall) { | |||
| 719 | // Allow creating illegal types during DAG building for the basic block. | |||
| 720 | CurDAG->NewNodesMustHaveLegalTypes = false; | |||
| 721 | ||||
| 722 | // Lower the instructions. If a call is emitted as a tail call, cease emitting | |||
| 723 | // nodes for this block. | |||
| 724 | for (BasicBlock::const_iterator I = Begin; I != End && !SDB->HasTailCall; ++I) { | |||
| 725 | if (!ElidedArgCopyInstrs.count(&*I)) | |||
| 726 | SDB->visit(*I); | |||
| 727 | } | |||
| 728 | ||||
| 729 | // Make sure the root of the DAG is up-to-date. | |||
| 730 | CurDAG->setRoot(SDB->getControlRoot()); | |||
| 731 | HadTailCall = SDB->HasTailCall; | |||
| 732 | SDB->resolveOrClearDbgInfo(); | |||
| 733 | SDB->clear(); | |||
| 734 | ||||
| 735 | // Final step, emit the lowered DAG as machine code. | |||
| 736 | CodeGenAndEmitDAG(); | |||
| 737 | } | |||
| 738 | ||||
| 739 | void SelectionDAGISel::ComputeLiveOutVRegInfo() { | |||
| 740 | SmallPtrSet<SDNode *, 16> Added; | |||
| 741 | SmallVector<SDNode*, 128> Worklist; | |||
| 742 | ||||
| 743 | Worklist.push_back(CurDAG->getRoot().getNode()); | |||
| 744 | Added.insert(CurDAG->getRoot().getNode()); | |||
| 745 | ||||
| 746 | KnownBits Known; | |||
| 747 | ||||
| 748 | do { | |||
| 749 | SDNode *N = Worklist.pop_back_val(); | |||
| 750 | ||||
| 751 | // Otherwise, add all chain operands to the worklist. | |||
| 752 | for (const SDValue &Op : N->op_values()) | |||
| 753 | if (Op.getValueType() == MVT::Other && Added.insert(Op.getNode()).second) | |||
| 754 | Worklist.push_back(Op.getNode()); | |||
| 755 | ||||
| 756 | // If this is a CopyToReg with a vreg dest, process it. | |||
| 757 | if (N->getOpcode() != ISD::CopyToReg) | |||
| 758 | continue; | |||
| 759 | ||||
| 760 | unsigned DestReg = cast<RegisterSDNode>(N->getOperand(1))->getReg(); | |||
| 761 | if (!Register::isVirtualRegister(DestReg)) | |||
| 762 | continue; | |||
| 763 | ||||
| 764 | // Ignore non-integer values. | |||
| 765 | SDValue Src = N->getOperand(2); | |||
| 766 | EVT SrcVT = Src.getValueType(); | |||
| 767 | if (!SrcVT.isInteger()) | |||
| 768 | continue; | |||
| 769 | ||||
| 770 | unsigned NumSignBits = CurDAG->ComputeNumSignBits(Src); | |||
| 771 | Known = CurDAG->computeKnownBits(Src); | |||
| 772 | FuncInfo->AddLiveOutRegInfo(DestReg, NumSignBits, Known); | |||
| 773 | } while (!Worklist.empty()); | |||
| 774 | } | |||
| 775 | ||||
| 776 | void SelectionDAGISel::CodeGenAndEmitDAG() { | |||
| 777 | StringRef GroupName = "sdag"; | |||
| 778 | StringRef GroupDescription = "Instruction Selection and Scheduling"; | |||
| 779 | std::string BlockName; | |||
| 780 | bool MatchFilterBB = false; (void)MatchFilterBB; | |||
| 781 | #ifndef NDEBUG | |||
| 782 | TargetTransformInfo &TTI = | |||
| 783 | getAnalysis<TargetTransformInfoWrapperPass>().getTTI(*FuncInfo->Fn); | |||
| 784 | #endif | |||
| 785 | ||||
| 786 | // Pre-type legalization allow creation of any node types. | |||
| 787 | CurDAG->NewNodesMustHaveLegalTypes = false; | |||
| 788 | ||||
| 789 | #ifndef NDEBUG | |||
| 790 | MatchFilterBB = (FilterDAGBasicBlockName.empty() || | |||
| 791 | FilterDAGBasicBlockName == | |||
| 792 | FuncInfo->MBB->getBasicBlock()->getName()); | |||
| 793 | #endif | |||
| 794 | #ifdef NDEBUG | |||
| 795 | if (ViewDAGCombine1 || ViewLegalizeTypesDAGs || ViewDAGCombineLT || | |||
| 796 | ViewLegalizeDAGs || ViewDAGCombine2 || ViewISelDAGs || ViewSchedDAGs || | |||
| 797 | ViewSUnitDAGs) | |||
| 798 | #endif | |||
| 799 | { | |||
| 800 | BlockName = | |||
| 801 | (MF->getName() + ":" + FuncInfo->MBB->getBasicBlock()->getName()).str(); | |||
| 802 | } | |||
| 803 | LLVM_DEBUG(dbgs() << "Initial selection DAG: "do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("isel")) { dbgs() << "Initial selection DAG: " << printMBBReference(*FuncInfo->MBB) << " '" << BlockName << "'\n"; CurDAG->dump(); } } while (false) | |||
| 804 | << printMBBReference(*FuncInfo->MBB) << " '" << BlockNamedo { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("isel")) { dbgs() << "Initial selection DAG: " << printMBBReference(*FuncInfo->MBB) << " '" << BlockName << "'\n"; CurDAG->dump(); } } while (false) | |||
| 805 | << "'\n";do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("isel")) { dbgs() << "Initial selection DAG: " << printMBBReference(*FuncInfo->MBB) << " '" << BlockName << "'\n"; CurDAG->dump(); } } while (false) | |||
| 806 | CurDAG->dump())do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("isel")) { dbgs() << "Initial selection DAG: " << printMBBReference(*FuncInfo->MBB) << " '" << BlockName << "'\n"; CurDAG->dump(); } } while (false); | |||
| 807 | ||||
| 808 | if (ViewDAGCombine1 && MatchFilterBB) | |||
| 809 | CurDAG->viewGraph("dag-combine1 input for " + BlockName); | |||
| 810 | ||||
| 811 | // Run the DAG combiner in pre-legalize mode. | |||
| 812 | { | |||
| 813 | NamedRegionTimer T("combine1", "DAG Combining 1", GroupName, | |||
| 814 | GroupDescription, TimePassesIsEnabled); | |||
| 815 | CurDAG->Combine(BeforeLegalizeTypes, AA, OptLevel); | |||
| 816 | } | |||
| 817 | ||||
| 818 | #ifndef NDEBUG | |||
| 819 | if (TTI.hasBranchDivergence()) | |||
| 820 | CurDAG->VerifyDAGDiverence(); | |||
| 821 | #endif | |||
| 822 | ||||
| 823 | LLVM_DEBUG(dbgs() << "Optimized lowered selection DAG: "do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("isel")) { dbgs() << "Optimized lowered selection DAG: " << printMBBReference(*FuncInfo->MBB) << " '" << BlockName << "'\n"; CurDAG->dump(); } } while (false ) | |||
| 824 | << printMBBReference(*FuncInfo->MBB) << " '" << BlockNamedo { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("isel")) { dbgs() << "Optimized lowered selection DAG: " << printMBBReference(*FuncInfo->MBB) << " '" << BlockName << "'\n"; CurDAG->dump(); } } while (false ) | |||
| 825 | << "'\n";do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("isel")) { dbgs() << "Optimized lowered selection DAG: " << printMBBReference(*FuncInfo->MBB) << " '" << BlockName << "'\n"; CurDAG->dump(); } } while (false ) | |||
| 826 | CurDAG->dump())do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("isel")) { dbgs() << "Optimized lowered selection DAG: " << printMBBReference(*FuncInfo->MBB) << " '" << BlockName << "'\n"; CurDAG->dump(); } } while (false ); | |||
| 827 | ||||
| 828 | // Second step, hack on the DAG until it only uses operations and types that | |||
| 829 | // the target supports. | |||
| 830 | if (ViewLegalizeTypesDAGs && MatchFilterBB) | |||
| 831 | CurDAG->viewGraph("legalize-types input for " + BlockName); | |||
| 832 | ||||
| 833 | bool Changed; | |||
| 834 | { | |||
| 835 | NamedRegionTimer T("legalize_types", "Type Legalization", GroupName, | |||
| 836 | GroupDescription, TimePassesIsEnabled); | |||
| 837 | Changed = CurDAG->LegalizeTypes(); | |||
| 838 | } | |||
| 839 | ||||
| 840 | #ifndef NDEBUG | |||
| 841 | if (TTI.hasBranchDivergence()) | |||
| 842 | CurDAG->VerifyDAGDiverence(); | |||
| 843 | #endif | |||
| 844 | ||||
| 845 | LLVM_DEBUG(dbgs() << "Type-legalized selection DAG: "do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("isel")) { dbgs() << "Type-legalized selection DAG: " << printMBBReference(*FuncInfo->MBB) << " '" << BlockName << "'\n"; CurDAG->dump(); } } while (false) | |||
| 846 | << printMBBReference(*FuncInfo->MBB) << " '" << BlockNamedo { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("isel")) { dbgs() << "Type-legalized selection DAG: " << printMBBReference(*FuncInfo->MBB) << " '" << BlockName << "'\n"; CurDAG->dump(); } } while (false) | |||
| 847 | << "'\n";do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("isel")) { dbgs() << "Type-legalized selection DAG: " << printMBBReference(*FuncInfo->MBB) << " '" << BlockName << "'\n"; CurDAG->dump(); } } while (false) | |||
| 848 | CurDAG->dump())do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("isel")) { dbgs() << "Type-legalized selection DAG: " << printMBBReference(*FuncInfo->MBB) << " '" << BlockName << "'\n"; CurDAG->dump(); } } while (false); | |||
| 849 | ||||
| 850 | // Only allow creation of legal node types. | |||
| 851 | CurDAG->NewNodesMustHaveLegalTypes = true; | |||
| 852 | ||||
| 853 | if (Changed) { | |||
| 854 | if (ViewDAGCombineLT && MatchFilterBB) | |||
| 855 | CurDAG->viewGraph("dag-combine-lt input for " + BlockName); | |||
| 856 | ||||
| 857 | // Run the DAG combiner in post-type-legalize mode. | |||
| 858 | { | |||
| 859 | NamedRegionTimer T("combine_lt", "DAG Combining after legalize types", | |||
| 860 | GroupName, GroupDescription, TimePassesIsEnabled); | |||
| 861 | CurDAG->Combine(AfterLegalizeTypes, AA, OptLevel); | |||
| 862 | } | |||
| 863 | ||||
| 864 | #ifndef NDEBUG | |||
| 865 | if (TTI.hasBranchDivergence()) | |||
| 866 | CurDAG->VerifyDAGDiverence(); | |||
| 867 | #endif | |||
| 868 | ||||
| 869 | LLVM_DEBUG(dbgs() << "Optimized type-legalized selection DAG: "do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("isel")) { dbgs() << "Optimized type-legalized selection DAG: " << printMBBReference(*FuncInfo->MBB) << " '" << BlockName << "'\n"; CurDAG->dump(); } } while (false ) | |||
| 870 | << printMBBReference(*FuncInfo->MBB) << " '" << BlockNamedo { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("isel")) { dbgs() << "Optimized type-legalized selection DAG: " << printMBBReference(*FuncInfo->MBB) << " '" << BlockName << "'\n"; CurDAG->dump(); } } while (false ) | |||
| 871 | << "'\n";do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("isel")) { dbgs() << "Optimized type-legalized selection DAG: " << printMBBReference(*FuncInfo->MBB) << " '" << BlockName << "'\n"; CurDAG->dump(); } } while (false ) | |||
| 872 | CurDAG->dump())do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("isel")) { dbgs() << "Optimized type-legalized selection DAG: " << printMBBReference(*FuncInfo->MBB) << " '" << BlockName << "'\n"; CurDAG->dump(); } } while (false ); | |||
| 873 | } | |||
| 874 | ||||
| 875 | { | |||
| 876 | NamedRegionTimer T("legalize_vec", "Vector Legalization", GroupName, | |||
| 877 | GroupDescription, TimePassesIsEnabled); | |||
| 878 | Changed = CurDAG->LegalizeVectors(); | |||
| 879 | } | |||
| 880 | ||||
| 881 | if (Changed) { | |||
| 882 | LLVM_DEBUG(dbgs() << "Vector-legalized selection DAG: "do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("isel")) { dbgs() << "Vector-legalized selection DAG: " << printMBBReference(*FuncInfo->MBB) << " '" << BlockName << "'\n"; CurDAG->dump(); } } while (false ) | |||
| 883 | << printMBBReference(*FuncInfo->MBB) << " '" << BlockNamedo { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("isel")) { dbgs() << "Vector-legalized selection DAG: " << printMBBReference(*FuncInfo->MBB) << " '" << BlockName << "'\n"; CurDAG->dump(); } } while (false ) | |||
| 884 | << "'\n";do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("isel")) { dbgs() << "Vector-legalized selection DAG: " << printMBBReference(*FuncInfo->MBB) << " '" << BlockName << "'\n"; CurDAG->dump(); } } while (false ) | |||
| 885 | CurDAG->dump())do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("isel")) { dbgs() << "Vector-legalized selection DAG: " << printMBBReference(*FuncInfo->MBB) << " '" << BlockName << "'\n"; CurDAG->dump(); } } while (false ); | |||
| 886 | ||||
| 887 | { | |||
| 888 | NamedRegionTimer T("legalize_types2", "Type Legalization 2", GroupName, | |||
| 889 | GroupDescription, TimePassesIsEnabled); | |||
| 890 | CurDAG->LegalizeTypes(); | |||
| 891 | } | |||
| 892 | ||||
| 893 | LLVM_DEBUG(dbgs() << "Vector/type-legalized selection DAG: "do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("isel")) { dbgs() << "Vector/type-legalized selection DAG: " << printMBBReference(*FuncInfo->MBB) << " '" << BlockName << "'\n"; CurDAG->dump(); } } while (false ) | |||
| 894 | << printMBBReference(*FuncInfo->MBB) << " '" << BlockNamedo { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("isel")) { dbgs() << "Vector/type-legalized selection DAG: " << printMBBReference(*FuncInfo->MBB) << " '" << BlockName << "'\n"; CurDAG->dump(); } } while (false ) | |||
| 895 | << "'\n";do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("isel")) { dbgs() << "Vector/type-legalized selection DAG: " << printMBBReference(*FuncInfo->MBB) << " '" << BlockName << "'\n"; CurDAG->dump(); } } while (false ) | |||
| 896 | CurDAG->dump())do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("isel")) { dbgs() << "Vector/type-legalized selection DAG: " << printMBBReference(*FuncInfo->MBB) << " '" << BlockName << "'\n"; CurDAG->dump(); } } while (false ); | |||
| 897 | ||||
| 898 | if (ViewDAGCombineLT && MatchFilterBB) | |||
| 899 | CurDAG->viewGraph("dag-combine-lv input for " + BlockName); | |||
| 900 | ||||
| 901 | // Run the DAG combiner in post-type-legalize mode. | |||
| 902 | { | |||
| 903 | NamedRegionTimer T("combine_lv", "DAG Combining after legalize vectors", | |||
| 904 | GroupName, GroupDescription, TimePassesIsEnabled); | |||
| 905 | CurDAG->Combine(AfterLegalizeVectorOps, AA, OptLevel); | |||
| 906 | } | |||
| 907 | ||||
| 908 | LLVM_DEBUG(dbgs() << "Optimized vector-legalized selection DAG: "do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("isel")) { dbgs() << "Optimized vector-legalized selection DAG: " << printMBBReference(*FuncInfo->MBB) << " '" << BlockName << "'\n"; CurDAG->dump(); } } while (false ) | |||
| 909 | << printMBBReference(*FuncInfo->MBB) << " '" << BlockNamedo { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("isel")) { dbgs() << "Optimized vector-legalized selection DAG: " << printMBBReference(*FuncInfo->MBB) << " '" << BlockName << "'\n"; CurDAG->dump(); } } while (false ) | |||
| 910 | << "'\n";do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("isel")) { dbgs() << "Optimized vector-legalized selection DAG: " << printMBBReference(*FuncInfo->MBB) << " '" << BlockName << "'\n"; CurDAG->dump(); } } while (false ) | |||
| 911 | CurDAG->dump())do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("isel")) { dbgs() << "Optimized vector-legalized selection DAG: " << printMBBReference(*FuncInfo->MBB) << " '" << BlockName << "'\n"; CurDAG->dump(); } } while (false ); | |||
| 912 | ||||
| 913 | #ifndef NDEBUG | |||
| 914 | if (TTI.hasBranchDivergence()) | |||
| 915 | CurDAG->VerifyDAGDiverence(); | |||
| 916 | #endif | |||
| 917 | } | |||
| 918 | ||||
| 919 | if (ViewLegalizeDAGs && MatchFilterBB) | |||
| 920 | CurDAG->viewGraph("legalize input for " + BlockName); | |||
| 921 | ||||
| 922 | { | |||
| 923 | NamedRegionTimer T("legalize", "DAG Legalization", GroupName, | |||
| 924 | GroupDescription, TimePassesIsEnabled); | |||
| 925 | CurDAG->Legalize(); | |||
| 926 | } | |||
| 927 | ||||
| 928 | #ifndef NDEBUG | |||
| 929 | if (TTI.hasBranchDivergence()) | |||
| 930 | CurDAG->VerifyDAGDiverence(); | |||
| 931 | #endif | |||
| 932 | ||||
| 933 | LLVM_DEBUG(dbgs() << "Legalized selection DAG: "do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("isel")) { dbgs() << "Legalized selection DAG: " << printMBBReference(*FuncInfo->MBB) << " '" << BlockName << "'\n"; CurDAG->dump(); } } while (false) | |||
| 934 | << printMBBReference(*FuncInfo->MBB) << " '" << BlockNamedo { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("isel")) { dbgs() << "Legalized selection DAG: " << printMBBReference(*FuncInfo->MBB) << " '" << BlockName << "'\n"; CurDAG->dump(); } } while (false) | |||
| 935 | << "'\n";do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("isel")) { dbgs() << "Legalized selection DAG: " << printMBBReference(*FuncInfo->MBB) << " '" << BlockName << "'\n"; CurDAG->dump(); } } while (false) | |||
| 936 | CurDAG->dump())do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("isel")) { dbgs() << "Legalized selection DAG: " << printMBBReference(*FuncInfo->MBB) << " '" << BlockName << "'\n"; CurDAG->dump(); } } while (false); | |||
| 937 | ||||
| 938 | if (ViewDAGCombine2 && MatchFilterBB) | |||
| 939 | CurDAG->viewGraph("dag-combine2 input for " + BlockName); | |||
| 940 | ||||
| 941 | // Run the DAG combiner in post-legalize mode. | |||
| 942 | { | |||
| 943 | NamedRegionTimer T("combine2", "DAG Combining 2", GroupName, | |||
| 944 | GroupDescription, TimePassesIsEnabled); | |||
| 945 | CurDAG->Combine(AfterLegalizeDAG, AA, OptLevel); | |||
| 946 | } | |||
| 947 | ||||
| 948 | #ifndef NDEBUG | |||
| 949 | if (TTI.hasBranchDivergence()) | |||
| 950 | CurDAG->VerifyDAGDiverence(); | |||
| 951 | #endif | |||
| 952 | ||||
| 953 | LLVM_DEBUG(dbgs() << "Optimized legalized selection DAG: "do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("isel")) { dbgs() << "Optimized legalized selection DAG: " << printMBBReference(*FuncInfo->MBB) << " '" << BlockName << "'\n"; CurDAG->dump(); } } while (false ) | |||
| 954 | << printMBBReference(*FuncInfo->MBB) << " '" << BlockNamedo { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("isel")) { dbgs() << "Optimized legalized selection DAG: " << printMBBReference(*FuncInfo->MBB) << " '" << BlockName << "'\n"; CurDAG->dump(); } } while (false ) | |||
| 955 | << "'\n";do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("isel")) { dbgs() << "Optimized legalized selection DAG: " << printMBBReference(*FuncInfo->MBB) << " '" << BlockName << "'\n"; CurDAG->dump(); } } while (false ) | |||
| 956 | CurDAG->dump())do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("isel")) { dbgs() << "Optimized legalized selection DAG: " << printMBBReference(*FuncInfo->MBB) << " '" << BlockName << "'\n"; CurDAG->dump(); } } while (false ); | |||
| 957 | ||||
| 958 | if (OptLevel != CodeGenOpt::None) | |||
| 959 | ComputeLiveOutVRegInfo(); | |||
| 960 | ||||
| 961 | if (ViewISelDAGs && MatchFilterBB) | |||
| 962 | CurDAG->viewGraph("isel input for " + BlockName); | |||
| 963 | ||||
| 964 | // Third, instruction select all of the operations to machine code, adding the | |||
| 965 | // code to the MachineBasicBlock. | |||
| 966 | { | |||
| 967 | NamedRegionTimer T("isel", "Instruction Selection", GroupName, | |||
| 968 | GroupDescription, TimePassesIsEnabled); | |||
| 969 | DoInstructionSelection(); | |||
| 970 | } | |||
| 971 | ||||
| 972 | LLVM_DEBUG(dbgs() << "Selected selection DAG: "do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("isel")) { dbgs() << "Selected selection DAG: " << printMBBReference(*FuncInfo->MBB) << " '" << BlockName << "'\n"; CurDAG->dump(); } } while (false) | |||
| 973 | << printMBBReference(*FuncInfo->MBB) << " '" << BlockNamedo { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("isel")) { dbgs() << "Selected selection DAG: " << printMBBReference(*FuncInfo->MBB) << " '" << BlockName << "'\n"; CurDAG->dump(); } } while (false) | |||
| 974 | << "'\n";do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("isel")) { dbgs() << "Selected selection DAG: " << printMBBReference(*FuncInfo->MBB) << " '" << BlockName << "'\n"; CurDAG->dump(); } } while (false) | |||
| 975 | CurDAG->dump())do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("isel")) { dbgs() << "Selected selection DAG: " << printMBBReference(*FuncInfo->MBB) << " '" << BlockName << "'\n"; CurDAG->dump(); } } while (false); | |||
| 976 | ||||
| 977 | if (ViewSchedDAGs && MatchFilterBB) | |||
| 978 | CurDAG->viewGraph("scheduler input for " + BlockName); | |||
| 979 | ||||
| 980 | // Schedule machine code. | |||
| 981 | ScheduleDAGSDNodes *Scheduler = CreateScheduler(); | |||
| 982 | { | |||
| 983 | NamedRegionTimer T("sched", "Instruction Scheduling", GroupName, | |||
| 984 | GroupDescription, TimePassesIsEnabled); | |||
| 985 | Scheduler->Run(CurDAG, FuncInfo->MBB); | |||
| 986 | } | |||
| 987 | ||||
| 988 | if (ViewSUnitDAGs && MatchFilterBB) | |||
| 989 | Scheduler->viewGraph(); | |||
| 990 | ||||
| 991 | // Emit machine code to BB. This can change 'BB' to the last block being | |||
| 992 | // inserted into. | |||
| 993 | MachineBasicBlock *FirstMBB = FuncInfo->MBB, *LastMBB; | |||
| 994 | { | |||
| 995 | NamedRegionTimer T("emit", "Instruction Creation", GroupName, | |||
| 996 | GroupDescription, TimePassesIsEnabled); | |||
| 997 | ||||
| 998 | // FuncInfo->InsertPt is passed by reference and set to the end of the | |||
| 999 | // scheduled instructions. | |||
| 1000 | LastMBB = FuncInfo->MBB = Scheduler->EmitSchedule(FuncInfo->InsertPt); | |||
| 1001 | } | |||
| 1002 | ||||
| 1003 | // If the block was split, make sure we update any references that are used to | |||
| 1004 | // update PHI nodes later on. | |||
| 1005 | if (FirstMBB != LastMBB) | |||
| 1006 | SDB->UpdateSplitBlock(FirstMBB, LastMBB); | |||
| 1007 | ||||
| 1008 | // Free the scheduler state. | |||
| 1009 | { | |||
| 1010 | NamedRegionTimer T("cleanup", "Instruction Scheduling Cleanup", GroupName, | |||
| 1011 | GroupDescription, TimePassesIsEnabled); | |||
| 1012 | delete Scheduler; | |||
| 1013 | } | |||
| 1014 | ||||
| 1015 | // Free the SelectionDAG state, now that we're finished with it. | |||
| 1016 | CurDAG->clear(); | |||
| 1017 | } | |||
| 1018 | ||||
| 1019 | namespace { | |||
| 1020 | ||||
| 1021 | /// ISelUpdater - helper class to handle updates of the instruction selection | |||
| 1022 | /// graph. | |||
| 1023 | class ISelUpdater : public SelectionDAG::DAGUpdateListener { | |||
| 1024 | SelectionDAG::allnodes_iterator &ISelPosition; | |||
| 1025 | ||||
| 1026 | public: | |||
| 1027 | ISelUpdater(SelectionDAG &DAG, SelectionDAG::allnodes_iterator &isp) | |||
| 1028 | : SelectionDAG::DAGUpdateListener(DAG), ISelPosition(isp) {} | |||
| 1029 | ||||
| 1030 | /// NodeDeleted - Handle nodes deleted from the graph. If the node being | |||
| 1031 | /// deleted is the current ISelPosition node, update ISelPosition. | |||
| 1032 | /// | |||
| 1033 | void NodeDeleted(SDNode *N, SDNode *E) override { | |||
| 1034 | if (ISelPosition == SelectionDAG::allnodes_iterator(N)) | |||
| 1035 | ++ISelPosition; | |||
| 1036 | } | |||
| 1037 | }; | |||
| 1038 | ||||
| 1039 | } // end anonymous namespace | |||
| 1040 | ||||
| 1041 | // This function is used to enforce the topological node id property | |||
| 1042 | // property leveraged during Instruction selection. Before selection all | |||
| 1043 | // nodes are given a non-negative id such that all nodes have a larger id than | |||
| 1044 | // their operands. As this holds transitively we can prune checks that a node N | |||
| 1045 | // is a predecessor of M another by not recursively checking through M's | |||
| 1046 | // operands if N's ID is larger than M's ID. This is significantly improves | |||
| 1047 | // performance of for various legality checks (e.g. IsLegalToFold / | |||
| 1048 | // UpdateChains). | |||
| 1049 | ||||
| 1050 | // However, when we fuse multiple nodes into a single node | |||
| 1051 | // during selection we may induce a predecessor relationship between inputs and | |||
| 1052 | // outputs of distinct nodes being merged violating the topological property. | |||
| 1053 | // Should a fused node have a successor which has yet to be selected, our | |||
| 1054 | // legality checks would be incorrect. To avoid this we mark all unselected | |||
| 1055 | // sucessor nodes, i.e. id != -1 as invalid for pruning by bit-negating (x => | |||
| 1056 | // (-(x+1))) the ids and modify our pruning check to ignore negative Ids of M. | |||
| 1057 | // We use bit-negation to more clearly enforce that node id -1 can only be | |||
| 1058 | // achieved by selected nodes). As the conversion is reversable the original Id, | |||
| 1059 | // topological pruning can still be leveraged when looking for unselected nodes. | |||
| 1060 | // This method is call internally in all ISel replacement calls. | |||
| 1061 | void SelectionDAGISel::EnforceNodeIdInvariant(SDNode *Node) { | |||
| 1062 | SmallVector<SDNode *, 4> Nodes; | |||
| 1063 | Nodes.push_back(Node); | |||
| 1064 | ||||
| 1065 | while (!Nodes.empty()) { | |||
| 1066 | SDNode *N = Nodes.pop_back_val(); | |||
| 1067 | for (auto *U : N->uses()) { | |||
| 1068 | auto UId = U->getNodeId(); | |||
| 1069 | if (UId > 0) { | |||
| 1070 | InvalidateNodeId(U); | |||
| 1071 | Nodes.push_back(U); | |||
| 1072 | } | |||
| 1073 | } | |||
| 1074 | } | |||
| 1075 | } | |||
| 1076 | ||||
| 1077 | // InvalidateNodeId - As discusses in EnforceNodeIdInvariant, mark a | |||
| 1078 | // NodeId with the equivalent node id which is invalid for topological | |||
| 1079 | // pruning. | |||
| 1080 | void SelectionDAGISel::InvalidateNodeId(SDNode *N) { | |||
| 1081 | int InvalidId = -(N->getNodeId() + 1); | |||
| 1082 | N->setNodeId(InvalidId); | |||
| 1083 | } | |||
| 1084 | ||||
| 1085 | // getUninvalidatedNodeId - get original uninvalidated node id. | |||
| 1086 | int SelectionDAGISel::getUninvalidatedNodeId(SDNode *N) { | |||
| 1087 | int Id = N->getNodeId(); | |||
| 1088 | if (Id < -1) | |||
| 1089 | return -(Id + 1); | |||
| 1090 | return Id; | |||
| 1091 | } | |||
| 1092 | ||||
| 1093 | void SelectionDAGISel::DoInstructionSelection() { | |||
| 1094 | LLVM_DEBUG(dbgs() << "===== Instruction selection begins: "do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("isel")) { dbgs() << "===== Instruction selection begins: " << printMBBReference(*FuncInfo->MBB) << " '" << FuncInfo->MBB->getName() << "'\n"; } } while (false ) | |||
| 1095 | << printMBBReference(*FuncInfo->MBB) << " '"do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("isel")) { dbgs() << "===== Instruction selection begins: " << printMBBReference(*FuncInfo->MBB) << " '" << FuncInfo->MBB->getName() << "'\n"; } } while (false ) | |||
| 1096 | << FuncInfo->MBB->getName() << "'\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("isel")) { dbgs() << "===== Instruction selection begins: " << printMBBReference(*FuncInfo->MBB) << " '" << FuncInfo->MBB->getName() << "'\n"; } } while (false ); | |||
| 1097 | ||||
| 1098 | PreprocessISelDAG(); | |||
| 1099 | ||||
| 1100 | // Select target instructions for the DAG. | |||
| 1101 | { | |||
| 1102 | // Number all nodes with a topological order and set DAGSize. | |||
| 1103 | DAGSize = CurDAG->AssignTopologicalOrder(); | |||
| 1104 | ||||
| 1105 | // Create a dummy node (which is not added to allnodes), that adds | |||
| 1106 | // a reference to the root node, preventing it from being deleted, | |||
| 1107 | // and tracking any changes of the root. | |||
| 1108 | HandleSDNode Dummy(CurDAG->getRoot()); | |||
| 1109 | SelectionDAG::allnodes_iterator ISelPosition (CurDAG->getRoot().getNode()); | |||
| 1110 | ++ISelPosition; | |||
| 1111 | ||||
| 1112 | // Make sure that ISelPosition gets properly updated when nodes are deleted | |||
| 1113 | // in calls made from this function. | |||
| 1114 | ISelUpdater ISU(*CurDAG, ISelPosition); | |||
| 1115 | ||||
| 1116 | // The AllNodes list is now topological-sorted. Visit the | |||
| 1117 | // nodes by starting at the end of the list (the root of the | |||
| 1118 | // graph) and preceding back toward the beginning (the entry | |||
| 1119 | // node). | |||
| 1120 | while (ISelPosition != CurDAG->allnodes_begin()) { | |||
| 1121 | SDNode *Node = &*--ISelPosition; | |||
| 1122 | // Skip dead nodes. DAGCombiner is expected to eliminate all dead nodes, | |||
| 1123 | // but there are currently some corner cases that it misses. Also, this | |||
| 1124 | // makes it theoretically possible to disable the DAGCombiner. | |||
| 1125 | if (Node->use_empty()) | |||
| 1126 | continue; | |||
| 1127 | ||||
| 1128 | #ifndef NDEBUG | |||
| 1129 | SmallVector<SDNode *, 4> Nodes; | |||
| 1130 | Nodes.push_back(Node); | |||
| 1131 | ||||
| 1132 | while (!Nodes.empty()) { | |||
| 1133 | auto N = Nodes.pop_back_val(); | |||
| 1134 | if (N->getOpcode() == ISD::TokenFactor || N->getNodeId() < 0) | |||
| 1135 | continue; | |||
| 1136 | for (const SDValue &Op : N->op_values()) { | |||
| 1137 | if (Op->getOpcode() == ISD::TokenFactor) | |||
| 1138 | Nodes.push_back(Op.getNode()); | |||
| 1139 | else { | |||
| 1140 | // We rely on topological ordering of node ids for checking for | |||
| 1141 | // cycles when fusing nodes during selection. All unselected nodes | |||
| 1142 | // successors of an already selected node should have a negative id. | |||
| 1143 | // This assertion will catch such cases. If this assertion triggers | |||
| 1144 | // it is likely you using DAG-level Value/Node replacement functions | |||
| 1145 | // (versus equivalent ISEL replacement) in backend-specific | |||
| 1146 | // selections. See comment in EnforceNodeIdInvariant for more | |||
| 1147 | // details. | |||
| 1148 | assert(Op->getNodeId() != -1 &&((Op->getNodeId() != -1 && "Node has already selected predecessor node" ) ? static_cast<void> (0) : __assert_fail ("Op->getNodeId() != -1 && \"Node has already selected predecessor node\"" , "/build/llvm-toolchain-snapshot-10~++20200112100611+7fa5290d5bd/llvm/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp" , 1149, __PRETTY_FUNCTION__)) | |||
| 1149 | "Node has already selected predecessor node")((Op->getNodeId() != -1 && "Node has already selected predecessor node" ) ? static_cast<void> (0) : __assert_fail ("Op->getNodeId() != -1 && \"Node has already selected predecessor node\"" , "/build/llvm-toolchain-snapshot-10~++20200112100611+7fa5290d5bd/llvm/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp" , 1149, __PRETTY_FUNCTION__)); | |||
| 1150 | } | |||
| 1151 | } | |||
| 1152 | } | |||
| 1153 | #endif | |||
| 1154 | ||||
| 1155 | // When we are using non-default rounding modes or FP exception behavior | |||
| 1156 | // FP operations are represented by StrictFP pseudo-operations. For | |||
| 1157 | // targets that do not (yet) understand strict FP operations directly, | |||
| 1158 | // we convert them to normal FP opcodes instead at this point. This | |||
| 1159 | // will allow them to be handled by existing target-specific instruction | |||
| 1160 | // selectors. | |||
| 1161 | if (!TLI->isStrictFPEnabled() && Node->isStrictFPOpcode()) { | |||
| 1162 | // For some opcodes, we need to call TLI->getOperationAction using | |||
| 1163 | // the first operand type instead of the result type. Note that this | |||
| 1164 | // must match what SelectionDAGLegalize::LegalizeOp is doing. | |||
| 1165 | EVT ActionVT; | |||
| 1166 | switch (Node->getOpcode()) { | |||
| 1167 | case ISD::STRICT_SINT_TO_FP: | |||
| 1168 | case ISD::STRICT_UINT_TO_FP: | |||
| 1169 | case ISD::STRICT_LRINT: | |||
| 1170 | case ISD::STRICT_LLRINT: | |||
| 1171 | case ISD::STRICT_LROUND: | |||
| 1172 | case ISD::STRICT_LLROUND: | |||
| 1173 | case ISD::STRICT_FSETCC: | |||
| 1174 | case ISD::STRICT_FSETCCS: | |||
| 1175 | ActionVT = Node->getOperand(1).getValueType(); | |||
| 1176 | break; | |||
| 1177 | default: | |||
| 1178 | ActionVT = Node->getValueType(0); | |||
| 1179 | break; | |||
| 1180 | } | |||
| 1181 | if (TLI->getOperationAction(Node->getOpcode(), ActionVT) | |||
| 1182 | == TargetLowering::Expand) | |||
| 1183 | Node = CurDAG->mutateStrictFPToFP(Node); | |||
| 1184 | } | |||
| 1185 | ||||
| 1186 | LLVM_DEBUG(dbgs() << "\nISEL: Starting selection on root node: ";do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("isel")) { dbgs() << "\nISEL: Starting selection on root node: " ; Node->dump(CurDAG); } } while (false) | |||
| 1187 | Node->dump(CurDAG))do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("isel")) { dbgs() << "\nISEL: Starting selection on root node: " ; Node->dump(CurDAG); } } while (false); | |||
| 1188 | ||||
| 1189 | Select(Node); | |||
| 1190 | } | |||
| 1191 | ||||
| 1192 | CurDAG->setRoot(Dummy.getValue()); | |||
| 1193 | } | |||
| 1194 | ||||
| 1195 | LLVM_DEBUG(dbgs() << "\n===== Instruction selection ends:\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("isel")) { dbgs() << "\n===== Instruction selection ends:\n" ; } } while (false); | |||
| 1196 | ||||
| 1197 | PostprocessISelDAG(); | |||
| 1198 | } | |||
| 1199 | ||||
| 1200 | static bool hasExceptionPointerOrCodeUser(const CatchPadInst *CPI) { | |||
| 1201 | for (const User *U : CPI->users()) { | |||
| 1202 | if (const IntrinsicInst *EHPtrCall = dyn_cast<IntrinsicInst>(U)) { | |||
| 1203 | Intrinsic::ID IID = EHPtrCall->getIntrinsicID(); | |||
| 1204 | if (IID == Intrinsic::eh_exceptionpointer || | |||
| 1205 | IID == Intrinsic::eh_exceptioncode) | |||
| 1206 | return true; | |||
| 1207 | } | |||
| 1208 | } | |||
| 1209 | return false; | |||
| 1210 | } | |||
| 1211 | ||||
| 1212 | // wasm.landingpad.index intrinsic is for associating a landing pad index number | |||
| 1213 | // with a catchpad instruction. Retrieve the landing pad index in the intrinsic | |||
| 1214 | // and store the mapping in the function. | |||
| 1215 | static void mapWasmLandingPadIndex(MachineBasicBlock *MBB, | |||
| 1216 | const CatchPadInst *CPI) { | |||
| 1217 | MachineFunction *MF = MBB->getParent(); | |||
| 1218 | // In case of single catch (...), we don't emit LSDA, so we don't need | |||
| 1219 | // this information. | |||
| 1220 | bool IsSingleCatchAllClause = | |||
| 1221 | CPI->getNumArgOperands() == 1 && | |||
| 1222 | cast<Constant>(CPI->getArgOperand(0))->isNullValue(); | |||
| 1223 | if (!IsSingleCatchAllClause) { | |||
| 1224 | // Create a mapping from landing pad label to landing pad index. | |||
| 1225 | bool IntrFound = false; | |||
| 1226 | for (const User *U : CPI->users()) { | |||
| 1227 | if (const auto *Call = dyn_cast<IntrinsicInst>(U)) { | |||
| 1228 | Intrinsic::ID IID = Call->getIntrinsicID(); | |||
| 1229 | if (IID == Intrinsic::wasm_landingpad_index) { | |||
| 1230 | Value *IndexArg = Call->getArgOperand(1); | |||
| 1231 | int Index = cast<ConstantInt>(IndexArg)->getZExtValue(); | |||
| 1232 | MF->setWasmLandingPadIndex(MBB, Index); | |||
| 1233 | IntrFound = true; | |||
| 1234 | break; | |||
| 1235 | } | |||
| 1236 | } | |||
| 1237 | } | |||
| 1238 | assert(IntrFound && "wasm.landingpad.index intrinsic not found!")((IntrFound && "wasm.landingpad.index intrinsic not found!" ) ? static_cast<void> (0) : __assert_fail ("IntrFound && \"wasm.landingpad.index intrinsic not found!\"" , "/build/llvm-toolchain-snapshot-10~++20200112100611+7fa5290d5bd/llvm/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp" , 1238, __PRETTY_FUNCTION__)); | |||
| 1239 | (void)IntrFound; | |||
| 1240 | } | |||
| 1241 | } | |||
| 1242 | ||||
| 1243 | /// PrepareEHLandingPad - Emit an EH_LABEL, set up live-in registers, and | |||
| 1244 | /// do other setup for EH landing-pad blocks. | |||
| 1245 | bool SelectionDAGISel::PrepareEHLandingPad() { | |||
| 1246 | MachineBasicBlock *MBB = FuncInfo->MBB; | |||
| 1247 | const Constant *PersonalityFn = FuncInfo->Fn->getPersonalityFn(); | |||
| 1248 | const BasicBlock *LLVMBB = MBB->getBasicBlock(); | |||
| 1249 | const TargetRegisterClass *PtrRC = | |||
| 1250 | TLI->getRegClassFor(TLI->getPointerTy(CurDAG->getDataLayout())); | |||
| 1251 | ||||
| 1252 | auto Pers = classifyEHPersonality(PersonalityFn); | |||
| 1253 | ||||
| 1254 | // Catchpads have one live-in register, which typically holds the exception | |||
| 1255 | // pointer or code. | |||
| 1256 | if (isFuncletEHPersonality(Pers)) { | |||
| 1257 | if (const auto *CPI = dyn_cast<CatchPadInst>(LLVMBB->getFirstNonPHI())) { | |||
| 1258 | if (hasExceptionPointerOrCodeUser(CPI)) { | |||
| 1259 | // Get or create the virtual register to hold the pointer or code. Mark | |||
| 1260 | // the live in physreg and copy into the vreg. | |||
| 1261 | MCPhysReg EHPhysReg = TLI->getExceptionPointerRegister(PersonalityFn); | |||
| 1262 | assert(EHPhysReg && "target lacks exception pointer register")((EHPhysReg && "target lacks exception pointer register" ) ? static_cast<void> (0) : __assert_fail ("EHPhysReg && \"target lacks exception pointer register\"" , "/build/llvm-toolchain-snapshot-10~++20200112100611+7fa5290d5bd/llvm/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp" , 1262, __PRETTY_FUNCTION__)); | |||
| 1263 | MBB->addLiveIn(EHPhysReg); | |||
| 1264 | unsigned VReg = FuncInfo->getCatchPadExceptionPointerVReg(CPI, PtrRC); | |||
| 1265 | BuildMI(*MBB, FuncInfo->InsertPt, SDB->getCurDebugLoc(), | |||
| 1266 | TII->get(TargetOpcode::COPY), VReg) | |||
| 1267 | .addReg(EHPhysReg, RegState::Kill); | |||
| 1268 | } | |||
| 1269 | } | |||
| 1270 | return true; | |||
| 1271 | } | |||
| 1272 | ||||
| 1273 | // Add a label to mark the beginning of the landing pad. Deletion of the | |||
| 1274 | // landing pad can thus be detected via the MachineModuleInfo. | |||
| 1275 | MCSymbol *Label = MF->addLandingPad(MBB); | |||
| 1276 | ||||
| 1277 | const MCInstrDesc &II = TII->get(TargetOpcode::EH_LABEL); | |||
| 1278 | BuildMI(*MBB, FuncInfo->InsertPt, SDB->getCurDebugLoc(), II) | |||
| 1279 | .addSym(Label); | |||
| 1280 | ||||
| 1281 | if (Pers == EHPersonality::Wasm_CXX) { | |||
| 1282 | if (const auto *CPI = dyn_cast<CatchPadInst>(LLVMBB->getFirstNonPHI())) | |||
| 1283 | mapWasmLandingPadIndex(MBB, CPI); | |||
| 1284 | } else { | |||
| 1285 | // Assign the call site to the landing pad's begin label. | |||
| 1286 | MF->setCallSiteLandingPad(Label, SDB->LPadToCallSiteMap[MBB]); | |||
| 1287 | // Mark exception register as live in. | |||
| 1288 | if (unsigned Reg = TLI->getExceptionPointerRegister(PersonalityFn)) | |||
| 1289 | FuncInfo->ExceptionPointerVirtReg = MBB->addLiveIn(Reg, PtrRC); | |||
| 1290 | // Mark exception selector register as live in. | |||
| 1291 | if (unsigned Reg = TLI->getExceptionSelectorRegister(PersonalityFn)) | |||
| 1292 | FuncInfo->ExceptionSelectorVirtReg = MBB->addLiveIn(Reg, PtrRC); | |||
| 1293 | } | |||
| 1294 | ||||
| 1295 | return true; | |||
| 1296 | } | |||
| 1297 | ||||
| 1298 | /// isFoldedOrDeadInstruction - Return true if the specified instruction is | |||
| 1299 | /// side-effect free and is either dead or folded into a generated instruction. | |||
| 1300 | /// Return false if it needs to be emitted. | |||
| 1301 | static bool isFoldedOrDeadInstruction(const Instruction *I, | |||
| 1302 | const FunctionLoweringInfo &FuncInfo) { | |||
| 1303 | return !I->mayWriteToMemory() && // Side-effecting instructions aren't folded. | |||
| 1304 | !I->isTerminator() && // Terminators aren't folded. | |||
| 1305 | !isa<DbgInfoIntrinsic>(I) && // Debug instructions aren't folded. | |||
| 1306 | !I->isEHPad() && // EH pad instructions aren't folded. | |||
| 1307 | !FuncInfo.isExportedInst(I); // Exported instrs must be computed. | |||
| 1308 | } | |||
| 1309 | ||||
| 1310 | /// Collect llvm.dbg.declare information. This is done after argument lowering | |||
| 1311 | /// in case the declarations refer to arguments. | |||
| 1312 | static void processDbgDeclares(FunctionLoweringInfo &FuncInfo) { | |||
| 1313 | MachineFunction *MF = FuncInfo.MF; | |||
| 1314 | const DataLayout &DL = MF->getDataLayout(); | |||
| 1315 | for (const BasicBlock &BB : *FuncInfo.Fn) { | |||
| 1316 | for (const Instruction &I : BB) { | |||
| 1317 | const DbgDeclareInst *DI = dyn_cast<DbgDeclareInst>(&I); | |||
| 1318 | if (!DI) | |||
| 1319 | continue; | |||
| 1320 | ||||
| 1321 | assert(DI->getVariable() && "Missing variable")((DI->getVariable() && "Missing variable") ? static_cast <void> (0) : __assert_fail ("DI->getVariable() && \"Missing variable\"" , "/build/llvm-toolchain-snapshot-10~++20200112100611+7fa5290d5bd/llvm/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp" , 1321, __PRETTY_FUNCTION__)); | |||
| 1322 | assert(DI->getDebugLoc() && "Missing location")((DI->getDebugLoc() && "Missing location") ? static_cast <void> (0) : __assert_fail ("DI->getDebugLoc() && \"Missing location\"" , "/build/llvm-toolchain-snapshot-10~++20200112100611+7fa5290d5bd/llvm/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp" , 1322, __PRETTY_FUNCTION__)); | |||
| 1323 | const Value *Address = DI->getAddress(); | |||
| 1324 | if (!Address) | |||
| 1325 | continue; | |||
| 1326 | ||||
| 1327 | // Look through casts and constant offset GEPs. These mostly come from | |||
| 1328 | // inalloca. | |||
| 1329 | APInt Offset(DL.getTypeSizeInBits(Address->getType()), 0); | |||
| 1330 | Address = Address->stripAndAccumulateInBoundsConstantOffsets(DL, Offset); | |||
| 1331 | ||||
| 1332 | // Check if the variable is a static alloca or a byval or inalloca | |||
| 1333 | // argument passed in memory. If it is not, then we will ignore this | |||
| 1334 | // intrinsic and handle this during isel like dbg.value. | |||
| 1335 | int FI = std::numeric_limits<int>::max(); | |||
| 1336 | if (const auto *AI = dyn_cast<AllocaInst>(Address)) { | |||
| 1337 | auto SI = FuncInfo.StaticAllocaMap.find(AI); | |||
| 1338 | if (SI != FuncInfo.StaticAllocaMap.end()) | |||
| 1339 | FI = SI->second; | |||
| 1340 | } else if (const auto *Arg = dyn_cast<Argument>(Address)) | |||
| 1341 | FI = FuncInfo.getArgumentFrameIndex(Arg); | |||
| 1342 | ||||
| 1343 | if (FI == std::numeric_limits<int>::max()) | |||
| 1344 | continue; | |||
| 1345 | ||||
| 1346 | DIExpression *Expr = DI->getExpression(); | |||
| 1347 | if (Offset.getBoolValue()) | |||
| 1348 | Expr = DIExpression::prepend(Expr, DIExpression::ApplyOffset, | |||
| 1349 | Offset.getZExtValue()); | |||
| 1350 | MF->setVariableDbgInfo(DI->getVariable(), Expr, FI, DI->getDebugLoc()); | |||
| 1351 | } | |||
| 1352 | } | |||
| 1353 | } | |||
| 1354 | ||||
| 1355 | void SelectionDAGISel::SelectAllBasicBlocks(const Function &Fn) { | |||
| 1356 | FastISelFailed = false; | |||
| 1357 | // Initialize the Fast-ISel state, if needed. | |||
| 1358 | FastISel *FastIS = nullptr; | |||
| 1359 | if (TM.Options.EnableFastISel) { | |||
| 1360 | LLVM_DEBUG(dbgs() << "Enabling fast-isel\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("isel")) { dbgs() << "Enabling fast-isel\n"; } } while (false); | |||
| 1361 | FastIS = TLI->createFastISel(*FuncInfo, LibInfo); | |||
| 1362 | } | |||
| 1363 | ||||
| 1364 | ReversePostOrderTraversal<const Function*> RPOT(&Fn); | |||
| 1365 | ||||
| 1366 | // Lower arguments up front. An RPO iteration always visits the entry block | |||
| 1367 | // first. | |||
| 1368 | assert(*RPOT.begin() == &Fn.getEntryBlock())((*RPOT.begin() == &Fn.getEntryBlock()) ? static_cast< void> (0) : __assert_fail ("*RPOT.begin() == &Fn.getEntryBlock()" , "/build/llvm-toolchain-snapshot-10~++20200112100611+7fa5290d5bd/llvm/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp" , 1368, __PRETTY_FUNCTION__)); | |||
| 1369 | ++NumEntryBlocks; | |||
| 1370 | ||||
| 1371 | // Set up FuncInfo for ISel. Entry blocks never have PHIs. | |||
| 1372 | FuncInfo->MBB = FuncInfo->MBBMap[&Fn.getEntryBlock()]; | |||
| 1373 | FuncInfo->InsertPt = FuncInfo->MBB->begin(); | |||
| 1374 | ||||
| 1375 | CurDAG->setFunctionLoweringInfo(FuncInfo.get()); | |||
| 1376 | ||||
| 1377 | if (!FastIS) { | |||
| 1378 | LowerArguments(Fn); | |||
| 1379 | } else { | |||
| 1380 | // See if fast isel can lower the arguments. | |||
| 1381 | FastIS->startNewBlock(); | |||
| 1382 | if (!FastIS->lowerArguments()) { | |||
| 1383 | FastISelFailed = true; | |||
| 1384 | // Fast isel failed to lower these arguments | |||
| 1385 | ++NumFastIselFailLowerArguments; | |||
| 1386 | ||||
| 1387 | OptimizationRemarkMissed R("sdagisel", "FastISelFailure", | |||
| 1388 | Fn.getSubprogram(), | |||
| 1389 | &Fn.getEntryBlock()); | |||
| 1390 | R << "FastISel didn't lower all arguments: " | |||
| 1391 | << ore::NV("Prototype", Fn.getType()); | |||
| 1392 | reportFastISelFailure(*MF, *ORE, R, EnableFastISelAbort > 1); | |||
| 1393 | ||||
| 1394 | // Use SelectionDAG argument lowering | |||
| 1395 | LowerArguments(Fn); | |||
| 1396 | CurDAG->setRoot(SDB->getControlRoot()); | |||
| 1397 | SDB->clear(); | |||
| 1398 | CodeGenAndEmitDAG(); | |||
| 1399 | } | |||
| 1400 | ||||
| 1401 | // If we inserted any instructions at the beginning, make a note of | |||
| 1402 | // where they are, so we can be sure to emit subsequent instructions | |||
| 1403 | // after them. | |||
| 1404 | if (FuncInfo->InsertPt != FuncInfo->MBB->begin()) | |||
| 1405 | FastIS->setLastLocalValue(&*std::prev(FuncInfo->InsertPt)); | |||
| 1406 | else | |||
| 1407 | FastIS->setLastLocalValue(nullptr); | |||
| 1408 | } | |||
| 1409 | ||||
| 1410 | bool Inserted = SwiftError->createEntriesInEntryBlock(SDB->getCurDebugLoc()); | |||
| 1411 | ||||
| 1412 | if (FastIS && Inserted) | |||
| 1413 | FastIS->setLastLocalValue(&*std::prev(FuncInfo->InsertPt)); | |||
| 1414 | ||||
| 1415 | processDbgDeclares(*FuncInfo); | |||
| 1416 | ||||
| 1417 | // Iterate over all basic blocks in the function. | |||
| 1418 | StackProtector &SP = getAnalysis<StackProtector>(); | |||
| 1419 | for (const BasicBlock *LLVMBB : RPOT) { | |||
| 1420 | if (OptLevel != CodeGenOpt::None) { | |||
| 1421 | bool AllPredsVisited = true; | |||
| 1422 | for (const_pred_iterator PI = pred_begin(LLVMBB), PE = pred_end(LLVMBB); | |||
| 1423 | PI != PE; ++PI) { | |||
| 1424 | if (!FuncInfo->VisitedBBs.count(*PI)) { | |||
| 1425 | AllPredsVisited = false; | |||
| 1426 | break; | |||
| 1427 | } | |||
| 1428 | } | |||
| 1429 | ||||
| 1430 | if (AllPredsVisited) { | |||
| 1431 | for (const PHINode &PN : LLVMBB->phis()) | |||
| 1432 | FuncInfo->ComputePHILiveOutRegInfo(&PN); | |||
| 1433 | } else { | |||
| 1434 | for (const PHINode &PN : LLVMBB->phis()) | |||
| 1435 | FuncInfo->InvalidatePHILiveOutRegInfo(&PN); | |||
| 1436 | } | |||
| 1437 | ||||
| 1438 | FuncInfo->VisitedBBs.insert(LLVMBB); | |||
| 1439 | } | |||
| 1440 | ||||
| 1441 | BasicBlock::const_iterator const Begin = | |||
| 1442 | LLVMBB->getFirstNonPHI()->getIterator(); | |||
| 1443 | BasicBlock::const_iterator const End = LLVMBB->end(); | |||
| 1444 | BasicBlock::const_iterator BI = End; | |||
| 1445 | ||||
| 1446 | FuncInfo->MBB = FuncInfo->MBBMap[LLVMBB]; | |||
| 1447 | if (!FuncInfo->MBB) | |||
| 1448 | continue; // Some blocks like catchpads have no code or MBB. | |||
| 1449 | ||||
| 1450 | // Insert new instructions after any phi or argument setup code. | |||
| 1451 | FuncInfo->InsertPt = FuncInfo->MBB->end(); | |||
| 1452 | ||||
| 1453 | // Setup an EH landing-pad block. | |||
| 1454 | FuncInfo->ExceptionPointerVirtReg = 0; | |||
| 1455 | FuncInfo->ExceptionSelectorVirtReg = 0; | |||
| 1456 | if (LLVMBB->isEHPad()) | |||
| 1457 | if (!PrepareEHLandingPad()) | |||
| 1458 | continue; | |||
| 1459 | ||||
| 1460 | // Before doing SelectionDAG ISel, see if FastISel has been requested. | |||
| 1461 | if (FastIS) { | |||
| 1462 | if (LLVMBB != &Fn.getEntryBlock()) | |||
| 1463 | FastIS->startNewBlock(); | |||
| 1464 | ||||
| 1465 | unsigned NumFastIselRemaining = std::distance(Begin, End); | |||
| 1466 | ||||
| 1467 | // Pre-assign swifterror vregs. | |||
| 1468 | SwiftError->preassignVRegs(FuncInfo->MBB, Begin, End); | |||
| 1469 | ||||
| 1470 | // Do FastISel on as many instructions as possible. | |||
| 1471 | for (; BI != Begin; --BI) { | |||
| 1472 | const Instruction *Inst = &*std::prev(BI); | |||
| 1473 | ||||
| 1474 | // If we no longer require this instruction, skip it. | |||
| 1475 | if (isFoldedOrDeadInstruction(Inst, *FuncInfo) || | |||
| 1476 | ElidedArgCopyInstrs.count(Inst)) { | |||
| 1477 | --NumFastIselRemaining; | |||
| 1478 | continue; | |||
| 1479 | } | |||
| 1480 | ||||
| 1481 | // Bottom-up: reset the insert pos at the top, after any local-value | |||
| 1482 | // instructions. | |||
| 1483 | FastIS->recomputeInsertPt(); | |||
| 1484 | ||||
| 1485 | // Try to select the instruction with FastISel. | |||
| 1486 | if (FastIS->selectInstruction(Inst)) { | |||
| 1487 | --NumFastIselRemaining; | |||
| 1488 | ++NumFastIselSuccess; | |||
| 1489 | // If fast isel succeeded, skip over all the folded instructions, and | |||
| 1490 | // then see if there is a load right before the selected instructions. | |||
| 1491 | // Try to fold the load if so. | |||
| 1492 | const Instruction *BeforeInst = Inst; | |||
| 1493 | while (BeforeInst != &*Begin) { | |||
| 1494 | BeforeInst = &*std::prev(BasicBlock::const_iterator(BeforeInst)); | |||
| 1495 | if (!isFoldedOrDeadInstruction(BeforeInst, *FuncInfo)) | |||
| 1496 | break; | |||
| 1497 | } | |||
| 1498 | if (BeforeInst != Inst && isa<LoadInst>(BeforeInst) && | |||
| 1499 | BeforeInst->hasOneUse() && | |||
| 1500 | FastIS->tryToFoldLoad(cast<LoadInst>(BeforeInst), Inst)) { | |||
| 1501 | // If we succeeded, don't re-select the load. | |||
| 1502 | BI = std::next(BasicBlock::const_iterator(BeforeInst)); | |||
| 1503 | --NumFastIselRemaining; | |||
| 1504 | ++NumFastIselSuccess; | |||
| 1505 | } | |||
| 1506 | continue; | |||
| 1507 | } | |||
| 1508 | ||||
| 1509 | FastISelFailed = true; | |||
| 1510 | ||||
| 1511 | // Then handle certain instructions as single-LLVM-Instruction blocks. | |||
| 1512 | // We cannot separate out GCrelocates to their own blocks since we need | |||
| 1513 | // to keep track of gc-relocates for a particular gc-statepoint. This is | |||
| 1514 | // done by SelectionDAGBuilder::LowerAsSTATEPOINT, called before | |||
| 1515 | // visitGCRelocate. | |||
| 1516 | if (isa<CallInst>(Inst) && !isStatepoint(Inst) && !isGCRelocate(Inst) && | |||
| 1517 | !isGCResult(Inst)) { | |||
| 1518 | OptimizationRemarkMissed R("sdagisel", "FastISelFailure", | |||
| 1519 | Inst->getDebugLoc(), LLVMBB); | |||
| 1520 | ||||
| 1521 | R << "FastISel missed call"; | |||
| 1522 | ||||
| 1523 | if (R.isEnabled() || EnableFastISelAbort) { | |||
| 1524 | std::string InstStrStorage; | |||
| 1525 | raw_string_ostream InstStr(InstStrStorage); | |||
| 1526 | InstStr << *Inst; | |||
| 1527 | ||||
| 1528 | R << ": " << InstStr.str(); | |||
| 1529 | } | |||
| 1530 | ||||
| 1531 | reportFastISelFailure(*MF, *ORE, R, EnableFastISelAbort > 2); | |||
| 1532 | ||||
| 1533 | if (!Inst->getType()->isVoidTy() && !Inst->getType()->isTokenTy() && | |||
| 1534 | !Inst->use_empty()) { | |||
| 1535 | unsigned &R = FuncInfo->ValueMap[Inst]; | |||
| 1536 | if (!R) | |||
| 1537 | R = FuncInfo->CreateRegs(Inst); | |||
| 1538 | } | |||
| 1539 | ||||
| 1540 | bool HadTailCall = false; | |||
| 1541 | MachineBasicBlock::iterator SavedInsertPt = FuncInfo->InsertPt; | |||
| 1542 | SelectBasicBlock(Inst->getIterator(), BI, HadTailCall); | |||
| 1543 | ||||
| 1544 | // If the call was emitted as a tail call, we're done with the block. | |||
| 1545 | // We also need to delete any previously emitted instructions. | |||
| 1546 | if (HadTailCall) { | |||
| 1547 | FastIS->removeDeadCode(SavedInsertPt, FuncInfo->MBB->end()); | |||
| 1548 | --BI; | |||
| 1549 | break; | |||
| 1550 | } | |||
| 1551 | ||||
| 1552 | // Recompute NumFastIselRemaining as Selection DAG instruction | |||
| 1553 | // selection may have handled the call, input args, etc. | |||
| 1554 | unsigned RemainingNow = std::distance(Begin, BI); | |||
| 1555 | NumFastIselFailures += NumFastIselRemaining - RemainingNow; | |||
| 1556 | NumFastIselRemaining = RemainingNow; | |||
| 1557 | continue; | |||
| 1558 | } | |||
| 1559 | ||||
| 1560 | OptimizationRemarkMissed R("sdagisel", "FastISelFailure", | |||
| 1561 | Inst->getDebugLoc(), LLVMBB); | |||
| 1562 | ||||
| 1563 | bool ShouldAbort = EnableFastISelAbort; | |||
| 1564 | if (Inst->isTerminator()) { | |||
| 1565 | // Use a different message for terminator misses. | |||
| 1566 | R << "FastISel missed terminator"; | |||
| 1567 | // Don't abort for terminator unless the level is really high | |||
| 1568 | ShouldAbort = (EnableFastISelAbort > 2); | |||
| 1569 | } else { | |||
| 1570 | R << "FastISel missed"; | |||
| 1571 | } | |||
| 1572 | ||||
| 1573 | if (R.isEnabled() || EnableFastISelAbort) { | |||
| 1574 | std::string InstStrStorage; | |||
| 1575 | raw_string_ostream InstStr(InstStrStorage); | |||
| 1576 | InstStr << *Inst; | |||
| 1577 | R << ": " << InstStr.str(); | |||
| 1578 | } | |||
| 1579 | ||||
| 1580 | reportFastISelFailure(*MF, *ORE, R, ShouldAbort); | |||
| 1581 | ||||
| 1582 | NumFastIselFailures += NumFastIselRemaining; | |||
| 1583 | break; | |||
| 1584 | } | |||
| 1585 | ||||
| 1586 | FastIS->recomputeInsertPt(); | |||
| 1587 | } | |||
| 1588 | ||||
| 1589 | if (SP.shouldEmitSDCheck(*LLVMBB)) { | |||
| 1590 | bool FunctionBasedInstrumentation = | |||
| 1591 | TLI->getSSPStackGuardCheck(*Fn.getParent()); | |||
| 1592 | SDB->SPDescriptor.initialize(LLVMBB, FuncInfo->MBBMap[LLVMBB], | |||
| 1593 | FunctionBasedInstrumentation); | |||
| 1594 | } | |||
| 1595 | ||||
| 1596 | if (Begin != BI) | |||
| 1597 | ++NumDAGBlocks; | |||
| 1598 | else | |||
| 1599 | ++NumFastIselBlocks; | |||
| 1600 | ||||
| 1601 | if (Begin != BI) { | |||
| 1602 | // Run SelectionDAG instruction selection on the remainder of the block | |||
| 1603 | // not handled by FastISel. If FastISel is not run, this is the entire | |||
| 1604 | // block. | |||
| 1605 | bool HadTailCall; | |||
| 1606 | SelectBasicBlock(Begin, BI, HadTailCall); | |||
| 1607 | ||||
| 1608 | // But if FastISel was run, we already selected some of the block. | |||
| 1609 | // If we emitted a tail-call, we need to delete any previously emitted | |||
| 1610 | // instruction that follows it. | |||
| 1611 | if (FastIS && HadTailCall && FuncInfo->InsertPt != FuncInfo->MBB->end()) | |||
| 1612 | FastIS->removeDeadCode(FuncInfo->InsertPt, FuncInfo->MBB->end()); | |||
| 1613 | } | |||
| 1614 | ||||
| 1615 | if (FastIS) | |||
| 1616 | FastIS->finishBasicBlock(); | |||
| 1617 | FinishBasicBlock(); | |||
| 1618 | FuncInfo->PHINodesToUpdate.clear(); | |||
| 1619 | ElidedArgCopyInstrs.clear(); | |||
| 1620 | } | |||
| 1621 | ||||
| 1622 | SP.copyToMachineFrameInfo(MF->getFrameInfo()); | |||
| 1623 | ||||
| 1624 | SwiftError->propagateVRegs(); | |||
| 1625 | ||||
| 1626 | delete FastIS; | |||
| 1627 | SDB->clearDanglingDebugInfo(); | |||
| 1628 | SDB->SPDescriptor.resetPerFunctionState(); | |||
| 1629 | } | |||
| 1630 | ||||
| 1631 | /// Given that the input MI is before a partial terminator sequence TSeq, return | |||
| 1632 | /// true if M + TSeq also a partial terminator sequence. | |||
| 1633 | /// | |||
| 1634 | /// A Terminator sequence is a sequence of MachineInstrs which at this point in | |||
| 1635 | /// lowering copy vregs into physical registers, which are then passed into | |||
| 1636 | /// terminator instructors so we can satisfy ABI constraints. A partial | |||
| 1637 | /// terminator sequence is an improper subset of a terminator sequence (i.e. it | |||
| 1638 | /// may be the whole terminator sequence). | |||
| 1639 | static bool MIIsInTerminatorSequence(const MachineInstr &MI) { | |||
| 1640 | // If we do not have a copy or an implicit def, we return true if and only if | |||
| 1641 | // MI is a debug value. | |||
| 1642 | if (!MI.isCopy() && !MI.isImplicitDef()) | |||
| 1643 | // Sometimes DBG_VALUE MI sneak in between the copies from the vregs to the | |||
| 1644 | // physical registers if there is debug info associated with the terminator | |||
| 1645 | // of our mbb. We want to include said debug info in our terminator | |||
| 1646 | // sequence, so we return true in that case. | |||
| 1647 | return MI.isDebugValue(); | |||
| 1648 | ||||
| 1649 | // We have left the terminator sequence if we are not doing one of the | |||
| 1650 | // following: | |||
| 1651 | // | |||
| 1652 | // 1. Copying a vreg into a physical register. | |||
| 1653 | // 2. Copying a vreg into a vreg. | |||
| 1654 | // 3. Defining a register via an implicit def. | |||
| 1655 | ||||
| 1656 | // OPI should always be a register definition... | |||
| 1657 | MachineInstr::const_mop_iterator OPI = MI.operands_begin(); | |||
| 1658 | if (!OPI->isReg() || !OPI->isDef()) | |||
| 1659 | return false; | |||
| 1660 | ||||
| 1661 | // Defining any register via an implicit def is always ok. | |||
| 1662 | if (MI.isImplicitDef()) | |||
| 1663 | return true; | |||
| 1664 | ||||
| 1665 | // Grab the copy source... | |||
| 1666 | MachineInstr::const_mop_iterator OPI2 = OPI; | |||
| 1667 | ++OPI2; | |||
| 1668 | assert(OPI2 != MI.operands_end()((OPI2 != MI.operands_end() && "Should have a copy implying we should have 2 arguments." ) ? static_cast<void> (0) : __assert_fail ("OPI2 != MI.operands_end() && \"Should have a copy implying we should have 2 arguments.\"" , "/build/llvm-toolchain-snapshot-10~++20200112100611+7fa5290d5bd/llvm/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp" , 1669, __PRETTY_FUNCTION__)) | |||
| 1669 | && "Should have a copy implying we should have 2 arguments.")((OPI2 != MI.operands_end() && "Should have a copy implying we should have 2 arguments." ) ? static_cast<void> (0) : __assert_fail ("OPI2 != MI.operands_end() && \"Should have a copy implying we should have 2 arguments.\"" , "/build/llvm-toolchain-snapshot-10~++20200112100611+7fa5290d5bd/llvm/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp" , 1669, __PRETTY_FUNCTION__)); | |||
| 1670 | ||||
| 1671 | // Make sure that the copy dest is not a vreg when the copy source is a | |||
| 1672 | // physical register. | |||
| 1673 | if (!OPI2->isReg() || (!Register::isPhysicalRegister(OPI->getReg()) && | |||
| 1674 | Register::isPhysicalRegister(OPI2->getReg()))) | |||
| 1675 | return false; | |||
| 1676 | ||||
| 1677 | return true; | |||
| 1678 | } | |||
| 1679 | ||||
| 1680 | /// Find the split point at which to splice the end of BB into its success stack | |||
| 1681 | /// protector check machine basic block. | |||
| 1682 | /// | |||
| 1683 | /// On many platforms, due to ABI constraints, terminators, even before register | |||
| 1684 | /// allocation, use physical registers. This creates an issue for us since | |||
| 1685 | /// physical registers at this point can not travel across basic | |||
| 1686 | /// blocks. Luckily, selectiondag always moves physical registers into vregs | |||
| 1687 | /// when they enter functions and moves them through a sequence of copies back | |||
| 1688 | /// into the physical registers right before the terminator creating a | |||
| 1689 | /// ``Terminator Sequence''. This function is searching for the beginning of the | |||
| 1690 | /// terminator sequence so that we can ensure that we splice off not just the | |||
| 1691 | /// terminator, but additionally the copies that move the vregs into the | |||
| 1692 | /// physical registers. | |||
| 1693 | static MachineBasicBlock::iterator | |||
| 1694 | FindSplitPointForStackProtector(MachineBasicBlock *BB) { | |||
| 1695 | MachineBasicBlock::iterator SplitPoint = BB->getFirstTerminator(); | |||
| 1696 | // | |||
| 1697 | if (SplitPoint == BB->begin()) | |||
| 1698 | return SplitPoint; | |||
| 1699 | ||||
| 1700 | MachineBasicBlock::iterator Start = BB->begin(); | |||
| 1701 | MachineBasicBlock::iterator Previous = SplitPoint; | |||
| 1702 | --Previous; | |||
| 1703 | ||||
| 1704 | while (MIIsInTerminatorSequence(*Previous)) { | |||
| 1705 | SplitPoint = Previous; | |||
| 1706 | if (Previous == Start) | |||
| 1707 | break; | |||
| 1708 | --Previous; | |||
| 1709 | } | |||
| 1710 | ||||
| 1711 | return SplitPoint; | |||
| 1712 | } | |||
| 1713 | ||||
| 1714 | void | |||
| 1715 | SelectionDAGISel::FinishBasicBlock() { | |||
| 1716 | LLVM_DEBUG(dbgs() << "Total amount of phi nodes to update: "do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("isel")) { dbgs() << "Total amount of phi nodes to update: " << FuncInfo->PHINodesToUpdate.size() << "\n"; for (unsigned i = 0, e = FuncInfo->PHINodesToUpdate.size( ); i != e; ++i) dbgs() << "Node " << i << " : (" << FuncInfo->PHINodesToUpdate[i].first << ", " << FuncInfo->PHINodesToUpdate[i].second << ")\n" ; } } while (false) | |||
| 1717 | << FuncInfo->PHINodesToUpdate.size() << "\n";do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("isel")) { dbgs() << "Total amount of phi nodes to update: " << FuncInfo->PHINodesToUpdate.size() << "\n"; for (unsigned i = 0, e = FuncInfo->PHINodesToUpdate.size( ); i != e; ++i) dbgs() << "Node " << i << " : (" << FuncInfo->PHINodesToUpdate[i].first << ", " << FuncInfo->PHINodesToUpdate[i].second << ")\n" ; } } while (false) | |||
| 1718 | for (unsigned i = 0, e = FuncInfo->PHINodesToUpdate.size(); i != e;do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("isel")) { dbgs() << "Total amount of phi nodes to update: " << FuncInfo->PHINodesToUpdate.size() << "\n"; for (unsigned i = 0, e = FuncInfo->PHINodesToUpdate.size( ); i != e; ++i) dbgs() << "Node " << i << " : (" << FuncInfo->PHINodesToUpdate[i].first << ", " << FuncInfo->PHINodesToUpdate[i].second << ")\n" ; } } while (false) | |||
| 1719 | ++i) dbgs()do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("isel")) { dbgs() << "Total amount of phi nodes to update: " << FuncInfo->PHINodesToUpdate.size() << "\n"; for (unsigned i = 0, e = FuncInfo->PHINodesToUpdate.size( ); i != e; ++i) dbgs() << "Node " << i << " : (" << FuncInfo->PHINodesToUpdate[i].first << ", " << FuncInfo->PHINodesToUpdate[i].second << ")\n" ; } } while (false) | |||
| 1720 | << "Node " << i << " : (" << FuncInfo->PHINodesToUpdate[i].firstdo { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("isel")) { dbgs() << "Total amount of phi nodes to update: " << FuncInfo->PHINodesToUpdate.size() << "\n"; for (unsigned i = 0, e = FuncInfo->PHINodesToUpdate.size( ); i != e; ++i) dbgs() << "Node " << i << " : (" << FuncInfo->PHINodesToUpdate[i].first << ", " << FuncInfo->PHINodesToUpdate[i].second << ")\n" ; } } while (false) | |||
| 1721 | << ", " << FuncInfo->PHINodesToUpdate[i].second << ")\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("isel")) { dbgs() << "Total amount of phi nodes to update: " << FuncInfo->PHINodesToUpdate.size() << "\n"; for (unsigned i = 0, e = FuncInfo->PHINodesToUpdate.size( ); i != e; ++i) dbgs() << "Node " << i << " : (" << FuncInfo->PHINodesToUpdate[i].first << ", " << FuncInfo->PHINodesToUpdate[i].second << ")\n" ; } } while (false); | |||
| 1722 | ||||
| 1723 | // Next, now that we know what the last MBB the LLVM BB expanded is, update | |||
| 1724 | // PHI nodes in successors. | |||
| 1725 | for (unsigned i = 0, e = FuncInfo->PHINodesToUpdate.size(); i != e; ++i) { | |||
| 1726 | MachineInstrBuilder PHI(*MF, FuncInfo->PHINodesToUpdate[i].first); | |||
| 1727 | assert(PHI->isPHI() &&((PHI->isPHI() && "This is not a machine PHI node that we are updating!" ) ? static_cast<void> (0) : __assert_fail ("PHI->isPHI() && \"This is not a machine PHI node that we are updating!\"" , "/build/llvm-toolchain-snapshot-10~++20200112100611+7fa5290d5bd/llvm/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp" , 1728, __PRETTY_FUNCTION__)) | |||
| 1728 | "This is not a machine PHI node that we are updating!")((PHI->isPHI() && "This is not a machine PHI node that we are updating!" ) ? static_cast<void> (0) : __assert_fail ("PHI->isPHI() && \"This is not a machine PHI node that we are updating!\"" , "/build/llvm-toolchain-snapshot-10~++20200112100611+7fa5290d5bd/llvm/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp" , 1728, __PRETTY_FUNCTION__)); | |||
| 1729 | if (!FuncInfo->MBB->isSuccessor(PHI->getParent())) | |||
| 1730 | continue; | |||
| 1731 | PHI.addReg(FuncInfo->PHINodesToUpdate[i].second).addMBB(FuncInfo->MBB); | |||
| 1732 | } | |||
| 1733 | ||||
| 1734 | // Handle stack protector. | |||
| 1735 | if (SDB->SPDescriptor.shouldEmitFunctionBasedCheckStackProtector()) { | |||
| 1736 | // The target provides a guard check function. There is no need to | |||
| 1737 | // generate error handling code or to split current basic block. | |||
| 1738 | MachineBasicBlock *ParentMBB = SDB->SPDescriptor.getParentMBB(); | |||
| 1739 | ||||
| 1740 | // Add load and check to the basicblock. | |||
| 1741 | FuncInfo->MBB = ParentMBB; | |||
| 1742 | FuncInfo->InsertPt = | |||
| 1743 | FindSplitPointForStackProtector(ParentMBB); | |||
| 1744 | SDB->visitSPDescriptorParent(SDB->SPDescriptor, ParentMBB); | |||
| 1745 | CurDAG->setRoot(SDB->getRoot()); | |||
| 1746 | SDB->clear(); | |||
| 1747 | CodeGenAndEmitDAG(); | |||
| 1748 | ||||
| 1749 | // Clear the Per-BB State. | |||
| 1750 | SDB->SPDescriptor.resetPerBBState(); | |||
| 1751 | } else if (SDB->SPDescriptor.shouldEmitStackProtector()) { | |||
| 1752 | MachineBasicBlock *ParentMBB = SDB->SPDescriptor.getParentMBB(); | |||
| 1753 | MachineBasicBlock *SuccessMBB = SDB->SPDescriptor.getSuccessMBB(); | |||
| 1754 | ||||
| 1755 | // Find the split point to split the parent mbb. At the same time copy all | |||
| 1756 | // physical registers used in the tail of parent mbb into virtual registers | |||
| 1757 | // before the split point and back into physical registers after the split | |||
| 1758 | // point. This prevents us needing to deal with Live-ins and many other | |||
| 1759 | // register allocation issues caused by us splitting the parent mbb. The | |||
| 1760 | // register allocator will clean up said virtual copies later on. | |||
| 1761 | MachineBasicBlock::iterator SplitPoint = | |||
| 1762 | FindSplitPointForStackProtector(ParentMBB); | |||
| 1763 | ||||
| 1764 | // Splice the terminator of ParentMBB into SuccessMBB. | |||
| 1765 | SuccessMBB->splice(SuccessMBB->end(), ParentMBB, | |||
| 1766 | SplitPoint, | |||
| 1767 | ParentMBB->end()); | |||
| 1768 | ||||
| 1769 | // Add compare/jump on neq/jump to the parent BB. | |||
| 1770 | FuncInfo->MBB = ParentMBB; | |||
| 1771 | FuncInfo->InsertPt = ParentMBB->end(); | |||
| 1772 | SDB->visitSPDescriptorParent(SDB->SPDescriptor, ParentMBB); | |||
| 1773 | CurDAG->setRoot(SDB->getRoot()); | |||
| 1774 | SDB->clear(); | |||
| 1775 | CodeGenAndEmitDAG(); | |||
| 1776 | ||||
| 1777 | // CodeGen Failure MBB if we have not codegened it yet. | |||
| 1778 | MachineBasicBlock *FailureMBB = SDB->SPDescriptor.getFailureMBB(); | |||
| 1779 | if (FailureMBB->empty()) { | |||
| 1780 | FuncInfo->MBB = FailureMBB; | |||
| 1781 | FuncInfo->InsertPt = FailureMBB->end(); | |||
| 1782 | SDB->visitSPDescriptorFailure(SDB->SPDescriptor); | |||
| 1783 | CurDAG->setRoot(SDB->getRoot()); | |||
| 1784 | SDB->clear(); | |||
| 1785 | CodeGenAndEmitDAG(); | |||
| 1786 | } | |||
| 1787 | ||||
| 1788 | // Clear the Per-BB State. | |||
| 1789 | SDB->SPDescriptor.resetPerBBState(); | |||
| 1790 | } | |||
| 1791 | ||||
| 1792 | // Lower each BitTestBlock. | |||
| 1793 | for (auto &BTB : SDB->SL->BitTestCases) { | |||
| 1794 | // Lower header first, if it wasn't already lowered | |||
| 1795 | if (!BTB.Emitted) { | |||
| 1796 | // Set the current basic block to the mbb we wish to insert the code into | |||
| 1797 | FuncInfo->MBB = BTB.Parent; | |||
| 1798 | FuncInfo->InsertPt = FuncInfo->MBB->end(); | |||
| 1799 | // Emit the code | |||
| 1800 | SDB->visitBitTestHeader(BTB, FuncInfo->MBB); | |||
| 1801 | CurDAG->setRoot(SDB->getRoot()); | |||
| 1802 | SDB->clear(); | |||
| 1803 | CodeGenAndEmitDAG(); | |||
| 1804 | } | |||
| 1805 | ||||
| 1806 | BranchProbability UnhandledProb = BTB.Prob; | |||
| 1807 | for (unsigned j = 0, ej = BTB.Cases.size(); j != ej; ++j) { | |||
| 1808 | UnhandledProb -= BTB.Cases[j].ExtraProb; | |||
| 1809 | // Set the current basic block to the mbb we wish to insert the code into | |||
| 1810 | FuncInfo->MBB = BTB.Cases[j].ThisBB; | |||
| 1811 | FuncInfo->InsertPt = FuncInfo->MBB->end(); | |||
| 1812 | // Emit the code | |||
| 1813 | ||||
| 1814 | // If all cases cover a contiguous range, it is not necessary to jump to | |||
| 1815 | // the default block after the last bit test fails. This is because the | |||
| 1816 | // range check during bit test header creation has guaranteed that every | |||
| 1817 | // case here doesn't go outside the range. In this case, there is no need | |||
| 1818 | // to perform the last bit test, as it will always be true. Instead, make | |||
| 1819 | // the second-to-last bit-test fall through to the target of the last bit | |||
| 1820 | // test, and delete the last bit test. | |||
| 1821 | ||||
| 1822 | MachineBasicBlock *NextMBB; | |||
| 1823 | if (BTB.ContiguousRange && j + 2 == ej) { | |||
| 1824 | // Second-to-last bit-test with contiguous range: fall through to the | |||
| 1825 | // target of the final bit test. | |||
| 1826 | NextMBB = BTB.Cases[j + 1].TargetBB; | |||
| 1827 | } else if (j + 1 == ej) { | |||
| 1828 | // For the last bit test, fall through to Default. | |||
| 1829 | NextMBB = BTB.Default; | |||
| 1830 | } else { | |||
| 1831 | // Otherwise, fall through to the next bit test. | |||
| 1832 | NextMBB = BTB.Cases[j + 1].ThisBB; | |||
| 1833 | } | |||
| 1834 | ||||
| 1835 | SDB->visitBitTestCase(BTB, NextMBB, UnhandledProb, BTB.Reg, BTB.Cases[j], | |||
| 1836 | FuncInfo->MBB); | |||
| 1837 | ||||
| 1838 | CurDAG->setRoot(SDB->getRoot()); | |||
| 1839 | SDB->clear(); | |||
| 1840 | CodeGenAndEmitDAG(); | |||
| 1841 | ||||
| 1842 | if (BTB.ContiguousRange && j + 2 == ej) { | |||
| 1843 | // Since we're not going to use the final bit test, remove it. | |||
| 1844 | BTB.Cases.pop_back(); | |||
| 1845 | break; | |||
| 1846 | } | |||
| 1847 | } | |||
| 1848 | ||||
| 1849 | // Update PHI Nodes | |||
| 1850 | for (unsigned pi = 0, pe = FuncInfo->PHINodesToUpdate.size(); | |||
| 1851 | pi != pe; ++pi) { | |||
| 1852 | MachineInstrBuilder PHI(*MF, FuncInfo->PHINodesToUpdate[pi].first); | |||
| 1853 | MachineBasicBlock *PHIBB = PHI->getParent(); | |||
| 1854 | assert(PHI->isPHI() &&((PHI->isPHI() && "This is not a machine PHI node that we are updating!" ) ? static_cast<void> (0) : __assert_fail ("PHI->isPHI() && \"This is not a machine PHI node that we are updating!\"" , "/build/llvm-toolchain-snapshot-10~++20200112100611+7fa5290d5bd/llvm/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp" , 1855, __PRETTY_FUNCTION__)) | |||
| 1855 | "This is not a machine PHI node that we are updating!")((PHI->isPHI() && "This is not a machine PHI node that we are updating!" ) ? static_cast<void> (0) : __assert_fail ("PHI->isPHI() && \"This is not a machine PHI node that we are updating!\"" , "/build/llvm-toolchain-snapshot-10~++20200112100611+7fa5290d5bd/llvm/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp" , 1855, __PRETTY_FUNCTION__)); | |||
| 1856 | // This is "default" BB. We have two jumps to it. From "header" BB and | |||
| 1857 | // from last "case" BB, unless the latter was skipped. | |||
| 1858 | if (PHIBB == BTB.Default) { | |||
| 1859 | PHI.addReg(FuncInfo->PHINodesToUpdate[pi].second).addMBB(BTB.Parent); | |||
| 1860 | if (!BTB.ContiguousRange) { | |||
| 1861 | PHI.addReg(FuncInfo->PHINodesToUpdate[pi].second) | |||
| 1862 | .addMBB(BTB.Cases.back().ThisBB); | |||
| 1863 | } | |||
| 1864 | } | |||
| 1865 | // One of "cases" BB. | |||
| 1866 | for (unsigned j = 0, ej = BTB.Cases.size(); | |||
| 1867 | j != ej; ++j) { | |||
| 1868 | MachineBasicBlock* cBB = BTB.Cases[j].ThisBB; | |||
| 1869 | if (cBB->isSuccessor(PHIBB)) | |||
| 1870 | PHI.addReg(FuncInfo->PHINodesToUpdate[pi].second).addMBB(cBB); | |||
| 1871 | } | |||
| 1872 | } | |||
| 1873 | } | |||
| 1874 | SDB->SL->BitTestCases.clear(); | |||
| 1875 | ||||
| 1876 | // If the JumpTable record is filled in, then we need to emit a jump table. | |||
| 1877 | // Updating the PHI nodes is tricky in this case, since we need to determine | |||
| 1878 | // whether the PHI is a successor of the range check MBB or the jump table MBB | |||
| 1879 | for (unsigned i = 0, e = SDB->SL->JTCases.size(); i != e; ++i) { | |||
| 1880 | // Lower header first, if it wasn't already lowered | |||
| 1881 | if (!SDB->SL->JTCases[i].first.Emitted) { | |||
| 1882 | // Set the current basic block to the mbb we wish to insert the code into | |||
| 1883 | FuncInfo->MBB = SDB->SL->JTCases[i].first.HeaderBB; | |||
| 1884 | FuncInfo->InsertPt = FuncInfo->MBB->end(); | |||
| 1885 | // Emit the code | |||
| 1886 | SDB->visitJumpTableHeader(SDB->SL->JTCases[i].second, | |||
| 1887 | SDB->SL->JTCases[i].first, FuncInfo->MBB); | |||
| 1888 | CurDAG->setRoot(SDB->getRoot()); | |||
| 1889 | SDB->clear(); | |||
| 1890 | CodeGenAndEmitDAG(); | |||
| 1891 | } | |||
| 1892 | ||||
| 1893 | // Set the current basic block to the mbb we wish to insert the code into | |||
| 1894 | FuncInfo->MBB = SDB->SL->JTCases[i].second.MBB; | |||
| 1895 | FuncInfo->InsertPt = FuncInfo->MBB->end(); | |||
| 1896 | // Emit the code | |||
| 1897 | SDB->visitJumpTable(SDB->SL->JTCases[i].second); | |||
| 1898 | CurDAG->setRoot(SDB->getRoot()); | |||
| 1899 | SDB->clear(); | |||
| 1900 | CodeGenAndEmitDAG(); | |||
| 1901 | ||||
| 1902 | // Update PHI Nodes | |||
| 1903 | for (unsigned pi = 0, pe = FuncInfo->PHINodesToUpdate.size(); | |||
| 1904 | pi != pe; ++pi) { | |||
| 1905 | MachineInstrBuilder PHI(*MF, FuncInfo->PHINodesToUpdate[pi].first); | |||
| 1906 | MachineBasicBlock *PHIBB = PHI->getParent(); | |||
| 1907 | assert(PHI->isPHI() &&((PHI->isPHI() && "This is not a machine PHI node that we are updating!" ) ? static_cast<void> (0) : __assert_fail ("PHI->isPHI() && \"This is not a machine PHI node that we are updating!\"" , "/build/llvm-toolchain-snapshot-10~++20200112100611+7fa5290d5bd/llvm/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp" , 1908, __PRETTY_FUNCTION__)) | |||
| 1908 | "This is not a machine PHI node that we are updating!")((PHI->isPHI() && "This is not a machine PHI node that we are updating!" ) ? static_cast<void> (0) : __assert_fail ("PHI->isPHI() && \"This is not a machine PHI node that we are updating!\"" , "/build/llvm-toolchain-snapshot-10~++20200112100611+7fa5290d5bd/llvm/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp" , 1908, __PRETTY_FUNCTION__)); | |||
| 1909 | // "default" BB. We can go there only from header BB. | |||
| 1910 | if (PHIBB == SDB->SL->JTCases[i].second.Default) | |||
| 1911 | PHI.addReg(FuncInfo->PHINodesToUpdate[pi].second) | |||
| 1912 | .addMBB(SDB->SL->JTCases[i].first.HeaderBB); | |||
| 1913 | // JT BB. Just iterate over successors here | |||
| 1914 | if (FuncInfo->MBB->isSuccessor(PHIBB)) | |||
| 1915 | PHI.addReg(FuncInfo->PHINodesToUpdate[pi].second).addMBB(FuncInfo->MBB); | |||
| 1916 | } | |||
| 1917 | } | |||
| 1918 | SDB->SL->JTCases.clear(); | |||
| 1919 | ||||
| 1920 | // If we generated any switch lowering information, build and codegen any | |||
| 1921 | // additional DAGs necessary. | |||
| 1922 | for (unsigned i = 0, e = SDB->SL->SwitchCases.size(); i != e; ++i) { | |||
| 1923 | // Set the current basic block to the mbb we wish to insert the code into | |||
| 1924 | FuncInfo->MBB = SDB->SL->SwitchCases[i].ThisBB; | |||
| 1925 | FuncInfo->InsertPt = FuncInfo->MBB->end(); | |||
| 1926 | ||||
| 1927 | // Determine the unique successors. | |||
| 1928 | SmallVector<MachineBasicBlock *, 2> Succs; | |||
| 1929 | Succs.push_back(SDB->SL->SwitchCases[i].TrueBB); | |||
| 1930 | if (SDB->SL->SwitchCases[i].TrueBB != SDB->SL->SwitchCases[i].FalseBB) | |||
| 1931 | Succs.push_back(SDB->SL->SwitchCases[i].FalseBB); | |||
| 1932 | ||||
| 1933 | // Emit the code. Note that this could result in FuncInfo->MBB being split. | |||
| 1934 | SDB->visitSwitchCase(SDB->SL->SwitchCases[i], FuncInfo->MBB); | |||
| 1935 | CurDAG->setRoot(SDB->getRoot()); | |||
| 1936 | SDB->clear(); | |||
| 1937 | CodeGenAndEmitDAG(); | |||
| 1938 | ||||
| 1939 | // Remember the last block, now that any splitting is done, for use in | |||
| 1940 | // populating PHI nodes in successors. | |||
| 1941 | MachineBasicBlock *ThisBB = FuncInfo->MBB; | |||
| 1942 | ||||
| 1943 | // Handle any PHI nodes in successors of this chunk, as if we were coming | |||
| 1944 | // from the original BB before switch expansion. Note that PHI nodes can | |||
| 1945 | // occur multiple times in PHINodesToUpdate. We have to be very careful to | |||
| 1946 | // handle them the right number of times. | |||
| 1947 | for (unsigned i = 0, e = Succs.size(); i != e; ++i) { | |||
| 1948 | FuncInfo->MBB = Succs[i]; | |||
| 1949 | FuncInfo->InsertPt = FuncInfo->MBB->end(); | |||
| 1950 | // FuncInfo->MBB may have been removed from the CFG if a branch was | |||
| 1951 | // constant folded. | |||
| 1952 | if (ThisBB->isSuccessor(FuncInfo->MBB)) { | |||
| 1953 | for (MachineBasicBlock::iterator | |||
| 1954 | MBBI = FuncInfo->MBB->begin(), MBBE = FuncInfo->MBB->end(); | |||
| 1955 | MBBI != MBBE && MBBI->isPHI(); ++MBBI) { | |||
| 1956 | MachineInstrBuilder PHI(*MF, MBBI); | |||
| 1957 | // This value for this PHI node is recorded in PHINodesToUpdate. | |||
| 1958 | for (unsigned pn = 0; ; ++pn) { | |||
| 1959 | assert(pn != FuncInfo->PHINodesToUpdate.size() &&((pn != FuncInfo->PHINodesToUpdate.size() && "Didn't find PHI entry!" ) ? static_cast<void> (0) : __assert_fail ("pn != FuncInfo->PHINodesToUpdate.size() && \"Didn't find PHI entry!\"" , "/build/llvm-toolchain-snapshot-10~++20200112100611+7fa5290d5bd/llvm/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp" , 1960, __PRETTY_FUNCTION__)) | |||
| 1960 | "Didn't find PHI entry!")((pn != FuncInfo->PHINodesToUpdate.size() && "Didn't find PHI entry!" ) ? static_cast<void> (0) : __assert_fail ("pn != FuncInfo->PHINodesToUpdate.size() && \"Didn't find PHI entry!\"" , "/build/llvm-toolchain-snapshot-10~++20200112100611+7fa5290d5bd/llvm/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp" , 1960, __PRETTY_FUNCTION__)); | |||
| 1961 | if (FuncInfo->PHINodesToUpdate[pn].first == PHI) { | |||
| 1962 | PHI.addReg(FuncInfo->PHINodesToUpdate[pn].second).addMBB(ThisBB); | |||
| 1963 | break; | |||
| 1964 | } | |||
| 1965 | } | |||
| 1966 | } | |||
| 1967 | } | |||
| 1968 | } | |||
| 1969 | } | |||
| 1970 | SDB->SL->SwitchCases.clear(); | |||
| 1971 | } | |||
| 1972 | ||||
| 1973 | /// Create the scheduler. If a specific scheduler was specified | |||
| 1974 | /// via the SchedulerRegistry, use it, otherwise select the | |||
| 1975 | /// one preferred by the target. | |||
| 1976 | /// | |||
| 1977 | ScheduleDAGSDNodes *SelectionDAGISel::CreateScheduler() { | |||
| 1978 | return ISHeuristic(this, OptLevel); | |||
| 1979 | } | |||
| 1980 | ||||
| 1981 | //===----------------------------------------------------------------------===// | |||
| 1982 | // Helper functions used by the generated instruction selector. | |||
| 1983 | //===----------------------------------------------------------------------===// | |||
| 1984 | // Calls to these methods are generated by tblgen. | |||
| 1985 | ||||
| 1986 | /// CheckAndMask - The isel is trying to match something like (and X, 255). If | |||
| 1987 | /// the dag combiner simplified the 255, we still want to match. RHS is the | |||
| 1988 | /// actual value in the DAG on the RHS of an AND, and DesiredMaskS is the value | |||
| 1989 | /// specified in the .td file (e.g. 255). | |||
| 1990 | bool SelectionDAGISel::CheckAndMask(SDValue LHS, ConstantSDNode *RHS, | |||
| 1991 | int64_t DesiredMaskS) const { | |||
| 1992 | const APInt &ActualMask = RHS->getAPIntValue(); | |||
| 1993 | const APInt &DesiredMask = APInt(LHS.getValueSizeInBits(), DesiredMaskS); | |||
| 1994 | ||||
| 1995 | // If the actual mask exactly matches, success! | |||
| 1996 | if (ActualMask == DesiredMask) | |||
| 1997 | return true; | |||
| 1998 | ||||
| 1999 | // If the actual AND mask is allowing unallowed bits, this doesn't match. | |||
| 2000 | if (!ActualMask.isSubsetOf(DesiredMask)) | |||
| 2001 | return false; | |||
| 2002 | ||||
| 2003 | // Otherwise, the DAG Combiner may have proven that the value coming in is | |||
| 2004 | // either already zero or is not demanded. Check for known zero input bits. | |||
| 2005 | APInt NeededMask = DesiredMask & ~ActualMask; | |||
| 2006 | if (CurDAG->MaskedValueIsZero(LHS, NeededMask)) | |||
| 2007 | return true; | |||
| 2008 | ||||
| 2009 | // TODO: check to see if missing bits are just not demanded. | |||
| 2010 | ||||
| 2011 | // Otherwise, this pattern doesn't match. | |||
| 2012 | return false; | |||
| 2013 | } | |||
| 2014 | ||||
| 2015 | /// CheckOrMask - The isel is trying to match something like (or X, 255). If | |||
| 2016 | /// the dag combiner simplified the 255, we still want to match. RHS is the | |||
| 2017 | /// actual value in the DAG on the RHS of an OR, and DesiredMaskS is the value | |||
| 2018 | /// specified in the .td file (e.g. 255). | |||
| 2019 | bool SelectionDAGISel::CheckOrMask(SDValue LHS, ConstantSDNode *RHS, | |||
| 2020 | int64_t DesiredMaskS) const { | |||
| 2021 | const APInt &ActualMask = RHS->getAPIntValue(); | |||
| 2022 | const APInt &DesiredMask = APInt(LHS.getValueSizeInBits(), DesiredMaskS); | |||
| 2023 | ||||
| 2024 | // If the actual mask exactly matches, success! | |||
| 2025 | if (ActualMask == DesiredMask) | |||
| 2026 | return true; | |||
| 2027 | ||||
| 2028 | // If the actual AND mask is allowing unallowed bits, this doesn't match. | |||
| 2029 | if (!ActualMask.isSubsetOf(DesiredMask)) | |||
| 2030 | return false; | |||
| 2031 | ||||
| 2032 | // Otherwise, the DAG Combiner may have proven that the value coming in is | |||
| 2033 | // either already zero or is not demanded. Check for known zero input bits. | |||
| 2034 | APInt NeededMask = DesiredMask & ~ActualMask; | |||
| 2035 | KnownBits Known = CurDAG->computeKnownBits(LHS); | |||
| 2036 | ||||
| 2037 | // If all the missing bits in the or are already known to be set, match! | |||
| 2038 | if (NeededMask.isSubsetOf(Known.One)) | |||
| 2039 | return true; | |||
| 2040 | ||||
| 2041 | // TODO: check to see if missing bits are just not demanded. | |||
| 2042 | ||||
| 2043 | // Otherwise, this pattern doesn't match. | |||
| 2044 | return false; | |||
| 2045 | } | |||
| 2046 | ||||
| 2047 | /// SelectInlineAsmMemoryOperands - Calls to this are automatically generated | |||
| 2048 | /// by tblgen. Others should not call it. | |||
| 2049 | void SelectionDAGISel::SelectInlineAsmMemoryOperands(std::vector<SDValue> &Ops, | |||
| 2050 | const SDLoc &DL) { | |||
| 2051 | std::vector<SDValue> InOps; | |||
| 2052 | std::swap(InOps, Ops); | |||
| 2053 | ||||
| 2054 | Ops.push_back(InOps[InlineAsm::Op_InputChain]); // 0 | |||
| 2055 | Ops.push_back(InOps[InlineAsm::Op_AsmString]); // 1 | |||
| 2056 | Ops.push_back(InOps[InlineAsm::Op_MDNode]); // 2, !srcloc | |||
| 2057 | Ops.push_back(InOps[InlineAsm::Op_ExtraInfo]); // 3 (SideEffect, AlignStack) | |||
| 2058 | ||||
| 2059 | unsigned i = InlineAsm::Op_FirstOperand, e = InOps.size(); | |||
| 2060 | if (InOps[e-1].getValueType() == MVT::Glue) | |||
| 2061 | --e; // Don't process a glue operand if it is here. | |||
| 2062 | ||||
| 2063 | while (i != e) { | |||
| 2064 | unsigned Flags = cast<ConstantSDNode>(InOps[i])->getZExtValue(); | |||
| 2065 | if (!InlineAsm::isMemKind(Flags)) { | |||
| 2066 | // Just skip over this operand, copying the operands verbatim. | |||
| 2067 | Ops.insert(Ops.end(), InOps.begin()+i, | |||
| 2068 | InOps.begin()+i+InlineAsm::getNumOperandRegisters(Flags) + 1); | |||
| 2069 | i += InlineAsm::getNumOperandRegisters(Flags) + 1; | |||
| 2070 | } else { | |||
| 2071 | assert(InlineAsm::getNumOperandRegisters(Flags) == 1 &&((InlineAsm::getNumOperandRegisters(Flags) == 1 && "Memory operand with multiple values?" ) ? static_cast<void> (0) : __assert_fail ("InlineAsm::getNumOperandRegisters(Flags) == 1 && \"Memory operand with multiple values?\"" , "/build/llvm-toolchain-snapshot-10~++20200112100611+7fa5290d5bd/llvm/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp" , 2072, __PRETTY_FUNCTION__)) | |||
| 2072 | "Memory operand with multiple values?")((InlineAsm::getNumOperandRegisters(Flags) == 1 && "Memory operand with multiple values?" ) ? static_cast<void> (0) : __assert_fail ("InlineAsm::getNumOperandRegisters(Flags) == 1 && \"Memory operand with multiple values?\"" , "/build/llvm-toolchain-snapshot-10~++20200112100611+7fa5290d5bd/llvm/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp" , 2072, __PRETTY_FUNCTION__)); | |||
| 2073 | ||||
| 2074 | unsigned TiedToOperand; | |||
| 2075 | if (InlineAsm::isUseOperandTiedToDef(Flags, TiedToOperand)) { | |||
| 2076 | // We need the constraint ID from the operand this is tied to. | |||
| 2077 | unsigned CurOp = InlineAsm::Op_FirstOperand; | |||
| 2078 | Flags = cast<ConstantSDNode>(InOps[CurOp])->getZExtValue(); | |||
| 2079 | for (; TiedToOperand; --TiedToOperand) { | |||
| 2080 | CurOp += InlineAsm::getNumOperandRegisters(Flags)+1; | |||
| 2081 | Flags = cast<ConstantSDNode>(InOps[CurOp])->getZExtValue(); | |||
| 2082 | } | |||
| 2083 | } | |||
| 2084 | ||||
| 2085 | // Otherwise, this is a memory operand. Ask the target to select it. | |||
| 2086 | std::vector<SDValue> SelOps; | |||
| 2087 | unsigned ConstraintID = InlineAsm::getMemoryConstraintID(Flags); | |||
| 2088 | if (SelectInlineAsmMemoryOperand(InOps[i+1], ConstraintID, SelOps)) | |||
| 2089 | report_fatal_error("Could not match memory address. Inline asm" | |||
| 2090 | " failure!"); | |||
| 2091 | ||||
| 2092 | // Add this to the output node. | |||
| 2093 | unsigned NewFlags = | |||
| 2094 | InlineAsm::getFlagWord(InlineAsm::Kind_Mem, SelOps.size()); | |||
| 2095 | NewFlags = InlineAsm::getFlagWordForMem(NewFlags, ConstraintID); | |||
| 2096 | Ops.push_back(CurDAG->getTargetConstant(NewFlags, DL, MVT::i32)); | |||
| 2097 | Ops.insert(Ops.end(), SelOps.begin(), SelOps.end()); | |||
| 2098 | i += 2; | |||
| 2099 | } | |||
| 2100 | } | |||
| 2101 | ||||
| 2102 | // Add the glue input back if present. | |||
| 2103 | if (e != InOps.size()) | |||
| 2104 | Ops.push_back(InOps.back()); | |||
| 2105 | } | |||
| 2106 | ||||
| 2107 | /// findGlueUse - Return use of MVT::Glue value produced by the specified | |||
| 2108 | /// SDNode. | |||
| 2109 | /// | |||
| 2110 | static SDNode *findGlueUse(SDNode *N) { | |||
| 2111 | unsigned FlagResNo = N->getNumValues()-1; | |||
| 2112 | for (SDNode::use_iterator I = N->use_begin(), E = N->use_end(); I != E; ++I) { | |||
| 2113 | SDUse &Use = I.getUse(); | |||
| 2114 | if (Use.getResNo() == FlagResNo) | |||
| 2115 | return Use.getUser(); | |||
| 2116 | } | |||
| 2117 | return nullptr; | |||
| 2118 | } | |||
| 2119 | ||||
| 2120 | /// findNonImmUse - Return true if "Def" is a predecessor of "Root" via a path | |||
| 2121 | /// beyond "ImmedUse". We may ignore chains as they are checked separately. | |||
| 2122 | static bool findNonImmUse(SDNode *Root, SDNode *Def, SDNode *ImmedUse, | |||
| 2123 | bool IgnoreChains) { | |||
| 2124 | SmallPtrSet<const SDNode *, 16> Visited; | |||
| 2125 | SmallVector<const SDNode *, 16> WorkList; | |||
| 2126 | // Only check if we have non-immediate uses of Def. | |||
| 2127 | if (ImmedUse->isOnlyUserOf(Def)) | |||
| 2128 | return false; | |||
| 2129 | ||||
| 2130 | // We don't care about paths to Def that go through ImmedUse so mark it | |||
| 2131 | // visited and mark non-def operands as used. | |||
| 2132 | Visited.insert(ImmedUse); | |||
| 2133 | for (const SDValue &Op : ImmedUse->op_values()) { | |||
| 2134 | SDNode *N = Op.getNode(); | |||
| 2135 | // Ignore chain deps (they are validated by | |||
| 2136 | // HandleMergeInputChains) and immediate uses | |||
| 2137 | if ((Op.getValueType() == MVT::Other && IgnoreChains) || N == Def) | |||
| 2138 | continue; | |||
| 2139 | if (!Visited.insert(N).second) | |||
| 2140 | continue; | |||
| 2141 | WorkList.push_back(N); | |||
| 2142 | } | |||
| 2143 | ||||
| 2144 | // Initialize worklist to operands of Root. | |||
| 2145 | if (Root != ImmedUse) { | |||
| 2146 | for (const SDValue &Op : Root->op_values()) { | |||
| 2147 | SDNode *N = Op.getNode(); | |||
| 2148 | // Ignore chains (they are validated by HandleMergeInputChains) | |||
| 2149 | if ((Op.getValueType() == MVT::Other && IgnoreChains) || N == Def) | |||
| 2150 | continue; | |||
| 2151 | if (!Visited.insert(N).second) | |||
| 2152 | continue; | |||
| 2153 | WorkList.push_back(N); | |||
| 2154 | } | |||
| 2155 | } | |||
| 2156 | ||||
| 2157 | return SDNode::hasPredecessorHelper(Def, Visited, WorkList, 0, true); | |||
| 2158 | } | |||
| 2159 | ||||
| 2160 | /// IsProfitableToFold - Returns true if it's profitable to fold the specific | |||
| 2161 | /// operand node N of U during instruction selection that starts at Root. | |||
| 2162 | bool SelectionDAGISel::IsProfitableToFold(SDValue N, SDNode *U, | |||
| 2163 | SDNode *Root) const { | |||
| 2164 | if (OptLevel == CodeGenOpt::None) return false; | |||
| 2165 | return N.hasOneUse(); | |||
| 2166 | } | |||
| 2167 | ||||
| 2168 | /// IsLegalToFold - Returns true if the specific operand node N of | |||
| 2169 | /// U can be folded during instruction selection that starts at Root. | |||
| 2170 | bool SelectionDAGISel::IsLegalToFold(SDValue N, SDNode *U, SDNode *Root, | |||
| 2171 | CodeGenOpt::Level OptLevel, | |||
| 2172 | bool IgnoreChains) { | |||
| 2173 | if (OptLevel == CodeGenOpt::None) return false; | |||
| 2174 | ||||
| 2175 | // If Root use can somehow reach N through a path that that doesn't contain | |||
| 2176 | // U then folding N would create a cycle. e.g. In the following | |||
| 2177 | // diagram, Root can reach N through X. If N is folded into Root, then | |||
| 2178 | // X is both a predecessor and a successor of U. | |||
| 2179 | // | |||
| 2180 | // [N*] // | |||
| 2181 | // ^ ^ // | |||
| 2182 | // / \ // | |||
| 2183 | // [U*] [X]? // | |||
| 2184 | // ^ ^ // | |||
| 2185 | // \ / // | |||
| 2186 | // \ / // | |||
| 2187 | // [Root*] // | |||
| 2188 | // | |||
| 2189 | // * indicates nodes to be folded together. | |||
| 2190 | // | |||
| 2191 | // If Root produces glue, then it gets (even more) interesting. Since it | |||
| 2192 | // will be "glued" together with its glue use in the scheduler, we need to | |||
| 2193 | // check if it might reach N. | |||
| 2194 | // | |||
| 2195 | // [N*] // | |||
| 2196 | // ^ ^ // | |||
| 2197 | // / \ // | |||
| 2198 | // [U*] [X]? // | |||
| 2199 | // ^ ^ // | |||
| 2200 | // \ \ // | |||
| 2201 | // \ | // | |||
| 2202 | // [Root*] | // | |||
| 2203 | // ^ | // | |||
| 2204 | // f | // | |||
| 2205 | // | / // | |||
| 2206 | // [Y] / // | |||
| 2207 | // ^ / // | |||
| 2208 | // f / // | |||
| 2209 | // | / // | |||
| 2210 | // [GU] // | |||
| 2211 | // | |||
| 2212 | // If GU (glue use) indirectly reaches N (the load), and Root folds N | |||
| 2213 | // (call it Fold), then X is a predecessor of GU and a successor of | |||
| 2214 | // Fold. But since Fold and GU are glued together, this will create | |||
| 2215 | // a cycle in the scheduling graph. | |||
| 2216 | ||||
| 2217 | // If the node has glue, walk down the graph to the "lowest" node in the | |||
| 2218 | // glueged set. | |||
| 2219 | EVT VT = Root->getValueType(Root->getNumValues()-1); | |||
| 2220 | while (VT == MVT::Glue) { | |||
| 2221 | SDNode *GU = findGlueUse(Root); | |||
| 2222 | if (!GU) | |||
| 2223 | break; | |||
| 2224 | Root = GU; | |||
| 2225 | VT = Root->getValueType(Root->getNumValues()-1); | |||
| 2226 | ||||
| 2227 | // If our query node has a glue result with a use, we've walked up it. If | |||
| 2228 | // the user (which has already been selected) has a chain or indirectly uses | |||
| 2229 | // the chain, HandleMergeInputChains will not consider it. Because of | |||
| 2230 | // this, we cannot ignore chains in this predicate. | |||
| 2231 | IgnoreChains = false; | |||
| 2232 | } | |||
| 2233 | ||||
| 2234 | return !findNonImmUse(Root, N.getNode(), U, IgnoreChains); | |||
| 2235 | } | |||
| 2236 | ||||
| 2237 | void SelectionDAGISel::Select_INLINEASM(SDNode *N, bool Branch) { | |||
| 2238 | SDLoc DL(N); | |||
| 2239 | ||||
| 2240 | std::vector<SDValue> Ops(N->op_begin(), N->op_end()); | |||
| 2241 | SelectInlineAsmMemoryOperands(Ops, DL); | |||
| 2242 | ||||
| 2243 | const EVT VTs[] = {MVT::Other, MVT::Glue}; | |||
| 2244 | SDValue New = CurDAG->getNode(Branch ? ISD::INLINEASM_BR : ISD::INLINEASM, DL, VTs, Ops); | |||
| 2245 | New->setNodeId(-1); | |||
| 2246 | ReplaceUses(N, New.getNode()); | |||
| 2247 | CurDAG->RemoveDeadNode(N); | |||
| 2248 | } | |||
| 2249 | ||||
| 2250 | void SelectionDAGISel::Select_READ_REGISTER(SDNode *Op) { | |||
| 2251 | SDLoc dl(Op); | |||
| 2252 | MDNodeSDNode *MD = cast<MDNodeSDNode>(Op->getOperand(1)); | |||
| 2253 | const MDString *RegStr = cast<MDString>(MD->getMD()->getOperand(0)); | |||
| 2254 | ||||
| 2255 | EVT VT = Op->getValueType(0); | |||
| 2256 | LLT Ty = VT.isSimple() ? getLLTForMVT(VT.getSimpleVT()) : LLT(); | |||
| 2257 | Register Reg = | |||
| 2258 | TLI->getRegisterByName(RegStr->getString().data(), Ty, | |||
| 2259 | CurDAG->getMachineFunction()); | |||
| 2260 | SDValue New = CurDAG->getCopyFromReg( | |||
| 2261 | Op->getOperand(0), dl, Reg, Op->getValueType(0)); | |||
| 2262 | New->setNodeId(-1); | |||
| 2263 | ReplaceUses(Op, New.getNode()); | |||
| 2264 | CurDAG->RemoveDeadNode(Op); | |||
| 2265 | } | |||
| 2266 | ||||
| 2267 | void SelectionDAGISel::Select_WRITE_REGISTER(SDNode *Op) { | |||
| 2268 | SDLoc dl(Op); | |||
| 2269 | MDNodeSDNode *MD = cast<MDNodeSDNode>(Op->getOperand(1)); | |||
| 2270 | const MDString *RegStr = cast<MDString>(MD->getMD()->getOperand(0)); | |||
| 2271 | ||||
| 2272 | EVT VT = Op->getOperand(2).getValueType(); | |||
| 2273 | LLT Ty = VT.isSimple() ? getLLTForMVT(VT.getSimpleVT()) : LLT(); | |||
| 2274 | ||||
| 2275 | Register Reg = TLI->getRegisterByName(RegStr->getString().data(), Ty, | |||
| 2276 | CurDAG->getMachineFunction()); | |||
| 2277 | SDValue New = CurDAG->getCopyToReg( | |||
| 2278 | Op->getOperand(0), dl, Reg, Op->getOperand(2)); | |||
| 2279 | New->setNodeId(-1); | |||
| 2280 | ReplaceUses(Op, New.getNode()); | |||
| 2281 | CurDAG->RemoveDeadNode(Op); | |||
| 2282 | } | |||
| 2283 | ||||
| 2284 | void SelectionDAGISel::Select_UNDEF(SDNode *N) { | |||
| 2285 | CurDAG->SelectNodeTo(N, TargetOpcode::IMPLICIT_DEF, N->getValueType(0)); | |||
| 2286 | } | |||
| 2287 | ||||
| 2288 | /// GetVBR - decode a vbr encoding whose top bit is set. | |||
| 2289 | LLVM_ATTRIBUTE_ALWAYS_INLINE__attribute__((always_inline)) static inline uint64_t | |||
| 2290 | GetVBR(uint64_t Val, const unsigned char *MatcherTable, unsigned &Idx) { | |||
| 2291 | assert(Val >= 128 && "Not a VBR")((Val >= 128 && "Not a VBR") ? static_cast<void > (0) : __assert_fail ("Val >= 128 && \"Not a VBR\"" , "/build/llvm-toolchain-snapshot-10~++20200112100611+7fa5290d5bd/llvm/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp" , 2291, __PRETTY_FUNCTION__)); | |||
| 2292 | Val &= 127; // Remove first vbr bit. | |||
| 2293 | ||||
| 2294 | unsigned Shift = 7; | |||
| 2295 | uint64_t NextBits; | |||
| 2296 | do { | |||
| 2297 | NextBits = MatcherTable[Idx++]; | |||
| 2298 | Val |= (NextBits&127) << Shift; | |||
| 2299 | Shift += 7; | |||
| 2300 | } while (NextBits & 128); | |||
| 2301 | ||||
| 2302 | return Val; | |||
| 2303 | } | |||
| 2304 | ||||
| 2305 | /// When a match is complete, this method updates uses of interior chain results | |||
| 2306 | /// to use the new results. | |||
| 2307 | void SelectionDAGISel::UpdateChains( | |||
| 2308 | SDNode *NodeToMatch, SDValue InputChain, | |||
| 2309 | SmallVectorImpl<SDNode *> &ChainNodesMatched, bool isMorphNodeTo) { | |||
| 2310 | SmallVector<SDNode*, 4> NowDeadNodes; | |||
| 2311 | ||||
| 2312 | // Now that all the normal results are replaced, we replace the chain and | |||
| 2313 | // glue results if present. | |||
| 2314 | if (!ChainNodesMatched.empty()) { | |||
| 2315 | assert(InputChain.getNode() &&((InputChain.getNode() && "Matched input chains but didn't produce a chain" ) ? static_cast<void> (0) : __assert_fail ("InputChain.getNode() && \"Matched input chains but didn't produce a chain\"" , "/build/llvm-toolchain-snapshot-10~++20200112100611+7fa5290d5bd/llvm/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp" , 2316, __PRETTY_FUNCTION__)) | |||
| 2316 | "Matched input chains but didn't produce a chain")((InputChain.getNode() && "Matched input chains but didn't produce a chain" ) ? static_cast<void> (0) : __assert_fail ("InputChain.getNode() && \"Matched input chains but didn't produce a chain\"" , "/build/llvm-toolchain-snapshot-10~++20200112100611+7fa5290d5bd/llvm/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp" , 2316, __PRETTY_FUNCTION__)); | |||
| 2317 | // Loop over all of the nodes we matched that produced a chain result. | |||
| 2318 | // Replace all the chain results with the final chain we ended up with. | |||
| 2319 | for (unsigned i = 0, e = ChainNodesMatched.size(); i != e; ++i) { | |||
| 2320 | SDNode *ChainNode = ChainNodesMatched[i]; | |||
| 2321 | // If ChainNode is null, it's because we replaced it on a previous | |||
| 2322 | // iteration and we cleared it out of the map. Just skip it. | |||
| 2323 | if (!ChainNode) | |||
| 2324 | continue; | |||
| 2325 | ||||
| 2326 | assert(ChainNode->getOpcode() != ISD::DELETED_NODE &&((ChainNode->getOpcode() != ISD::DELETED_NODE && "Deleted node left in chain" ) ? static_cast<void> (0) : __assert_fail ("ChainNode->getOpcode() != ISD::DELETED_NODE && \"Deleted node left in chain\"" , "/build/llvm-toolchain-snapshot-10~++20200112100611+7fa5290d5bd/llvm/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp" , 2327, __PRETTY_FUNCTION__)) | |||
| 2327 | "Deleted node left in chain")((ChainNode->getOpcode() != ISD::DELETED_NODE && "Deleted node left in chain" ) ? static_cast<void> (0) : __assert_fail ("ChainNode->getOpcode() != ISD::DELETED_NODE && \"Deleted node left in chain\"" , "/build/llvm-toolchain-snapshot-10~++20200112100611+7fa5290d5bd/llvm/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp" , 2327, __PRETTY_FUNCTION__)); | |||
| 2328 | ||||
| 2329 | // Don't replace the results of the root node if we're doing a | |||
| 2330 | // MorphNodeTo. | |||
| 2331 | if (ChainNode == NodeToMatch && isMorphNodeTo) | |||
| 2332 | continue; | |||
| 2333 | ||||
| 2334 | SDValue ChainVal = SDValue(ChainNode, ChainNode->getNumValues()-1); | |||
| 2335 | if (ChainVal.getValueType() == MVT::Glue) | |||
| 2336 | ChainVal = ChainVal.getValue(ChainVal->getNumValues()-2); | |||
| 2337 | assert(ChainVal.getValueType() == MVT::Other && "Not a chain?")((ChainVal.getValueType() == MVT::Other && "Not a chain?" ) ? static_cast<void> (0) : __assert_fail ("ChainVal.getValueType() == MVT::Other && \"Not a chain?\"" , "/build/llvm-toolchain-snapshot-10~++20200112100611+7fa5290d5bd/llvm/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp" , 2337, __PRETTY_FUNCTION__)); | |||
| 2338 | SelectionDAG::DAGNodeDeletedListener NDL( | |||
| 2339 | *CurDAG, [&](SDNode *N, SDNode *E) { | |||
| 2340 | std::replace(ChainNodesMatched.begin(), ChainNodesMatched.end(), N, | |||
| 2341 | static_cast<SDNode *>(nullptr)); | |||
| 2342 | }); | |||
| 2343 | if (ChainNode->getOpcode() != ISD::TokenFactor) | |||
| 2344 | ReplaceUses(ChainVal, InputChain); | |||
| 2345 | ||||
| 2346 | // If the node became dead and we haven't already seen it, delete it. | |||
| 2347 | if (ChainNode != NodeToMatch && ChainNode->use_empty() && | |||
| 2348 | !std::count(NowDeadNodes.begin(), NowDeadNodes.end(), ChainNode)) | |||
| 2349 | NowDeadNodes.push_back(ChainNode); | |||
| 2350 | } | |||
| 2351 | } | |||
| 2352 | ||||
| 2353 | if (!NowDeadNodes.empty()) | |||
| 2354 | CurDAG->RemoveDeadNodes(NowDeadNodes); | |||
| 2355 | ||||
| 2356 | LLVM_DEBUG(dbgs() << "ISEL: Match complete!\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("isel")) { dbgs() << "ISEL: Match complete!\n"; } } while (false); | |||
| 2357 | } | |||
| 2358 | ||||
| 2359 | /// HandleMergeInputChains - This implements the OPC_EmitMergeInputChains | |||
| 2360 | /// operation for when the pattern matched at least one node with a chains. The | |||
| 2361 | /// input vector contains a list of all of the chained nodes that we match. We | |||
| 2362 | /// must determine if this is a valid thing to cover (i.e. matching it won't | |||
| 2363 | /// induce cycles in the DAG) and if so, creating a TokenFactor node. that will | |||
| 2364 | /// be used as the input node chain for the generated nodes. | |||
| 2365 | static SDValue | |||
| 2366 | HandleMergeInputChains(SmallVectorImpl<SDNode*> &ChainNodesMatched, | |||
| 2367 | SelectionDAG *CurDAG) { | |||
| 2368 | ||||
| 2369 | SmallPtrSet<const SDNode *, 16> Visited; | |||
| 2370 | SmallVector<const SDNode *, 8> Worklist; | |||
| 2371 | SmallVector<SDValue, 3> InputChains; | |||
| 2372 | unsigned int Max = 8192; | |||
| 2373 | ||||
| 2374 | // Quick exit on trivial merge. | |||
| 2375 | if (ChainNodesMatched.size() == 1) | |||
| 2376 | return ChainNodesMatched[0]->getOperand(0); | |||
| 2377 | ||||
| 2378 | // Add chains that aren't already added (internal). Peek through | |||
| 2379 | // token factors. | |||
| 2380 | std::function<void(const SDValue)> AddChains = [&](const SDValue V) { | |||
| 2381 | if (V.getValueType() != MVT::Other) | |||
| 2382 | return; | |||
| 2383 | if (V->getOpcode() == ISD::EntryToken) | |||
| 2384 | return; | |||
| 2385 | if (!Visited.insert(V.getNode()).second) | |||
| 2386 | return; | |||
| 2387 | if (V->getOpcode() == ISD::TokenFactor) { | |||
| 2388 | for (const SDValue &Op : V->op_values()) | |||
| 2389 | AddChains(Op); | |||
| 2390 | } else | |||
| 2391 | InputChains.push_back(V); | |||
| 2392 | }; | |||
| 2393 | ||||
| 2394 | for (auto *N : ChainNodesMatched) { | |||
| 2395 | Worklist.push_back(N); | |||
| 2396 | Visited.insert(N); | |||
| 2397 | } | |||
| 2398 | ||||
| 2399 | while (!Worklist.empty()) | |||
| 2400 | AddChains(Worklist.pop_back_val()->getOperand(0)); | |||
| 2401 | ||||
| 2402 | // Skip the search if there are no chain dependencies. | |||
| 2403 | if (InputChains.size() == 0) | |||
| 2404 | return CurDAG->getEntryNode(); | |||
| 2405 | ||||
| 2406 | // If one of these chains is a successor of input, we must have a | |||
| 2407 | // node that is both the predecessor and successor of the | |||
| 2408 | // to-be-merged nodes. Fail. | |||
| 2409 | Visited.clear(); | |||
| 2410 | for (SDValue V : InputChains) | |||
| 2411 | Worklist.push_back(V.getNode()); | |||
| 2412 | ||||
| 2413 | for (auto *N : ChainNodesMatched) | |||
| 2414 | if (SDNode::hasPredecessorHelper(N, Visited, Worklist, Max, true)) | |||
| 2415 | return SDValue(); | |||
| 2416 | ||||
| 2417 | // Return merged chain. | |||
| 2418 | if (InputChains.size() == 1) | |||
| 2419 | return InputChains[0]; | |||
| 2420 | return CurDAG->getNode(ISD::TokenFactor, SDLoc(ChainNodesMatched[0]), | |||
| 2421 | MVT::Other, InputChains); | |||
| 2422 | } | |||
| 2423 | ||||
| 2424 | /// MorphNode - Handle morphing a node in place for the selector. | |||
| 2425 | SDNode *SelectionDAGISel:: | |||
| 2426 | MorphNode(SDNode *Node, unsigned TargetOpc, SDVTList VTList, | |||
| 2427 | ArrayRef<SDValue> Ops, unsigned EmitNodeInfo) { | |||
| 2428 | // It is possible we're using MorphNodeTo to replace a node with no | |||
| 2429 | // normal results with one that has a normal result (or we could be | |||
| 2430 | // adding a chain) and the input could have glue and chains as well. | |||
| 2431 | // In this case we need to shift the operands down. | |||
| 2432 | // FIXME: This is a horrible hack and broken in obscure cases, no worse | |||
| 2433 | // than the old isel though. | |||
| 2434 | int OldGlueResultNo = -1, OldChainResultNo = -1; | |||
| 2435 | ||||
| 2436 | unsigned NTMNumResults = Node->getNumValues(); | |||
| 2437 | if (Node->getValueType(NTMNumResults-1) == MVT::Glue) { | |||
| 2438 | OldGlueResultNo = NTMNumResults-1; | |||
| 2439 | if (NTMNumResults != 1 && | |||
| 2440 | Node->getValueType(NTMNumResults-2) == MVT::Other) | |||
| 2441 | OldChainResultNo = NTMNumResults-2; | |||
| 2442 | } else if (Node->getValueType(NTMNumResults-1) == MVT::Other) | |||
| 2443 | OldChainResultNo = NTMNumResults-1; | |||
| 2444 | ||||
| 2445 | // Call the underlying SelectionDAG routine to do the transmogrification. Note | |||
| 2446 | // that this deletes operands of the old node that become dead. | |||
| 2447 | SDNode *Res = CurDAG->MorphNodeTo(Node, ~TargetOpc, VTList, Ops); | |||
| 2448 | ||||
| 2449 | // MorphNodeTo can operate in two ways: if an existing node with the | |||
| 2450 | // specified operands exists, it can just return it. Otherwise, it | |||
| 2451 | // updates the node in place to have the requested operands. | |||
| 2452 | if (Res == Node) { | |||
| 2453 | // If we updated the node in place, reset the node ID. To the isel, | |||
| 2454 | // this should be just like a newly allocated machine node. | |||
| 2455 | Res->setNodeId(-1); | |||
| 2456 | } | |||
| 2457 | ||||
| 2458 | unsigned ResNumResults = Res->getNumValues(); | |||
| 2459 | // Move the glue if needed. | |||
| 2460 | if ((EmitNodeInfo & OPFL_GlueOutput) && OldGlueResultNo != -1 && | |||
| 2461 | (unsigned)OldGlueResultNo != ResNumResults-1) | |||
| 2462 | ReplaceUses(SDValue(Node, OldGlueResultNo), | |||
| 2463 | SDValue(Res, ResNumResults - 1)); | |||
| 2464 | ||||
| 2465 | if ((EmitNodeInfo & OPFL_GlueOutput) != 0) | |||
| 2466 | --ResNumResults; | |||
| 2467 | ||||
| 2468 | // Move the chain reference if needed. | |||
| 2469 | if ((EmitNodeInfo & OPFL_Chain) && OldChainResultNo != -1 && | |||
| 2470 | (unsigned)OldChainResultNo != ResNumResults-1) | |||
| 2471 | ReplaceUses(SDValue(Node, OldChainResultNo), | |||
| 2472 | SDValue(Res, ResNumResults - 1)); | |||
| 2473 | ||||
| 2474 | // Otherwise, no replacement happened because the node already exists. Replace | |||
| 2475 | // Uses of the old node with the new one. | |||
| 2476 | if (Res != Node) { | |||
| 2477 | ReplaceNode(Node, Res); | |||
| 2478 | } else { | |||
| 2479 | EnforceNodeIdInvariant(Res); | |||
| 2480 | } | |||
| 2481 | ||||
| 2482 | return Res; | |||
| 2483 | } | |||
| 2484 | ||||
| 2485 | /// CheckSame - Implements OP_CheckSame. | |||
| 2486 | LLVM_ATTRIBUTE_ALWAYS_INLINE__attribute__((always_inline)) static inline bool | |||
| 2487 | CheckSame(const unsigned char *MatcherTable, unsigned &MatcherIndex, | |||
| 2488 | SDValue N, | |||
| 2489 | const SmallVectorImpl<std::pair<SDValue, SDNode*>> &RecordedNodes) { | |||
| 2490 | // Accept if it is exactly the same as a previously recorded node. | |||
| 2491 | unsigned RecNo = MatcherTable[MatcherIndex++]; | |||
| 2492 | assert(RecNo < RecordedNodes.size() && "Invalid CheckSame")((RecNo < RecordedNodes.size() && "Invalid CheckSame" ) ? static_cast<void> (0) : __assert_fail ("RecNo < RecordedNodes.size() && \"Invalid CheckSame\"" , "/build/llvm-toolchain-snapshot-10~++20200112100611+7fa5290d5bd/llvm/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp" , 2492, __PRETTY_FUNCTION__)); | |||
| 2493 | return N == RecordedNodes[RecNo].first; | |||
| 2494 | } | |||
| 2495 | ||||
| 2496 | /// CheckChildSame - Implements OP_CheckChildXSame. | |||
| 2497 | LLVM_ATTRIBUTE_ALWAYS_INLINE__attribute__((always_inline)) static inline bool | |||
| 2498 | CheckChildSame(const unsigned char *MatcherTable, unsigned &MatcherIndex, | |||
| 2499 | SDValue N, | |||
| 2500 | const SmallVectorImpl<std::pair<SDValue, SDNode*>> &RecordedNodes, | |||
| 2501 | unsigned ChildNo) { | |||
| 2502 | if (ChildNo >= N.getNumOperands()) | |||
| 2503 | return false; // Match fails if out of range child #. | |||
| 2504 | return ::CheckSame(MatcherTable, MatcherIndex, N.getOperand(ChildNo), | |||
| 2505 | RecordedNodes); | |||
| 2506 | } | |||
| 2507 | ||||
| 2508 | /// CheckPatternPredicate - Implements OP_CheckPatternPredicate. | |||
| 2509 | LLVM_ATTRIBUTE_ALWAYS_INLINE__attribute__((always_inline)) static inline bool | |||
| 2510 | CheckPatternPredicate(const unsigned char *MatcherTable, unsigned &MatcherIndex, | |||
| 2511 | const SelectionDAGISel &SDISel) { | |||
| 2512 | return SDISel.CheckPatternPredicate(MatcherTable[MatcherIndex++]); | |||
| 2513 | } | |||
| 2514 | ||||
| 2515 | /// CheckNodePredicate - Implements OP_CheckNodePredicate. | |||
| 2516 | LLVM_ATTRIBUTE_ALWAYS_INLINE__attribute__((always_inline)) static inline bool | |||
| 2517 | CheckNodePredicate(const unsigned char *MatcherTable, unsigned &MatcherIndex, | |||
| 2518 | const SelectionDAGISel &SDISel, SDNode *N) { | |||
| 2519 | return SDISel.CheckNodePredicate(N, MatcherTable[MatcherIndex++]); | |||
| 2520 | } | |||
| 2521 | ||||
| 2522 | LLVM_ATTRIBUTE_ALWAYS_INLINE__attribute__((always_inline)) static inline bool | |||
| 2523 | CheckOpcode(const unsigned char *MatcherTable, unsigned &MatcherIndex, | |||
| 2524 | SDNode *N) { | |||
| 2525 | uint16_t Opc = MatcherTable[MatcherIndex++]; | |||
| 2526 | Opc |= (unsigned short)MatcherTable[MatcherIndex++] << 8; | |||
| 2527 | return N->getOpcode() == Opc; | |||
| 2528 | } | |||
| 2529 | ||||
| 2530 | LLVM_ATTRIBUTE_ALWAYS_INLINE__attribute__((always_inline)) static inline bool | |||
| 2531 | CheckType(const unsigned char *MatcherTable, unsigned &MatcherIndex, SDValue N, | |||
| 2532 | const TargetLowering *TLI, const DataLayout &DL) { | |||
| 2533 | MVT::SimpleValueType VT = (MVT::SimpleValueType)MatcherTable[MatcherIndex++]; | |||
| 2534 | if (N.getValueType() == VT) return true; | |||
| 2535 | ||||
| 2536 | // Handle the case when VT is iPTR. | |||
| 2537 | return VT == MVT::iPTR && N.getValueType() == TLI->getPointerTy(DL); | |||
| 2538 | } | |||
| 2539 | ||||
| 2540 | LLVM_ATTRIBUTE_ALWAYS_INLINE__attribute__((always_inline)) static inline bool | |||
| 2541 | CheckChildType(const unsigned char *MatcherTable, unsigned &MatcherIndex, | |||
| 2542 | SDValue N, const TargetLowering *TLI, const DataLayout &DL, | |||
| 2543 | unsigned ChildNo) { | |||
| 2544 | if (ChildNo >= N.getNumOperands()) | |||
| 2545 | return false; // Match fails if out of range child #. | |||
| 2546 | return ::CheckType(MatcherTable, MatcherIndex, N.getOperand(ChildNo), TLI, | |||
| 2547 | DL); | |||
| 2548 | } | |||
| 2549 | ||||
| 2550 | LLVM_ATTRIBUTE_ALWAYS_INLINE__attribute__((always_inline)) static inline bool | |||
| 2551 | CheckCondCode(const unsigned char *MatcherTable, unsigned &MatcherIndex, | |||
| 2552 | SDValue N) { | |||
| 2553 | return cast<CondCodeSDNode>(N)->get() == | |||
| 2554 | (ISD::CondCode)MatcherTable[MatcherIndex++]; | |||
| 2555 | } | |||
| 2556 | ||||
| 2557 | LLVM_ATTRIBUTE_ALWAYS_INLINE__attribute__((always_inline)) static inline bool | |||
| 2558 | CheckChild2CondCode(const unsigned char *MatcherTable, unsigned &MatcherIndex, | |||
| 2559 | SDValue N) { | |||
| 2560 | if (2 >= N.getNumOperands()) | |||
| 2561 | return false; | |||
| 2562 | return ::CheckCondCode(MatcherTable, MatcherIndex, N.getOperand(2)); | |||
| 2563 | } | |||
| 2564 | ||||
| 2565 | LLVM_ATTRIBUTE_ALWAYS_INLINE__attribute__((always_inline)) static inline bool | |||
| 2566 | CheckValueType(const unsigned char *MatcherTable, unsigned &MatcherIndex, | |||
| 2567 | SDValue N, const TargetLowering *TLI, const DataLayout &DL) { | |||
| 2568 | MVT::SimpleValueType VT = (MVT::SimpleValueType)MatcherTable[MatcherIndex++]; | |||
| 2569 | if (cast<VTSDNode>(N)->getVT() == VT) | |||
| 2570 | return true; | |||
| 2571 | ||||
| 2572 | // Handle the case when VT is iPTR. | |||
| 2573 | return VT == MVT::iPTR && cast<VTSDNode>(N)->getVT() == TLI->getPointerTy(DL); | |||
| 2574 | } | |||
| 2575 | ||||
| 2576 | LLVM_ATTRIBUTE_ALWAYS_INLINE__attribute__((always_inline)) static inline bool | |||
| 2577 | CheckInteger(const unsigned char *MatcherTable, unsigned &MatcherIndex, | |||
| 2578 | SDValue N) { | |||
| 2579 | int64_t Val = MatcherTable[MatcherIndex++]; | |||
| 2580 | if (Val & 128) | |||
| 2581 | Val = GetVBR(Val, MatcherTable, MatcherIndex); | |||
| 2582 | ||||
| 2583 | ConstantSDNode *C = dyn_cast<ConstantSDNode>(N); | |||
| 2584 | return C && C->getSExtValue() == Val; | |||
| 2585 | } | |||
| 2586 | ||||
| 2587 | LLVM_ATTRIBUTE_ALWAYS_INLINE__attribute__((always_inline)) static inline bool | |||
| 2588 | CheckChildInteger(const unsigned char *MatcherTable, unsigned &MatcherIndex, | |||
| 2589 | SDValue N, unsigned ChildNo) { | |||
| 2590 | if (ChildNo >= N.getNumOperands()) | |||
| 2591 | return false; // Match fails if out of range child #. | |||
| 2592 | return ::CheckInteger(MatcherTable, MatcherIndex, N.getOperand(ChildNo)); | |||
| 2593 | } | |||
| 2594 | ||||
| 2595 | LLVM_ATTRIBUTE_ALWAYS_INLINE__attribute__((always_inline)) static inline bool | |||
| 2596 | CheckAndImm(const unsigned char *MatcherTable, unsigned &MatcherIndex, | |||
| 2597 | SDValue N, const SelectionDAGISel &SDISel) { | |||
| 2598 | int64_t Val = MatcherTable[MatcherIndex++]; | |||
| 2599 | if (Val & 128) | |||
| 2600 | Val = GetVBR(Val, MatcherTable, MatcherIndex); | |||
| 2601 | ||||
| 2602 | if (N->getOpcode() != ISD::AND) return false; | |||
| 2603 | ||||
| 2604 | ConstantSDNode *C = dyn_cast<ConstantSDNode>(N->getOperand(1)); | |||
| 2605 | return C && SDISel.CheckAndMask(N.getOperand(0), C, Val); | |||
| 2606 | } | |||
| 2607 | ||||
| 2608 | LLVM_ATTRIBUTE_ALWAYS_INLINE__attribute__((always_inline)) static inline bool | |||
| 2609 | CheckOrImm(const unsigned char *MatcherTable, unsigned &MatcherIndex, | |||
| 2610 | SDValue N, const SelectionDAGISel &SDISel) { | |||
| 2611 | int64_t Val = MatcherTable[MatcherIndex++]; | |||
| 2612 | if (Val & 128) | |||
| 2613 | Val = GetVBR(Val, MatcherTable, MatcherIndex); | |||
| 2614 | ||||
| 2615 | if (N->getOpcode() != ISD::OR) return false; | |||
| 2616 | ||||
| 2617 | ConstantSDNode *C = dyn_cast<ConstantSDNode>(N->getOperand(1)); | |||
| 2618 | return C && SDISel.CheckOrMask(N.getOperand(0), C, Val); | |||
| 2619 | } | |||
| 2620 | ||||
| 2621 | /// IsPredicateKnownToFail - If we know how and can do so without pushing a | |||
| 2622 | /// scope, evaluate the current node. If the current predicate is known to | |||
| 2623 | /// fail, set Result=true and return anything. If the current predicate is | |||
| 2624 | /// known to pass, set Result=false and return the MatcherIndex to continue | |||
| 2625 | /// with. If the current predicate is unknown, set Result=false and return the | |||
| 2626 | /// MatcherIndex to continue with. | |||
| 2627 | static unsigned IsPredicateKnownToFail(const unsigned char *Table, | |||
| 2628 | unsigned Index, SDValue N, | |||
| 2629 | bool &Result, | |||
| 2630 | const SelectionDAGISel &SDISel, | |||
| 2631 | SmallVectorImpl<std::pair<SDValue, SDNode*>> &RecordedNodes) { | |||
| 2632 | switch (Table[Index++]) { | |||
| 2633 | default: | |||
| 2634 | Result = false; | |||
| 2635 | return Index-1; // Could not evaluate this predicate. | |||
| 2636 | case SelectionDAGISel::OPC_CheckSame: | |||
| 2637 | Result = !::CheckSame(Table, Index, N, RecordedNodes); | |||
| 2638 | return Index; | |||
| 2639 | case SelectionDAGISel::OPC_CheckChild0Same: | |||
| 2640 | case SelectionDAGISel::OPC_CheckChild1Same: | |||
| 2641 | case SelectionDAGISel::OPC_CheckChild2Same: | |||
| 2642 | case SelectionDAGISel::OPC_CheckChild3Same: | |||
| 2643 | Result = !::CheckChildSame(Table, Index, N, RecordedNodes, | |||
| 2644 | Table[Index-1] - SelectionDAGISel::OPC_CheckChild0Same); | |||
| 2645 | return Index; | |||
| 2646 | case SelectionDAGISel::OPC_CheckPatternPredicate: | |||
| 2647 | Result = !::CheckPatternPredicate(Table, Index, SDISel); | |||
| 2648 | return Index; | |||
| 2649 | case SelectionDAGISel::OPC_CheckPredicate: | |||
| 2650 | Result = !::CheckNodePredicate(Table, Index, SDISel, N.getNode()); | |||
| 2651 | return Index; | |||
| 2652 | case SelectionDAGISel::OPC_CheckOpcode: | |||
| 2653 | Result = !::CheckOpcode(Table, Index, N.getNode()); | |||
| 2654 | return Index; | |||
| 2655 | case SelectionDAGISel::OPC_CheckType: | |||
| 2656 | Result = !::CheckType(Table, Index, N, SDISel.TLI, | |||
| 2657 | SDISel.CurDAG->getDataLayout()); | |||
| 2658 | return Index; | |||
| 2659 | case SelectionDAGISel::OPC_CheckTypeRes: { | |||
| 2660 | unsigned Res = Table[Index++]; | |||
| 2661 | Result = !::CheckType(Table, Index, N.getValue(Res), SDISel.TLI, | |||
| 2662 | SDISel.CurDAG->getDataLayout()); | |||
| 2663 | return Index; | |||
| 2664 | } | |||
| 2665 | case SelectionDAGISel::OPC_CheckChild0Type: | |||
| 2666 | case SelectionDAGISel::OPC_CheckChild1Type: | |||
| 2667 | case SelectionDAGISel::OPC_CheckChild2Type: | |||
| 2668 | case SelectionDAGISel::OPC_CheckChild3Type: | |||
| 2669 | case SelectionDAGISel::OPC_CheckChild4Type: | |||
| 2670 | case SelectionDAGISel::OPC_CheckChild5Type: | |||
| 2671 | case SelectionDAGISel::OPC_CheckChild6Type: | |||
| 2672 | case SelectionDAGISel::OPC_CheckChild7Type: | |||
| 2673 | Result = !::CheckChildType( | |||
| 2674 | Table, Index, N, SDISel.TLI, SDISel.CurDAG->getDataLayout(), | |||
| 2675 | Table[Index - 1] - SelectionDAGISel::OPC_CheckChild0Type); | |||
| 2676 | return Index; | |||
| 2677 | case SelectionDAGISel::OPC_CheckCondCode: | |||
| 2678 | Result = !::CheckCondCode(Table, Index, N); | |||
| 2679 | return Index; | |||
| 2680 | case SelectionDAGISel::OPC_CheckChild2CondCode: | |||
| 2681 | Result = !::CheckChild2CondCode(Table, Index, N); | |||
| 2682 | return Index; | |||
| 2683 | case SelectionDAGISel::OPC_CheckValueType: | |||
| 2684 | Result = !::CheckValueType(Table, Index, N, SDISel.TLI, | |||
| 2685 | SDISel.CurDAG->getDataLayout()); | |||
| 2686 | return Index; | |||
| 2687 | case SelectionDAGISel::OPC_CheckInteger: | |||
| 2688 | Result = !::CheckInteger(Table, Index, N); | |||
| 2689 | return Index; | |||
| 2690 | case SelectionDAGISel::OPC_CheckChild0Integer: | |||
| 2691 | case SelectionDAGISel::OPC_CheckChild1Integer: | |||
| 2692 | case SelectionDAGISel::OPC_CheckChild2Integer: | |||
| 2693 | case SelectionDAGISel::OPC_CheckChild3Integer: | |||
| 2694 | case SelectionDAGISel::OPC_CheckChild4Integer: | |||
| 2695 | Result = !::CheckChildInteger(Table, Index, N, | |||
| 2696 | Table[Index-1] - SelectionDAGISel::OPC_CheckChild0Integer); | |||
| 2697 | return Index; | |||
| 2698 | case SelectionDAGISel::OPC_CheckAndImm: | |||
| 2699 | Result = !::CheckAndImm(Table, Index, N, SDISel); | |||
| 2700 | return Index; | |||
| 2701 | case SelectionDAGISel::OPC_CheckOrImm: | |||
| 2702 | Result = !::CheckOrImm(Table, Index, N, SDISel); | |||
| 2703 | return Index; | |||
| 2704 | } | |||
| 2705 | } | |||
| 2706 | ||||
| 2707 | namespace { | |||
| 2708 | ||||
| 2709 | struct MatchScope { | |||
| 2710 | /// FailIndex - If this match fails, this is the index to continue with. | |||
| 2711 | unsigned FailIndex; | |||
| 2712 | ||||
| 2713 | /// NodeStack - The node stack when the scope was formed. | |||
| 2714 | SmallVector<SDValue, 4> NodeStack; | |||
| 2715 | ||||
| 2716 | /// NumRecordedNodes - The number of recorded nodes when the scope was formed. | |||
| 2717 | unsigned NumRecordedNodes; | |||
| 2718 | ||||
| 2719 | /// NumMatchedMemRefs - The number of matched memref entries. | |||
| 2720 | unsigned NumMatchedMemRefs; | |||
| 2721 | ||||
| 2722 | /// InputChain/InputGlue - The current chain/glue | |||
| 2723 | SDValue InputChain, InputGlue; | |||
| 2724 | ||||
| 2725 | /// HasChainNodesMatched - True if the ChainNodesMatched list is non-empty. | |||
| 2726 | bool HasChainNodesMatched; | |||
| 2727 | }; | |||
| 2728 | ||||
| 2729 | /// \A DAG update listener to keep the matching state | |||
| 2730 | /// (i.e. RecordedNodes and MatchScope) uptodate if the target is allowed to | |||
| 2731 | /// change the DAG while matching. X86 addressing mode matcher is an example | |||
| 2732 | /// for this. | |||
| 2733 | class MatchStateUpdater : public SelectionDAG::DAGUpdateListener | |||
| 2734 | { | |||
| 2735 | SDNode **NodeToMatch; | |||
| 2736 | SmallVectorImpl<std::pair<SDValue, SDNode *>> &RecordedNodes; | |||
| 2737 | SmallVectorImpl<MatchScope> &MatchScopes; | |||
| 2738 | ||||
| 2739 | public: | |||
| 2740 | MatchStateUpdater(SelectionDAG &DAG, SDNode **NodeToMatch, | |||
| 2741 | SmallVectorImpl<std::pair<SDValue, SDNode *>> &RN, | |||
| 2742 | SmallVectorImpl<MatchScope> &MS) | |||
| 2743 | : SelectionDAG::DAGUpdateListener(DAG), NodeToMatch(NodeToMatch), | |||
| 2744 | RecordedNodes(RN), MatchScopes(MS) {} | |||
| 2745 | ||||
| 2746 | void NodeDeleted(SDNode *N, SDNode *E) override { | |||
| 2747 | // Some early-returns here to avoid the search if we deleted the node or | |||
| 2748 | // if the update comes from MorphNodeTo (MorphNodeTo is the last thing we | |||
| 2749 | // do, so it's unnecessary to update matching state at that point). | |||
| 2750 | // Neither of these can occur currently because we only install this | |||
| 2751 | // update listener during matching a complex patterns. | |||
| 2752 | if (!E || E->isMachineOpcode()) | |||
| 2753 | return; | |||
| 2754 | // Check if NodeToMatch was updated. | |||
| 2755 | if (N == *NodeToMatch) | |||
| 2756 | *NodeToMatch = E; | |||
| 2757 | // Performing linear search here does not matter because we almost never | |||
| 2758 | // run this code. You'd have to have a CSE during complex pattern | |||
| 2759 | // matching. | |||
| 2760 | for (auto &I : RecordedNodes) | |||
| 2761 | if (I.first.getNode() == N) | |||
| 2762 | I.first.setNode(E); | |||
| 2763 | ||||
| 2764 | for (auto &I : MatchScopes) | |||
| 2765 | for (auto &J : I.NodeStack) | |||
| 2766 | if (J.getNode() == N) | |||
| 2767 | J.setNode(E); | |||
| 2768 | } | |||
| 2769 | }; | |||
| 2770 | ||||
| 2771 | } // end anonymous namespace | |||
| 2772 | ||||
| 2773 | void SelectionDAGISel::SelectCodeCommon(SDNode *NodeToMatch, | |||
| 2774 | const unsigned char *MatcherTable, | |||
| 2775 | unsigned TableSize) { | |||
| 2776 | // FIXME: Should these even be selected? Handle these cases in the caller? | |||
| 2777 | switch (NodeToMatch->getOpcode()) { | |||
| 2778 | default: | |||
| 2779 | break; | |||
| 2780 | case ISD::EntryToken: // These nodes remain the same. | |||
| 2781 | case ISD::BasicBlock: | |||
| 2782 | case ISD::Register: | |||
| 2783 | case ISD::RegisterMask: | |||
| 2784 | case ISD::HANDLENODE: | |||
| 2785 | case ISD::MDNODE_SDNODE: | |||
| 2786 | case ISD::TargetConstant: | |||
| 2787 | case ISD::TargetConstantFP: | |||
| 2788 | case ISD::TargetConstantPool: | |||
| 2789 | case ISD::TargetFrameIndex: | |||
| 2790 | case ISD::TargetExternalSymbol: | |||
| 2791 | case ISD::MCSymbol: | |||
| 2792 | case ISD::TargetBlockAddress: | |||
| 2793 | case ISD::TargetJumpTable: | |||
| 2794 | case ISD::TargetGlobalTLSAddress: | |||
| 2795 | case ISD::TargetGlobalAddress: | |||
| 2796 | case ISD::TokenFactor: | |||
| 2797 | case ISD::CopyFromReg: | |||
| 2798 | case ISD::CopyToReg: | |||
| 2799 | case ISD::EH_LABEL: | |||
| 2800 | case ISD::ANNOTATION_LABEL: | |||
| 2801 | case ISD::LIFETIME_START: | |||
| 2802 | case ISD::LIFETIME_END: | |||
| 2803 | NodeToMatch->setNodeId(-1); // Mark selected. | |||
| 2804 | return; | |||
| 2805 | case ISD::AssertSext: | |||
| 2806 | case ISD::AssertZext: | |||
| 2807 | ReplaceUses(SDValue(NodeToMatch, 0), NodeToMatch->getOperand(0)); | |||
| 2808 | CurDAG->RemoveDeadNode(NodeToMatch); | |||
| 2809 | return; | |||
| 2810 | case ISD::INLINEASM: | |||
| 2811 | case ISD::INLINEASM_BR: | |||
| 2812 | Select_INLINEASM(NodeToMatch, | |||
| 2813 | NodeToMatch->getOpcode() == ISD::INLINEASM_BR); | |||
| 2814 | return; | |||
| 2815 | case ISD::READ_REGISTER: | |||
| 2816 | Select_READ_REGISTER(NodeToMatch); | |||
| 2817 | return; | |||
| 2818 | case ISD::WRITE_REGISTER: | |||
| 2819 | Select_WRITE_REGISTER(NodeToMatch); | |||
| 2820 | return; | |||
| 2821 | case ISD::UNDEF: | |||
| 2822 | Select_UNDEF(NodeToMatch); | |||
| 2823 | return; | |||
| 2824 | } | |||
| 2825 | ||||
| 2826 | assert(!NodeToMatch->isMachineOpcode() && "Node already selected!")((!NodeToMatch->isMachineOpcode() && "Node already selected!" ) ? static_cast<void> (0) : __assert_fail ("!NodeToMatch->isMachineOpcode() && \"Node already selected!\"" , "/build/llvm-toolchain-snapshot-10~++20200112100611+7fa5290d5bd/llvm/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp" , 2826, __PRETTY_FUNCTION__)); | |||
| 2827 | ||||
| 2828 | // Set up the node stack with NodeToMatch as the only node on the stack. | |||
| 2829 | SmallVector<SDValue, 8> NodeStack; | |||
| 2830 | SDValue N = SDValue(NodeToMatch, 0); | |||
| 2831 | NodeStack.push_back(N); | |||
| 2832 | ||||
| 2833 | // MatchScopes - Scopes used when matching, if a match failure happens, this | |||
| 2834 | // indicates where to continue checking. | |||
| 2835 | SmallVector<MatchScope, 8> MatchScopes; | |||
| 2836 | ||||
| 2837 | // RecordedNodes - This is the set of nodes that have been recorded by the | |||
| 2838 | // state machine. The second value is the parent of the node, or null if the | |||
| 2839 | // root is recorded. | |||
| 2840 | SmallVector<std::pair<SDValue, SDNode*>, 8> RecordedNodes; | |||
| 2841 | ||||
| 2842 | // MatchedMemRefs - This is the set of MemRef's we've seen in the input | |||
| 2843 | // pattern. | |||
| 2844 | SmallVector<MachineMemOperand*, 2> MatchedMemRefs; | |||
| 2845 | ||||
| 2846 | // These are the current input chain and glue for use when generating nodes. | |||
| 2847 | // Various Emit operations change these. For example, emitting a copytoreg | |||
| 2848 | // uses and updates these. | |||
| 2849 | SDValue InputChain, InputGlue; | |||
| 2850 | ||||
| 2851 | // ChainNodesMatched - If a pattern matches nodes that have input/output | |||
| 2852 | // chains, the OPC_EmitMergeInputChains operation is emitted which indicates | |||
| 2853 | // which ones they are. The result is captured into this list so that we can | |||
| 2854 | // update the chain results when the pattern is complete. | |||
| 2855 | SmallVector<SDNode*, 3> ChainNodesMatched; | |||
| 2856 | ||||
| 2857 | LLVM_DEBUG(dbgs() << "ISEL: Starting pattern match\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("isel")) { dbgs() << "ISEL: Starting pattern match\n"; } } while (false); | |||
| 2858 | ||||
| 2859 | // Determine where to start the interpreter. Normally we start at opcode #0, | |||
| 2860 | // but if the state machine starts with an OPC_SwitchOpcode, then we | |||
| 2861 | // accelerate the first lookup (which is guaranteed to be hot) with the | |||
| 2862 | // OpcodeOffset table. | |||
| 2863 | unsigned MatcherIndex = 0; | |||
| 2864 | ||||
| 2865 | if (!OpcodeOffset.empty()) { | |||
| 2866 | // Already computed the OpcodeOffset table, just index into it. | |||
| 2867 | if (N.getOpcode() < OpcodeOffset.size()) | |||
| 2868 | MatcherIndex = OpcodeOffset[N.getOpcode()]; | |||
| 2869 | LLVM_DEBUG(dbgs() << " Initial Opcode index to " << MatcherIndex << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("isel")) { dbgs() << " Initial Opcode index to " << MatcherIndex << "\n"; } } while (false); | |||
| 2870 | ||||
| 2871 | } else if (MatcherTable[0] == OPC_SwitchOpcode) { | |||
| 2872 | // Otherwise, the table isn't computed, but the state machine does start | |||
| 2873 | // with an OPC_SwitchOpcode instruction. Populate the table now, since this | |||
| 2874 | // is the first time we're selecting an instruction. | |||
| 2875 | unsigned Idx = 1; | |||
| 2876 | while (true) { | |||
| 2877 | // Get the size of this case. | |||
| 2878 | unsigned CaseSize = MatcherTable[Idx++]; | |||
| 2879 | if (CaseSize & 128) | |||
| 2880 | CaseSize = GetVBR(CaseSize, MatcherTable, Idx); | |||
| 2881 | if (CaseSize == 0) break; | |||
| 2882 | ||||
| 2883 | // Get the opcode, add the index to the table. | |||
| 2884 | uint16_t Opc = MatcherTable[Idx++]; | |||
| 2885 | Opc |= (unsigned short)MatcherTable[Idx++] << 8; | |||
| 2886 | if (Opc >= OpcodeOffset.size()) | |||
| 2887 | OpcodeOffset.resize((Opc+1)*2); | |||
| 2888 | OpcodeOffset[Opc] = Idx; | |||
| 2889 | Idx += CaseSize; | |||
| 2890 | } | |||
| 2891 | ||||
| 2892 | // Okay, do the lookup for the first opcode. | |||
| 2893 | if (N.getOpcode() < OpcodeOffset.size()) | |||
| 2894 | MatcherIndex = OpcodeOffset[N.getOpcode()]; | |||
| 2895 | } | |||
| 2896 | ||||
| 2897 | while (true) { | |||
| 2898 | assert(MatcherIndex < TableSize && "Invalid index")((MatcherIndex < TableSize && "Invalid index") ? static_cast <void> (0) : __assert_fail ("MatcherIndex < TableSize && \"Invalid index\"" , "/build/llvm-toolchain-snapshot-10~++20200112100611+7fa5290d5bd/llvm/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp" , 2898, __PRETTY_FUNCTION__)); | |||
| 2899 | #ifndef NDEBUG | |||
| 2900 | unsigned CurrentOpcodeIndex = MatcherIndex; | |||
| 2901 | #endif | |||
| 2902 | BuiltinOpcodes Opcode = (BuiltinOpcodes)MatcherTable[MatcherIndex++]; | |||
| 2903 | switch (Opcode) { | |||
| 2904 | case OPC_Scope: { | |||
| 2905 | // Okay, the semantics of this operation are that we should push a scope | |||
| 2906 | // then evaluate the first child. However, pushing a scope only to have | |||
| 2907 | // the first check fail (which then pops it) is inefficient. If we can | |||
| 2908 | // determine immediately that the first check (or first several) will | |||
| 2909 | // immediately fail, don't even bother pushing a scope for them. | |||
| 2910 | unsigned FailIndex; | |||
| 2911 | ||||
| 2912 | while (true) { | |||
| 2913 | unsigned NumToSkip = MatcherTable[MatcherIndex++]; | |||
| 2914 | if (NumToSkip & 128) | |||
| 2915 | NumToSkip = GetVBR(NumToSkip, MatcherTable, MatcherIndex); | |||
| 2916 | // Found the end of the scope with no match. | |||
| 2917 | if (NumToSkip == 0) { | |||
| 2918 | FailIndex = 0; | |||
| 2919 | break; | |||
| 2920 | } | |||
| 2921 | ||||
| 2922 | FailIndex = MatcherIndex+NumToSkip; | |||
| 2923 | ||||
| 2924 | unsigned MatcherIndexOfPredicate = MatcherIndex; | |||
| 2925 | (void)MatcherIndexOfPredicate; // silence warning. | |||
| 2926 | ||||
| 2927 | // If we can't evaluate this predicate without pushing a scope (e.g. if | |||
| 2928 | // it is a 'MoveParent') or if the predicate succeeds on this node, we | |||
| 2929 | // push the scope and evaluate the full predicate chain. | |||
| 2930 | bool Result; | |||
| 2931 | MatcherIndex = IsPredicateKnownToFail(MatcherTable, MatcherIndex, N, | |||
| 2932 | Result, *this, RecordedNodes); | |||
| 2933 | if (!Result) | |||
| 2934 | break; | |||
| 2935 | ||||
| 2936 | LLVM_DEBUG(do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("isel")) { dbgs() << " Skipped scope entry (due to false predicate) at " << "index " << MatcherIndexOfPredicate << ", continuing at " << FailIndex << "\n"; } } while (false) | |||
| 2937 | dbgs() << " Skipped scope entry (due to false predicate) at "do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("isel")) { dbgs() << " Skipped scope entry (due to false predicate) at " << "index " << MatcherIndexOfPredicate << ", continuing at " << FailIndex << "\n"; } } while (false) | |||
| 2938 | << "index " << MatcherIndexOfPredicate << ", continuing at "do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("isel")) { dbgs() << " Skipped scope entry (due to false predicate) at " << "index " << MatcherIndexOfPredicate << ", continuing at " << FailIndex << "\n"; } } while (false) | |||
| 2939 | << FailIndex << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("isel")) { dbgs() << " Skipped scope entry (due to false predicate) at " << "index " << MatcherIndexOfPredicate << ", continuing at " << FailIndex << "\n"; } } while (false); | |||
| 2940 | ++NumDAGIselRetries; | |||
| 2941 | ||||
| 2942 | // Otherwise, we know that this case of the Scope is guaranteed to fail, | |||
| 2943 | // move to the next case. | |||
| 2944 | MatcherIndex = FailIndex; | |||
| 2945 | } | |||
| 2946 | ||||
| 2947 | // If the whole scope failed to match, bail. | |||
| 2948 | if (FailIndex == 0) break; | |||
| 2949 | ||||
| 2950 | // Push a MatchScope which indicates where to go if the first child fails | |||
| 2951 | // to match. | |||
| 2952 | MatchScope NewEntry; | |||
| 2953 | NewEntry.FailIndex = FailIndex; | |||
| 2954 | NewEntry.NodeStack.append(NodeStack.begin(), NodeStack.end()); | |||
| 2955 | NewEntry.NumRecordedNodes = RecordedNodes.size(); | |||
| 2956 | NewEntry.NumMatchedMemRefs = MatchedMemRefs.size(); | |||
| 2957 | NewEntry.InputChain = InputChain; | |||
| 2958 | NewEntry.InputGlue = InputGlue; | |||
| 2959 | NewEntry.HasChainNodesMatched = !ChainNodesMatched.empty(); | |||
| 2960 | MatchScopes.push_back(NewEntry); | |||
| 2961 | continue; | |||
| 2962 | } | |||
| 2963 | case OPC_RecordNode: { | |||
| 2964 | // Remember this node, it may end up being an operand in the pattern. | |||
| 2965 | SDNode *Parent = nullptr; | |||
| 2966 | if (NodeStack.size() > 1) | |||
| 2967 | Parent = NodeStack[NodeStack.size()-2].getNode(); | |||
| 2968 | RecordedNodes.push_back(std::make_pair(N, Parent)); | |||
| 2969 | continue; | |||
| 2970 | } | |||
| 2971 | ||||
| 2972 | case OPC_RecordChild0: case OPC_RecordChild1: | |||
| 2973 | case OPC_RecordChild2: case OPC_RecordChild3: | |||
| 2974 | case OPC_RecordChild4: case OPC_RecordChild5: | |||
| 2975 | case OPC_RecordChild6: case OPC_RecordChild7: { | |||
| 2976 | unsigned ChildNo = Opcode-OPC_RecordChild0; | |||
| 2977 | if (ChildNo >= N.getNumOperands()) | |||
| 2978 | break; // Match fails if out of range child #. | |||
| 2979 | ||||
| 2980 | RecordedNodes.push_back(std::make_pair(N->getOperand(ChildNo), | |||
| 2981 | N.getNode())); | |||
| 2982 | continue; | |||
| 2983 | } | |||
| 2984 | case OPC_RecordMemRef: | |||
| 2985 | if (auto *MN = dyn_cast<MemSDNode>(N)) | |||
| 2986 | MatchedMemRefs.push_back(MN->getMemOperand()); | |||
| 2987 | else { | |||
| 2988 | LLVM_DEBUG(dbgs() << "Expected MemSDNode "; N->dump(CurDAG);do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("isel")) { dbgs() << "Expected MemSDNode "; N->dump (CurDAG); dbgs() << '\n'; } } while (false) | |||
| 2989 | dbgs() << '\n')do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("isel")) { dbgs() << "Expected MemSDNode "; N->dump (CurDAG); dbgs() << '\n'; } } while (false); | |||
| 2990 | } | |||
| 2991 | ||||
| 2992 | continue; | |||
| 2993 | ||||
| 2994 | case OPC_CaptureGlueInput: | |||
| 2995 | // If the current node has an input glue, capture it in InputGlue. | |||
| 2996 | if (N->getNumOperands() != 0 && | |||
| 2997 | N->getOperand(N->getNumOperands()-1).getValueType() == MVT::Glue) | |||
| 2998 | InputGlue = N->getOperand(N->getNumOperands()-1); | |||
| 2999 | continue; | |||
| 3000 | ||||
| 3001 | case OPC_MoveChild: { | |||
| 3002 | unsigned ChildNo = MatcherTable[MatcherIndex++]; | |||
| 3003 | if (ChildNo >= N.getNumOperands()) | |||
| 3004 | break; // Match fails if out of range child #. | |||
| 3005 | N = N.getOperand(ChildNo); | |||
| 3006 | NodeStack.push_back(N); | |||
| 3007 | continue; | |||
| 3008 | } | |||
| 3009 | ||||
| 3010 | case OPC_MoveChild0: case OPC_MoveChild1: | |||
| 3011 | case OPC_MoveChild2: case OPC_MoveChild3: | |||
| 3012 | case OPC_MoveChild4: case OPC_MoveChild5: | |||
| 3013 | case OPC_MoveChild6: case OPC_MoveChild7: { | |||
| 3014 | unsigned ChildNo = Opcode-OPC_MoveChild0; | |||
| 3015 | if (ChildNo >= N.getNumOperands()) | |||
| 3016 | break; // Match fails if out of range child #. | |||
| 3017 | N = N.getOperand(ChildNo); | |||
| 3018 | NodeStack.push_back(N); | |||
| 3019 | continue; | |||
| 3020 | } | |||
| 3021 | ||||
| 3022 | case OPC_MoveParent: | |||
| 3023 | // Pop the current node off the NodeStack. | |||
| 3024 | NodeStack.pop_back(); | |||
| 3025 | assert(!NodeStack.empty() && "Node stack imbalance!")((!NodeStack.empty() && "Node stack imbalance!") ? static_cast <void> (0) : __assert_fail ("!NodeStack.empty() && \"Node stack imbalance!\"" , "/build/llvm-toolchain-snapshot-10~++20200112100611+7fa5290d5bd/llvm/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp" , 3025, __PRETTY_FUNCTION__)); | |||
| 3026 | N = NodeStack.back(); | |||
| 3027 | continue; | |||
| 3028 | ||||
| 3029 | case OPC_CheckSame: | |||
| 3030 | if (!::CheckSame(MatcherTable, MatcherIndex, N, RecordedNodes)) break; | |||
| 3031 | continue; | |||
| 3032 | ||||
| 3033 | case OPC_CheckChild0Same: case OPC_CheckChild1Same: | |||
| 3034 | case OPC_CheckChild2Same: case OPC_CheckChild3Same: | |||
| 3035 | if (!::CheckChildSame(MatcherTable, MatcherIndex, N, RecordedNodes, | |||
| 3036 | Opcode-OPC_CheckChild0Same)) | |||
| 3037 | break; | |||
| 3038 | continue; | |||
| 3039 | ||||
| 3040 | case OPC_CheckPatternPredicate: | |||
| 3041 | if (!::CheckPatternPredicate(MatcherTable, MatcherIndex, *this)) break; | |||
| 3042 | continue; | |||
| 3043 | case OPC_CheckPredicate: | |||
| 3044 | if (!::CheckNodePredicate(MatcherTable, MatcherIndex, *this, | |||
| 3045 | N.getNode())) | |||
| 3046 | break; | |||
| 3047 | continue; | |||
| 3048 | case OPC_CheckPredicateWithOperands: { | |||
| 3049 | unsigned OpNum = MatcherTable[MatcherIndex++]; | |||
| 3050 | SmallVector<SDValue, 8> Operands; | |||
| 3051 | ||||
| 3052 | for (unsigned i = 0; i < OpNum; ++i) | |||
| 3053 | Operands.push_back(RecordedNodes[MatcherTable[MatcherIndex++]].first); | |||
| 3054 | ||||
| 3055 | unsigned PredNo = MatcherTable[MatcherIndex++]; | |||
| 3056 | if (!CheckNodePredicateWithOperands(N.getNode(), PredNo, Operands)) | |||
| 3057 | break; | |||
| 3058 | continue; | |||
| 3059 | } | |||
| 3060 | case OPC_CheckComplexPat: { | |||
| 3061 | unsigned CPNum = MatcherTable[MatcherIndex++]; | |||
| 3062 | unsigned RecNo = MatcherTable[MatcherIndex++]; | |||
| 3063 | assert(RecNo < RecordedNodes.size() && "Invalid CheckComplexPat")((RecNo < RecordedNodes.size() && "Invalid CheckComplexPat" ) ? static_cast<void> (0) : __assert_fail ("RecNo < RecordedNodes.size() && \"Invalid CheckComplexPat\"" , "/build/llvm-toolchain-snapshot-10~++20200112100611+7fa5290d5bd/llvm/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp" , 3063, __PRETTY_FUNCTION__)); | |||
| 3064 | ||||
| 3065 | // If target can modify DAG during matching, keep the matching state | |||
| 3066 | // consistent. | |||
| 3067 | std::unique_ptr<MatchStateUpdater> MSU; | |||
| 3068 | if (ComplexPatternFuncMutatesDAG()) | |||
| 3069 | MSU.reset(new MatchStateUpdater(*CurDAG, &NodeToMatch, RecordedNodes, | |||
| 3070 | MatchScopes)); | |||
| 3071 | ||||
| 3072 | if (!CheckComplexPattern(NodeToMatch, RecordedNodes[RecNo].second, | |||
| 3073 | RecordedNodes[RecNo].first, CPNum, | |||
| 3074 | RecordedNodes)) | |||
| 3075 | break; | |||
| 3076 | continue; | |||
| 3077 | } | |||
| 3078 | case OPC_CheckOpcode: | |||
| 3079 | if (!::CheckOpcode(MatcherTable, MatcherIndex, N.getNode())) break; | |||
| 3080 | continue; | |||
| 3081 | ||||
| 3082 | case OPC_CheckType: | |||
| 3083 | if (!::CheckType(MatcherTable, MatcherIndex, N, TLI, | |||
| 3084 | CurDAG->getDataLayout())) | |||
| 3085 | break; | |||
| 3086 | continue; | |||
| 3087 | ||||
| 3088 | case OPC_CheckTypeRes: { | |||
| 3089 | unsigned Res = MatcherTable[MatcherIndex++]; | |||
| 3090 | if (!::CheckType(MatcherTable, MatcherIndex, N.getValue(Res), TLI, | |||
| 3091 | CurDAG->getDataLayout())) | |||
| 3092 | break; | |||
| 3093 | continue; | |||
| 3094 | } | |||
| 3095 | ||||
| 3096 | case OPC_SwitchOpcode: { | |||
| 3097 | unsigned CurNodeOpcode = N.getOpcode(); | |||
| 3098 | unsigned SwitchStart = MatcherIndex-1; (void)SwitchStart; | |||
| 3099 | unsigned CaseSize; | |||
| 3100 | while (true) { | |||
| 3101 | // Get the size of this case. | |||
| 3102 | CaseSize = MatcherTable[MatcherIndex++]; | |||
| 3103 | if (CaseSize & 128) | |||
| 3104 | CaseSize = GetVBR(CaseSize, MatcherTable, MatcherIndex); | |||
| 3105 | if (CaseSize == 0) break; | |||
| 3106 | ||||
| 3107 | uint16_t Opc = MatcherTable[MatcherIndex++]; | |||
| 3108 | Opc |= (unsigned short)MatcherTable[MatcherIndex++] << 8; | |||
| 3109 | ||||
| 3110 | // If the opcode matches, then we will execute this case. | |||
| 3111 | if (CurNodeOpcode == Opc) | |||
| 3112 | break; | |||
| 3113 | ||||
| 3114 | // Otherwise, skip over this case. | |||
| 3115 | MatcherIndex += CaseSize; | |||
| 3116 | } | |||
| 3117 | ||||
| 3118 | // If no cases matched, bail out. | |||
| 3119 | if (CaseSize == 0) break; | |||
| 3120 | ||||
| 3121 | // Otherwise, execute the case we found. | |||
| 3122 | LLVM_DEBUG(dbgs() << " OpcodeSwitch from " << SwitchStart << " to "do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("isel")) { dbgs() << " OpcodeSwitch from " << SwitchStart << " to " << MatcherIndex << "\n"; } } while (false) | |||
| 3123 | << MatcherIndex << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("isel")) { dbgs() << " OpcodeSwitch from " << SwitchStart << " to " << MatcherIndex << "\n"; } } while (false); | |||
| 3124 | continue; | |||
| 3125 | } | |||
| 3126 | ||||
| 3127 | case OPC_SwitchType: { | |||
| 3128 | MVT CurNodeVT = N.getSimpleValueType(); | |||
| 3129 | unsigned SwitchStart = MatcherIndex-1; (void)SwitchStart; | |||
| 3130 | unsigned CaseSize; | |||
| 3131 | while (true) { | |||
| 3132 | // Get the size of this case. | |||
| 3133 | CaseSize = MatcherTable[MatcherIndex++]; | |||
| 3134 | if (CaseSize & 128) | |||
| 3135 | CaseSize = GetVBR(CaseSize, MatcherTable, MatcherIndex); | |||
| 3136 | if (CaseSize == 0) break; | |||
| 3137 | ||||
| 3138 | MVT CaseVT = (MVT::SimpleValueType)MatcherTable[MatcherIndex++]; | |||
| 3139 | if (CaseVT == MVT::iPTR) | |||
| 3140 | CaseVT = TLI->getPointerTy(CurDAG->getDataLayout()); | |||
| 3141 | ||||
| 3142 | // If the VT matches, then we will execute this case. | |||
| 3143 | if (CurNodeVT == CaseVT) | |||
| 3144 | break; | |||
| 3145 | ||||
| 3146 | // Otherwise, skip over this case. | |||
| 3147 | MatcherIndex += CaseSize; | |||
| 3148 | } | |||
| 3149 | ||||
| 3150 | // If no cases matched, bail out. | |||
| 3151 | if (CaseSize == 0) break; | |||
| 3152 | ||||
| 3153 | // Otherwise, execute the case we found. | |||
| 3154 | LLVM_DEBUG(dbgs() << " TypeSwitch[" << EVT(CurNodeVT).getEVTString()do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("isel")) { dbgs() << " TypeSwitch[" << EVT(CurNodeVT ).getEVTString() << "] from " << SwitchStart << " to " << MatcherIndex << '\n'; } } while (false ) | |||
| 3155 | << "] from " << SwitchStart << " to " << MatcherIndexdo { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("isel")) { dbgs() << " TypeSwitch[" << EVT(CurNodeVT ).getEVTString() << "] from " << SwitchStart << " to " << MatcherIndex << '\n'; } } while (false ) | |||
| 3156 | << '\n')do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("isel")) { dbgs() << " TypeSwitch[" << EVT(CurNodeVT ).getEVTString() << "] from " << SwitchStart << " to " << MatcherIndex << '\n'; } } while (false ); | |||
| 3157 | continue; | |||
| 3158 | } | |||
| 3159 | case OPC_CheckChild0Type: case OPC_CheckChild1Type: | |||
| 3160 | case OPC_CheckChild2Type: case OPC_CheckChild3Type: | |||
| 3161 | case OPC_CheckChild4Type: case OPC_CheckChild5Type: | |||
| 3162 | case OPC_CheckChild6Type: case OPC_CheckChild7Type: | |||
| 3163 | if (!::CheckChildType(MatcherTable, MatcherIndex, N, TLI, | |||
| 3164 | CurDAG->getDataLayout(), | |||
| 3165 | Opcode - OPC_CheckChild0Type)) | |||
| 3166 | break; | |||
| 3167 | continue; | |||
| 3168 | case OPC_CheckCondCode: | |||
| 3169 | if (!::CheckCondCode(MatcherTable, MatcherIndex, N)) break; | |||
| 3170 | continue; | |||
| 3171 | case OPC_CheckChild2CondCode: | |||
| 3172 | if (!::CheckChild2CondCode(MatcherTable, MatcherIndex, N)) break; | |||
| 3173 | continue; | |||
| 3174 | case OPC_CheckValueType: | |||
| 3175 | if (!::CheckValueType(MatcherTable, MatcherIndex, N, TLI, | |||
| 3176 | CurDAG->getDataLayout())) | |||
| 3177 | break; | |||
| 3178 | continue; | |||
| 3179 | case OPC_CheckInteger: | |||
| 3180 | if (!::CheckInteger(MatcherTable, MatcherIndex, N)) break; | |||
| 3181 | continue; | |||
| 3182 | case OPC_CheckChild0Integer: case OPC_CheckChild1Integer: | |||
| 3183 | case OPC_CheckChild2Integer: case OPC_CheckChild3Integer: | |||
| 3184 | case OPC_CheckChild4Integer: | |||
| 3185 | if (!::CheckChildInteger(MatcherTable, MatcherIndex, N, | |||
| 3186 | Opcode-OPC_CheckChild0Integer)) break; | |||
| 3187 | continue; | |||
| 3188 | case OPC_CheckAndImm: | |||
| 3189 | if (!::CheckAndImm(MatcherTable, MatcherIndex, N, *this)) break; | |||
| 3190 | continue; | |||
| 3191 | case OPC_CheckOrImm: | |||
| 3192 | if (!::CheckOrImm(MatcherTable, MatcherIndex, N, *this)) break; | |||
| 3193 | continue; | |||
| 3194 | case OPC_CheckImmAllOnesV: | |||
| 3195 | if (!ISD::isBuildVectorAllOnes(N.getNode())) break; | |||
| 3196 | continue; | |||
| 3197 | case OPC_CheckImmAllZerosV: | |||
| 3198 | if (!ISD::isBuildVectorAllZeros(N.getNode())) break; | |||
| 3199 | continue; | |||
| 3200 | ||||
| 3201 | case OPC_CheckFoldableChainNode: { | |||
| 3202 | assert(NodeStack.size() != 1 && "No parent node")((NodeStack.size() != 1 && "No parent node") ? static_cast <void> (0) : __assert_fail ("NodeStack.size() != 1 && \"No parent node\"" , "/build/llvm-toolchain-snapshot-10~++20200112100611+7fa5290d5bd/llvm/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp" , 3202, __PRETTY_FUNCTION__)); | |||
| 3203 | // Verify that all intermediate nodes between the root and this one have | |||
| 3204 | // a single use (ignoring chains, which are handled in UpdateChains). | |||
| 3205 | bool HasMultipleUses = false; | |||
| 3206 | for (unsigned i = 1, e = NodeStack.size()-1; i != e; ++i) { | |||
| 3207 | unsigned NNonChainUses = 0; | |||
| 3208 | SDNode *NS = NodeStack[i].getNode(); | |||
| 3209 | for (auto UI = NS->use_begin(), UE = NS->use_end(); UI != UE; ++UI) | |||
| 3210 | if (UI.getUse().getValueType() != MVT::Other) | |||
| 3211 | if (++NNonChainUses > 1) { | |||
| 3212 | HasMultipleUses = true; | |||
| 3213 | break; | |||
| 3214 | } | |||
| 3215 | if (HasMultipleUses) break; | |||
| 3216 | } | |||
| 3217 | if (HasMultipleUses) break; | |||
| 3218 | ||||
| 3219 | // Check to see that the target thinks this is profitable to fold and that | |||
| 3220 | // we can fold it without inducing cycles in the graph. | |||
| 3221 | if (!IsProfitableToFold(N, NodeStack[NodeStack.size()-2].getNode(), | |||
| 3222 | NodeToMatch) || | |||
| 3223 | !IsLegalToFold(N, NodeStack[NodeStack.size()-2].getNode(), | |||
| 3224 | NodeToMatch, OptLevel, | |||
| 3225 | true/*We validate our own chains*/)) | |||
| 3226 | break; | |||
| 3227 | ||||
| 3228 | continue; | |||
| 3229 | } | |||
| 3230 | case OPC_EmitInteger: { | |||
| 3231 | MVT::SimpleValueType VT = | |||
| 3232 | (MVT::SimpleValueType)MatcherTable[MatcherIndex++]; | |||
| 3233 | int64_t Val = MatcherTable[MatcherIndex++]; | |||
| 3234 | if (Val & 128) | |||
| 3235 | Val = GetVBR(Val, MatcherTable, MatcherIndex); | |||
| 3236 | RecordedNodes.push_back(std::pair<SDValue, SDNode*>( | |||
| 3237 | CurDAG->getTargetConstant(Val, SDLoc(NodeToMatch), | |||
| 3238 | VT), nullptr)); | |||
| 3239 | continue; | |||
| 3240 | } | |||
| 3241 | case OPC_EmitRegister: { | |||
| 3242 | MVT::SimpleValueType VT = | |||
| 3243 | (MVT::SimpleValueType)MatcherTable[MatcherIndex++]; | |||
| 3244 | unsigned RegNo = MatcherTable[MatcherIndex++]; | |||
| 3245 | RecordedNodes.push_back(std::pair<SDValue, SDNode*>( | |||
| 3246 | CurDAG->getRegister(RegNo, VT), nullptr)); | |||
| 3247 | continue; | |||
| 3248 | } | |||
| 3249 | case OPC_EmitRegister2: { | |||
| 3250 | // For targets w/ more than 256 register names, the register enum | |||
| 3251 | // values are stored in two bytes in the matcher table (just like | |||
| 3252 | // opcodes). | |||
| 3253 | MVT::SimpleValueType VT = | |||
| 3254 | (MVT::SimpleValueType)MatcherTable[MatcherIndex++]; | |||
| 3255 | unsigned RegNo = MatcherTable[MatcherIndex++]; | |||
| 3256 | RegNo |= MatcherTable[MatcherIndex++] << 8; | |||
| 3257 | RecordedNodes.push_back(std::pair<SDValue, SDNode*>( | |||
| 3258 | CurDAG->getRegister(RegNo, VT), nullptr)); | |||
| 3259 | continue; | |||
| 3260 | } | |||
| 3261 | ||||
| 3262 | case OPC_EmitConvertToTarget: { | |||
| 3263 | // Convert from IMM/FPIMM to target version. | |||
| 3264 | unsigned RecNo = MatcherTable[MatcherIndex++]; | |||
| 3265 | assert(RecNo < RecordedNodes.size() && "Invalid EmitConvertToTarget")((RecNo < RecordedNodes.size() && "Invalid EmitConvertToTarget" ) ? static_cast<void> (0) : __assert_fail ("RecNo < RecordedNodes.size() && \"Invalid EmitConvertToTarget\"" , "/build/llvm-toolchain-snapshot-10~++20200112100611+7fa5290d5bd/llvm/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp" , 3265, __PRETTY_FUNCTION__)); | |||
| 3266 | SDValue Imm = RecordedNodes[RecNo].first; | |||
| 3267 | ||||
| 3268 | if (Imm->getOpcode() == ISD::Constant) { | |||
| 3269 | const ConstantInt *Val=cast<ConstantSDNode>(Imm)->getConstantIntValue(); | |||
| 3270 | Imm = CurDAG->getTargetConstant(*Val, SDLoc(NodeToMatch), | |||
| 3271 | Imm.getValueType()); | |||
| 3272 | } else if (Imm->getOpcode() == ISD::ConstantFP) { | |||
| 3273 | const ConstantFP *Val=cast<ConstantFPSDNode>(Imm)->getConstantFPValue(); | |||
| 3274 | Imm = CurDAG->getTargetConstantFP(*Val, SDLoc(NodeToMatch), | |||
| 3275 | Imm.getValueType()); | |||
| 3276 | } | |||
| 3277 | ||||
| 3278 | RecordedNodes.push_back(std::make_pair(Imm, RecordedNodes[RecNo].second)); | |||
| 3279 | continue; | |||
| 3280 | } | |||
| 3281 | ||||
| 3282 | case OPC_EmitMergeInputChains1_0: // OPC_EmitMergeInputChains, 1, 0 | |||
| 3283 | case OPC_EmitMergeInputChains1_1: // OPC_EmitMergeInputChains, 1, 1 | |||
| 3284 | case OPC_EmitMergeInputChains1_2: { // OPC_EmitMergeInputChains, 1, 2 | |||
| 3285 | // These are space-optimized forms of OPC_EmitMergeInputChains. | |||
| 3286 | assert(!InputChain.getNode() &&((!InputChain.getNode() && "EmitMergeInputChains should be the first chain producing node" ) ? static_cast<void> (0) : __assert_fail ("!InputChain.getNode() && \"EmitMergeInputChains should be the first chain producing node\"" , "/build/llvm-toolchain-snapshot-10~++20200112100611+7fa5290d5bd/llvm/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp" , 3287, __PRETTY_FUNCTION__)) | |||
| 3287 | "EmitMergeInputChains should be the first chain producing node")((!InputChain.getNode() && "EmitMergeInputChains should be the first chain producing node" ) ? static_cast<void> (0) : __assert_fail ("!InputChain.getNode() && \"EmitMergeInputChains should be the first chain producing node\"" , "/build/llvm-toolchain-snapshot-10~++20200112100611+7fa5290d5bd/llvm/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp" , 3287, __PRETTY_FUNCTION__)); | |||
| 3288 | assert(ChainNodesMatched.empty() &&((ChainNodesMatched.empty() && "Should only have one EmitMergeInputChains per match" ) ? static_cast<void> (0) : __assert_fail ("ChainNodesMatched.empty() && \"Should only have one EmitMergeInputChains per match\"" , "/build/llvm-toolchain-snapshot-10~++20200112100611+7fa5290d5bd/llvm/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp" , 3289, __PRETTY_FUNCTION__)) | |||
| 3289 | "Should only have one EmitMergeInputChains per match")((ChainNodesMatched.empty() && "Should only have one EmitMergeInputChains per match" ) ? static_cast<void> (0) : __assert_fail ("ChainNodesMatched.empty() && \"Should only have one EmitMergeInputChains per match\"" , "/build/llvm-toolchain-snapshot-10~++20200112100611+7fa5290d5bd/llvm/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp" , 3289, __PRETTY_FUNCTION__)); | |||
| 3290 | ||||
| 3291 | // Read all of the chained nodes. | |||
| 3292 | unsigned RecNo = Opcode - OPC_EmitMergeInputChains1_0; | |||
| 3293 | assert(RecNo < RecordedNodes.size() && "Invalid EmitMergeInputChains")((RecNo < RecordedNodes.size() && "Invalid EmitMergeInputChains" ) ? static_cast<void> (0) : __assert_fail ("RecNo < RecordedNodes.size() && \"Invalid EmitMergeInputChains\"" , "/build/llvm-toolchain-snapshot-10~++20200112100611+7fa5290d5bd/llvm/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp" , 3293, __PRETTY_FUNCTION__)); | |||
| 3294 | ChainNodesMatched.push_back(RecordedNodes[RecNo].first.getNode()); | |||
| 3295 | ||||
| 3296 | // FIXME: What if other value results of the node have uses not matched | |||
| 3297 | // by this pattern? | |||
| 3298 | if (ChainNodesMatched.back() != NodeToMatch && | |||
| 3299 | !RecordedNodes[RecNo].first.hasOneUse()) { | |||
| 3300 | ChainNodesMatched.clear(); | |||
| 3301 | break; | |||
| 3302 | } | |||
| 3303 | ||||
| 3304 | // Merge the input chains if they are not intra-pattern references. | |||
| 3305 | InputChain = HandleMergeInputChains(ChainNodesMatched, CurDAG); | |||
| 3306 | ||||
| 3307 | if (!InputChain.getNode()) | |||
| 3308 | break; // Failed to merge. | |||
| 3309 | continue; | |||
| 3310 | } | |||
| 3311 | ||||
| 3312 | case OPC_EmitMergeInputChains: { | |||
| 3313 | assert(!InputChain.getNode() &&((!InputChain.getNode() && "EmitMergeInputChains should be the first chain producing node" ) ? static_cast<void> (0) : __assert_fail ("!InputChain.getNode() && \"EmitMergeInputChains should be the first chain producing node\"" , "/build/llvm-toolchain-snapshot-10~++20200112100611+7fa5290d5bd/llvm/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp" , 3314, __PRETTY_FUNCTION__)) | |||
| 3314 | "EmitMergeInputChains should be the first chain producing node")((!InputChain.getNode() && "EmitMergeInputChains should be the first chain producing node" ) ? static_cast<void> (0) : __assert_fail ("!InputChain.getNode() && \"EmitMergeInputChains should be the first chain producing node\"" , "/build/llvm-toolchain-snapshot-10~++20200112100611+7fa5290d5bd/llvm/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp" , 3314, __PRETTY_FUNCTION__)); | |||
| 3315 | // This node gets a list of nodes we matched in the input that have | |||
| 3316 | // chains. We want to token factor all of the input chains to these nodes | |||
| 3317 | // together. However, if any of the input chains is actually one of the | |||
| 3318 | // nodes matched in this pattern, then we have an intra-match reference. | |||
| 3319 | // Ignore these because the newly token factored chain should not refer to | |||
| 3320 | // the old nodes. | |||
| 3321 | unsigned NumChains = MatcherTable[MatcherIndex++]; | |||
| 3322 | assert(NumChains != 0 && "Can't TF zero chains")((NumChains != 0 && "Can't TF zero chains") ? static_cast <void> (0) : __assert_fail ("NumChains != 0 && \"Can't TF zero chains\"" , "/build/llvm-toolchain-snapshot-10~++20200112100611+7fa5290d5bd/llvm/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp" , 3322, __PRETTY_FUNCTION__)); | |||
| 3323 | ||||
| 3324 | assert(ChainNodesMatched.empty() &&((ChainNodesMatched.empty() && "Should only have one EmitMergeInputChains per match" ) ? static_cast<void> (0) : __assert_fail ("ChainNodesMatched.empty() && \"Should only have one EmitMergeInputChains per match\"" , "/build/llvm-toolchain-snapshot-10~++20200112100611+7fa5290d5bd/llvm/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp" , 3325, __PRETTY_FUNCTION__)) | |||
| 3325 | "Should only have one EmitMergeInputChains per match")((ChainNodesMatched.empty() && "Should only have one EmitMergeInputChains per match" ) ? static_cast<void> (0) : __assert_fail ("ChainNodesMatched.empty() && \"Should only have one EmitMergeInputChains per match\"" , "/build/llvm-toolchain-snapshot-10~++20200112100611+7fa5290d5bd/llvm/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp" , 3325, __PRETTY_FUNCTION__)); | |||
| 3326 | ||||
| 3327 | // Read all of the chained nodes. | |||
| 3328 | for (unsigned i = 0; i != NumChains; ++i) { | |||
| 3329 | unsigned RecNo = MatcherTable[MatcherIndex++]; | |||
| 3330 | assert(RecNo < RecordedNodes.size() && "Invalid EmitMergeInputChains")((RecNo < RecordedNodes.size() && "Invalid EmitMergeInputChains" ) ? static_cast<void> (0) : __assert_fail ("RecNo < RecordedNodes.size() && \"Invalid EmitMergeInputChains\"" , "/build/llvm-toolchain-snapshot-10~++20200112100611+7fa5290d5bd/llvm/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp" , 3330, __PRETTY_FUNCTION__)); | |||
| 3331 | ChainNodesMatched.push_back(RecordedNodes[RecNo].first.getNode()); | |||
| 3332 | ||||
| 3333 | // FIXME: What if other value results of the node have uses not matched | |||
| 3334 | // by this pattern? | |||
| 3335 | if (ChainNodesMatched.back() != NodeToMatch && | |||
| 3336 | !RecordedNodes[RecNo].first.hasOneUse()) { | |||
| 3337 | ChainNodesMatched.clear(); | |||
| 3338 | break; | |||
| 3339 | } | |||
| 3340 | } | |||
| 3341 | ||||
| 3342 | // If the inner loop broke out, the match fails. | |||
| 3343 | if (ChainNodesMatched.empty()) | |||
| 3344 | break; | |||
| 3345 | ||||
| 3346 | // Merge the input chains if they are not intra-pattern references. | |||
| 3347 | InputChain = HandleMergeInputChains(ChainNodesMatched, CurDAG); | |||
| 3348 | ||||
| 3349 | if (!InputChain.getNode()) | |||
| 3350 | break; // Failed to merge. | |||
| 3351 | ||||
| 3352 | continue; | |||
| 3353 | } | |||
| 3354 | ||||
| 3355 | case OPC_EmitCopyToReg: | |||
| 3356 | case OPC_EmitCopyToReg2: { | |||
| 3357 | unsigned RecNo = MatcherTable[MatcherIndex++]; | |||
| 3358 | assert(RecNo < RecordedNodes.size() && "Invalid EmitCopyToReg")((RecNo < RecordedNodes.size() && "Invalid EmitCopyToReg" ) ? static_cast<void> (0) : __assert_fail ("RecNo < RecordedNodes.size() && \"Invalid EmitCopyToReg\"" , "/build/llvm-toolchain-snapshot-10~++20200112100611+7fa5290d5bd/llvm/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp" , 3358, __PRETTY_FUNCTION__)); | |||
| 3359 | unsigned DestPhysReg = MatcherTable[MatcherIndex++]; | |||
| 3360 | if (Opcode == OPC_EmitCopyToReg2) | |||
| 3361 | DestPhysReg |= MatcherTable[MatcherIndex++] << 8; | |||
| 3362 | ||||
| 3363 | if (!InputChain.getNode()) | |||
| 3364 | InputChain = CurDAG->getEntryNode(); | |||
| 3365 | ||||
| 3366 | InputChain = CurDAG->getCopyToReg(InputChain, SDLoc(NodeToMatch), | |||
| 3367 | DestPhysReg, RecordedNodes[RecNo].first, | |||
| 3368 | InputGlue); | |||
| 3369 | ||||
| 3370 | InputGlue = InputChain.getValue(1); | |||
| 3371 | continue; | |||
| 3372 | } | |||
| 3373 | ||||
| 3374 | case OPC_EmitNodeXForm: { | |||
| 3375 | unsigned XFormNo = MatcherTable[MatcherIndex++]; | |||
| 3376 | unsigned RecNo = MatcherTable[MatcherIndex++]; | |||
| 3377 | assert(RecNo < RecordedNodes.size() && "Invalid EmitNodeXForm")((RecNo < RecordedNodes.size() && "Invalid EmitNodeXForm" ) ? static_cast<void> (0) : __assert_fail ("RecNo < RecordedNodes.size() && \"Invalid EmitNodeXForm\"" , "/build/llvm-toolchain-snapshot-10~++20200112100611+7fa5290d5bd/llvm/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp" , 3377, __PRETTY_FUNCTION__)); | |||
| 3378 | SDValue Res = RunSDNodeXForm(RecordedNodes[RecNo].first, XFormNo); | |||
| 3379 | RecordedNodes.push_back(std::pair<SDValue,SDNode*>(Res, nullptr)); | |||
| 3380 | continue; | |||
| 3381 | } | |||
| 3382 | case OPC_Coverage: { | |||
| 3383 | // This is emitted right before MorphNode/EmitNode. | |||
| 3384 | // So it should be safe to assume that this node has been selected | |||
| 3385 | unsigned index = MatcherTable[MatcherIndex++]; | |||
| 3386 | index |= (MatcherTable[MatcherIndex++] << 8); | |||
| 3387 | dbgs() << "COVERED: " << getPatternForIndex(index) << "\n"; | |||
| 3388 | dbgs() << "INCLUDED: " << getIncludePathForIndex(index) << "\n"; | |||
| 3389 | continue; | |||
| 3390 | } | |||
| 3391 | ||||
| 3392 | case OPC_EmitNode: case OPC_MorphNodeTo: | |||
| 3393 | case OPC_EmitNode0: case OPC_EmitNode1: case OPC_EmitNode2: | |||
| 3394 | case OPC_MorphNodeTo0: case OPC_MorphNodeTo1: case OPC_MorphNodeTo2: { | |||
| 3395 | uint16_t TargetOpc = MatcherTable[MatcherIndex++]; | |||
| 3396 | TargetOpc |= (unsigned short)MatcherTable[MatcherIndex++] << 8; | |||
| 3397 | unsigned EmitNodeInfo = MatcherTable[MatcherIndex++]; | |||
| 3398 | // Get the result VT list. | |||
| 3399 | unsigned NumVTs; | |||
| 3400 | // If this is one of the compressed forms, get the number of VTs based | |||
| 3401 | // on the Opcode. Otherwise read the next byte from the table. | |||
| 3402 | if (Opcode >= OPC_MorphNodeTo0 && Opcode <= OPC_MorphNodeTo2) | |||
| 3403 | NumVTs = Opcode - OPC_MorphNodeTo0; | |||
| 3404 | else if (Opcode >= OPC_EmitNode0 && Opcode <= OPC_EmitNode2) | |||
| 3405 | NumVTs = Opcode - OPC_EmitNode0; | |||
| 3406 | else | |||
| 3407 | NumVTs = MatcherTable[MatcherIndex++]; | |||
| 3408 | SmallVector<EVT, 4> VTs; | |||
| 3409 | for (unsigned i = 0; i != NumVTs; ++i) { | |||
| 3410 | MVT::SimpleValueType VT = | |||
| 3411 | (MVT::SimpleValueType)MatcherTable[MatcherIndex++]; | |||
| 3412 | if (VT == MVT::iPTR) | |||
| 3413 | VT = TLI->getPointerTy(CurDAG->getDataLayout()).SimpleTy; | |||
| 3414 | VTs.push_back(VT); | |||
| 3415 | } | |||
| 3416 | ||||
| 3417 | if (EmitNodeInfo & OPFL_Chain) | |||
| 3418 | VTs.push_back(MVT::Other); | |||
| 3419 | if (EmitNodeInfo & OPFL_GlueOutput) | |||
| 3420 | VTs.push_back(MVT::Glue); | |||
| 3421 | ||||
| 3422 | // This is hot code, so optimize the two most common cases of 1 and 2 | |||
| 3423 | // results. | |||
| 3424 | SDVTList VTList; | |||
| 3425 | if (VTs.size() == 1) | |||
| 3426 | VTList = CurDAG->getVTList(VTs[0]); | |||
| 3427 | else if (VTs.size() == 2) | |||
| 3428 | VTList = CurDAG->getVTList(VTs[0], VTs[1]); | |||
| 3429 | else | |||
| 3430 | VTList = CurDAG->getVTList(VTs); | |||
| 3431 | ||||
| 3432 | // Get the operand list. | |||
| 3433 | unsigned NumOps = MatcherTable[MatcherIndex++]; | |||
| 3434 | SmallVector<SDValue, 8> Ops; | |||
| 3435 | for (unsigned i = 0; i != NumOps; ++i) { | |||
| 3436 | unsigned RecNo = MatcherTable[MatcherIndex++]; | |||
| 3437 | if (RecNo & 128) | |||
| 3438 | RecNo = GetVBR(RecNo, MatcherTable, MatcherIndex); | |||
| 3439 | ||||
| 3440 | assert(RecNo < RecordedNodes.size() && "Invalid EmitNode")((RecNo < RecordedNodes.size() && "Invalid EmitNode" ) ? static_cast<void> (0) : __assert_fail ("RecNo < RecordedNodes.size() && \"Invalid EmitNode\"" , "/build/llvm-toolchain-snapshot-10~++20200112100611+7fa5290d5bd/llvm/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp" , 3440, __PRETTY_FUNCTION__)); | |||
| 3441 | Ops.push_back(RecordedNodes[RecNo].first); | |||
| 3442 | } | |||
| 3443 | ||||
| 3444 | // If there are variadic operands to add, handle them now. | |||
| 3445 | if (EmitNodeInfo & OPFL_VariadicInfo) { | |||
| 3446 | // Determine the start index to copy from. | |||
| 3447 | unsigned FirstOpToCopy = getNumFixedFromVariadicInfo(EmitNodeInfo); | |||
| 3448 | FirstOpToCopy += (EmitNodeInfo & OPFL_Chain) ? 1 : 0; | |||
| 3449 | assert(NodeToMatch->getNumOperands() >= FirstOpToCopy &&((NodeToMatch->getNumOperands() >= FirstOpToCopy && "Invalid variadic node") ? static_cast<void> (0) : __assert_fail ("NodeToMatch->getNumOperands() >= FirstOpToCopy && \"Invalid variadic node\"" , "/build/llvm-toolchain-snapshot-10~++20200112100611+7fa5290d5bd/llvm/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp" , 3450, __PRETTY_FUNCTION__)) | |||
| 3450 | "Invalid variadic node")((NodeToMatch->getNumOperands() >= FirstOpToCopy && "Invalid variadic node") ? static_cast<void> (0) : __assert_fail ("NodeToMatch->getNumOperands() >= FirstOpToCopy && \"Invalid variadic node\"" , "/build/llvm-toolchain-snapshot-10~++20200112100611+7fa5290d5bd/llvm/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp" , 3450, __PRETTY_FUNCTION__)); | |||
| 3451 | // Copy all of the variadic operands, not including a potential glue | |||
| 3452 | // input. | |||
| 3453 | for (unsigned i = FirstOpToCopy, e = NodeToMatch->getNumOperands(); | |||
| 3454 | i != e; ++i) { | |||
| 3455 | SDValue V = NodeToMatch->getOperand(i); | |||
| 3456 | if (V.getValueType() == MVT::Glue) break; | |||
| 3457 | Ops.push_back(V); | |||
| 3458 | } | |||
| 3459 | } | |||
| 3460 | ||||
| 3461 | // If this has chain/glue inputs, add them. | |||
| 3462 | if (EmitNodeInfo & OPFL_Chain) | |||
| 3463 | Ops.push_back(InputChain); | |||
| 3464 | if ((EmitNodeInfo & OPFL_GlueInput) && InputGlue.getNode() != nullptr) | |||
| 3465 | Ops.push_back(InputGlue); | |||
| 3466 | ||||
| 3467 | // Check whether any matched node could raise an FP exception. Since all | |||
| 3468 | // such nodes must have a chain, it suffices to check ChainNodesMatched. | |||
| 3469 | // We need to perform this check before potentially modifying one of the | |||
| 3470 | // nodes via MorphNode. | |||
| 3471 | bool MayRaiseFPException = false; | |||
| 3472 | for (auto *N : ChainNodesMatched) | |||
| 3473 | if (mayRaiseFPException(N) && !N->getFlags().hasNoFPExcept()) { | |||
| 3474 | MayRaiseFPException = true; | |||
| 3475 | break; | |||
| 3476 | } | |||
| 3477 | ||||
| 3478 | // Create the node. | |||
| 3479 | MachineSDNode *Res = nullptr; | |||
| 3480 | bool IsMorphNodeTo = Opcode == OPC_MorphNodeTo || | |||
| 3481 | (Opcode >= OPC_MorphNodeTo0 && Opcode <= OPC_MorphNodeTo2); | |||
| 3482 | if (!IsMorphNodeTo) { | |||
| 3483 | // If this is a normal EmitNode command, just create the new node and | |||
| 3484 | // add the results to the RecordedNodes list. | |||
| 3485 | Res = CurDAG->getMachineNode(TargetOpc, SDLoc(NodeToMatch), | |||
| 3486 | VTList, Ops); | |||
| 3487 | ||||
| 3488 | // Add all the non-glue/non-chain results to the RecordedNodes list. | |||
| 3489 | for (unsigned i = 0, e = VTs.size(); i != e; ++i) { | |||
| 3490 | if (VTs[i] == MVT::Other || VTs[i] == MVT::Glue) break; | |||
| 3491 | RecordedNodes.push_back(std::pair<SDValue,SDNode*>(SDValue(Res, i), | |||
| 3492 | nullptr)); | |||
| 3493 | } | |||
| 3494 | } else { | |||
| 3495 | assert(NodeToMatch->getOpcode() != ISD::DELETED_NODE &&((NodeToMatch->getOpcode() != ISD::DELETED_NODE && "NodeToMatch was removed partway through selection") ? static_cast <void> (0) : __assert_fail ("NodeToMatch->getOpcode() != ISD::DELETED_NODE && \"NodeToMatch was removed partway through selection\"" , "/build/llvm-toolchain-snapshot-10~++20200112100611+7fa5290d5bd/llvm/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp" , 3496, __PRETTY_FUNCTION__)) | |||
| 3496 | "NodeToMatch was removed partway through selection")((NodeToMatch->getOpcode() != ISD::DELETED_NODE && "NodeToMatch was removed partway through selection") ? static_cast <void> (0) : __assert_fail ("NodeToMatch->getOpcode() != ISD::DELETED_NODE && \"NodeToMatch was removed partway through selection\"" , "/build/llvm-toolchain-snapshot-10~++20200112100611+7fa5290d5bd/llvm/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp" , 3496, __PRETTY_FUNCTION__)); | |||
| 3497 | SelectionDAG::DAGNodeDeletedListener NDL(*CurDAG, [&](SDNode *N, | |||
| 3498 | SDNode *E) { | |||
| 3499 | CurDAG->salvageDebugInfo(*N); | |||
| 3500 | auto &Chain = ChainNodesMatched; | |||
| 3501 | assert((!E || !is_contained(Chain, N)) &&(((!E || !is_contained(Chain, N)) && "Chain node replaced during MorphNode" ) ? static_cast<void> (0) : __assert_fail ("(!E || !is_contained(Chain, N)) && \"Chain node replaced during MorphNode\"" , "/build/llvm-toolchain-snapshot-10~++20200112100611+7fa5290d5bd/llvm/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp" , 3502, __PRETTY_FUNCTION__)) | |||
| 3502 | "Chain node replaced during MorphNode")(((!E || !is_contained(Chain, N)) && "Chain node replaced during MorphNode" ) ? static_cast<void> (0) : __assert_fail ("(!E || !is_contained(Chain, N)) && \"Chain node replaced during MorphNode\"" , "/build/llvm-toolchain-snapshot-10~++20200112100611+7fa5290d5bd/llvm/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp" , 3502, __PRETTY_FUNCTION__)); | |||
| 3503 | Chain.erase(std::remove(Chain.begin(), Chain.end(), N), Chain.end()); | |||
| 3504 | }); | |||
| 3505 | Res = cast<MachineSDNode>(MorphNode(NodeToMatch, TargetOpc, VTList, | |||
| 3506 | Ops, EmitNodeInfo)); | |||
| 3507 | } | |||
| 3508 | ||||
| 3509 | // Set the NoFPExcept flag when no original matched node could | |||
| 3510 | // raise an FP exception, but the new node potentially might. | |||
| 3511 | if (!MayRaiseFPException && mayRaiseFPException(Res)) { | |||
| 3512 | SDNodeFlags Flags = Res->getFlags(); | |||
| 3513 | Flags.setNoFPExcept(true); | |||
| 3514 | Res->setFlags(Flags); | |||
| 3515 | } | |||
| 3516 | ||||
| 3517 | // If the node had chain/glue results, update our notion of the current | |||
| 3518 | // chain and glue. | |||
| 3519 | if (EmitNodeInfo & OPFL_GlueOutput) { | |||
| 3520 | InputGlue = SDValue(Res, VTs.size()-1); | |||
| 3521 | if (EmitNodeInfo & OPFL_Chain) | |||
| 3522 | InputChain = SDValue(Res, VTs.size()-2); | |||
| 3523 | } else if (EmitNodeInfo & OPFL_Chain) | |||
| 3524 | InputChain = SDValue(Res, VTs.size()-1); | |||
| 3525 | ||||
| 3526 | // If the OPFL_MemRefs glue is set on this node, slap all of the | |||
| 3527 | // accumulated memrefs onto it. | |||
| 3528 | // | |||
| 3529 | // FIXME: This is vastly incorrect for patterns with multiple outputs | |||
| 3530 | // instructions that access memory and for ComplexPatterns that match | |||
| 3531 | // loads. | |||
| 3532 | if (EmitNodeInfo & OPFL_MemRefs) { | |||
| 3533 | // Only attach load or store memory operands if the generated | |||
| 3534 | // instruction may load or store. | |||
| 3535 | const MCInstrDesc &MCID = TII->get(TargetOpc); | |||
| 3536 | bool mayLoad = MCID.mayLoad(); | |||
| 3537 | bool mayStore = MCID.mayStore(); | |||
| 3538 | ||||
| 3539 | // We expect to have relatively few of these so just filter them into a | |||
| 3540 | // temporary buffer so that we can easily add them to the instruction. | |||
| 3541 | SmallVector<MachineMemOperand *, 4> FilteredMemRefs; | |||
| 3542 | for (MachineMemOperand *MMO : MatchedMemRefs) { | |||
| 3543 | if (MMO->isLoad()) { | |||
| 3544 | if (mayLoad) | |||
| 3545 | FilteredMemRefs.push_back(MMO); | |||
| 3546 | } else if (MMO->isStore()) { | |||
| 3547 | if (mayStore) | |||
| 3548 | FilteredMemRefs.push_back(MMO); | |||
| 3549 | } else { | |||
| 3550 | FilteredMemRefs.push_back(MMO); | |||
| 3551 | } | |||
| 3552 | } | |||
| 3553 | ||||
| 3554 | CurDAG->setNodeMemRefs(Res, FilteredMemRefs); | |||
| 3555 | } | |||
| 3556 | ||||
| 3557 | LLVM_DEBUG(if (!MatchedMemRefs.empty() && Res->memoperands_empty()) dbgs()do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("isel")) { if (!MatchedMemRefs.empty() && Res->memoperands_empty ()) dbgs() << " Dropping mem operands\n"; dbgs() << " " << (IsMorphNodeTo ? "Morphed" : "Created") << " node: "; Res->dump(CurDAG);; } } while (false) | |||
| 3558 | << " Dropping mem operands\n";do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("isel")) { if (!MatchedMemRefs.empty() && Res->memoperands_empty ()) dbgs() << " Dropping mem operands\n"; dbgs() << " " << (IsMorphNodeTo ? "Morphed" : "Created") << " node: "; Res->dump(CurDAG);; } } while (false) | |||
| 3559 | dbgs() << " " << (IsMorphNodeTo ? "Morphed" : "Created")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("isel")) { if (!MatchedMemRefs.empty() && Res->memoperands_empty ()) dbgs() << " Dropping mem operands\n"; dbgs() << " " << (IsMorphNodeTo ? "Morphed" : "Created") << " node: "; Res->dump(CurDAG);; } } while (false) | |||
| 3560 | << " node: ";do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("isel")) { if (!MatchedMemRefs.empty() && Res->memoperands_empty ()) dbgs() << " Dropping mem operands\n"; dbgs() << " " << (IsMorphNodeTo ? "Morphed" : "Created") << " node: "; Res->dump(CurDAG);; } } while (false) | |||
| 3561 | Res->dump(CurDAG);)do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("isel")) { if (!MatchedMemRefs.empty() && Res->memoperands_empty ()) dbgs() << " Dropping mem operands\n"; dbgs() << " " << (IsMorphNodeTo ? "Morphed" : "Created") << " node: "; Res->dump(CurDAG);; } } while (false); | |||
| 3562 | ||||
| 3563 | // If this was a MorphNodeTo then we're completely done! | |||
| 3564 | if (IsMorphNodeTo) { | |||
| 3565 | // Update chain uses. | |||
| 3566 | UpdateChains(Res, InputChain, ChainNodesMatched, true); | |||
| 3567 | return; | |||
| 3568 | } | |||
| 3569 | continue; | |||
| 3570 | } | |||
| 3571 | ||||
| 3572 | case OPC_CompleteMatch: { | |||
| 3573 | // The match has been completed, and any new nodes (if any) have been | |||
| 3574 | // created. Patch up references to the matched dag to use the newly | |||
| 3575 | // created nodes. | |||
| 3576 | unsigned NumResults = MatcherTable[MatcherIndex++]; | |||
| 3577 | ||||
| 3578 | for (unsigned i = 0; i != NumResults; ++i) { | |||
| 3579 | unsigned ResSlot = MatcherTable[MatcherIndex++]; | |||
| 3580 | if (ResSlot & 128) | |||
| 3581 | ResSlot = GetVBR(ResSlot, MatcherTable, MatcherIndex); | |||
| 3582 | ||||
| 3583 | assert(ResSlot < RecordedNodes.size() && "Invalid CompleteMatch")((ResSlot < RecordedNodes.size() && "Invalid CompleteMatch" ) ? static_cast<void> (0) : __assert_fail ("ResSlot < RecordedNodes.size() && \"Invalid CompleteMatch\"" , "/build/llvm-toolchain-snapshot-10~++20200112100611+7fa5290d5bd/llvm/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp" , 3583, __PRETTY_FUNCTION__)); | |||
| 3584 | SDValue Res = RecordedNodes[ResSlot].first; | |||
| 3585 | ||||
| 3586 | assert(i < NodeToMatch->getNumValues() &&((i < NodeToMatch->getNumValues() && NodeToMatch ->getValueType(i) != MVT::Other && NodeToMatch-> getValueType(i) != MVT::Glue && "Invalid number of results to complete!" ) ? static_cast<void> (0) : __assert_fail ("i < NodeToMatch->getNumValues() && NodeToMatch->getValueType(i) != MVT::Other && NodeToMatch->getValueType(i) != MVT::Glue && \"Invalid number of results to complete!\"" , "/build/llvm-toolchain-snapshot-10~++20200112100611+7fa5290d5bd/llvm/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp" , 3589, __PRETTY_FUNCTION__)) | |||
| 3587 | NodeToMatch->getValueType(i) != MVT::Other &&((i < NodeToMatch->getNumValues() && NodeToMatch ->getValueType(i) != MVT::Other && NodeToMatch-> getValueType(i) != MVT::Glue && "Invalid number of results to complete!" ) ? static_cast<void> (0) : __assert_fail ("i < NodeToMatch->getNumValues() && NodeToMatch->getValueType(i) != MVT::Other && NodeToMatch->getValueType(i) != MVT::Glue && \"Invalid number of results to complete!\"" , "/build/llvm-toolchain-snapshot-10~++20200112100611+7fa5290d5bd/llvm/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp" , 3589, __PRETTY_FUNCTION__)) | |||
| 3588 | NodeToMatch->getValueType(i) != MVT::Glue &&((i < NodeToMatch->getNumValues() && NodeToMatch ->getValueType(i) != MVT::Other && NodeToMatch-> getValueType(i) != MVT::Glue && "Invalid number of results to complete!" ) ? static_cast<void> (0) : __assert_fail ("i < NodeToMatch->getNumValues() && NodeToMatch->getValueType(i) != MVT::Other && NodeToMatch->getValueType(i) != MVT::Glue && \"Invalid number of results to complete!\"" , "/build/llvm-toolchain-snapshot-10~++20200112100611+7fa5290d5bd/llvm/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp" , 3589, __PRETTY_FUNCTION__)) | |||
| 3589 | "Invalid number of results to complete!")((i < NodeToMatch->getNumValues() && NodeToMatch ->getValueType(i) != MVT::Other && NodeToMatch-> getValueType(i) != MVT::Glue && "Invalid number of results to complete!" ) ? static_cast<void> (0) : __assert_fail ("i < NodeToMatch->getNumValues() && NodeToMatch->getValueType(i) != MVT::Other && NodeToMatch->getValueType(i) != MVT::Glue && \"Invalid number of results to complete!\"" , "/build/llvm-toolchain-snapshot-10~++20200112100611+7fa5290d5bd/llvm/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp" , 3589, __PRETTY_FUNCTION__)); | |||
| 3590 | assert((NodeToMatch->getValueType(i) == Res.getValueType() ||(((NodeToMatch->getValueType(i) == Res.getValueType() || NodeToMatch ->getValueType(i) == MVT::iPTR || Res.getValueType() == MVT ::iPTR || NodeToMatch->getValueType(i).getSizeInBits() == Res .getValueSizeInBits()) && "invalid replacement") ? static_cast <void> (0) : __assert_fail ("(NodeToMatch->getValueType(i) == Res.getValueType() || NodeToMatch->getValueType(i) == MVT::iPTR || Res.getValueType() == MVT::iPTR || NodeToMatch->getValueType(i).getSizeInBits() == Res.getValueSizeInBits()) && \"invalid replacement\"" , "/build/llvm-toolchain-snapshot-10~++20200112100611+7fa5290d5bd/llvm/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp" , 3595, __PRETTY_FUNCTION__)) | |||
| 3591 | NodeToMatch->getValueType(i) == MVT::iPTR ||(((NodeToMatch->getValueType(i) == Res.getValueType() || NodeToMatch ->getValueType(i) == MVT::iPTR || Res.getValueType() == MVT ::iPTR || NodeToMatch->getValueType(i).getSizeInBits() == Res .getValueSizeInBits()) && "invalid replacement") ? static_cast <void> (0) : __assert_fail ("(NodeToMatch->getValueType(i) == Res.getValueType() || NodeToMatch->getValueType(i) == MVT::iPTR || Res.getValueType() == MVT::iPTR || NodeToMatch->getValueType(i).getSizeInBits() == Res.getValueSizeInBits()) && \"invalid replacement\"" , "/build/llvm-toolchain-snapshot-10~++20200112100611+7fa5290d5bd/llvm/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp" , 3595, __PRETTY_FUNCTION__)) | |||
| 3592 | Res.getValueType() == MVT::iPTR ||(((NodeToMatch->getValueType(i) == Res.getValueType() || NodeToMatch ->getValueType(i) == MVT::iPTR || Res.getValueType() == MVT ::iPTR || NodeToMatch->getValueType(i).getSizeInBits() == Res .getValueSizeInBits()) && "invalid replacement") ? static_cast <void> (0) : __assert_fail ("(NodeToMatch->getValueType(i) == Res.getValueType() || NodeToMatch->getValueType(i) == MVT::iPTR || Res.getValueType() == MVT::iPTR || NodeToMatch->getValueType(i).getSizeInBits() == Res.getValueSizeInBits()) && \"invalid replacement\"" , "/build/llvm-toolchain-snapshot-10~++20200112100611+7fa5290d5bd/llvm/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp" , 3595, __PRETTY_FUNCTION__)) | |||
| 3593 | NodeToMatch->getValueType(i).getSizeInBits() ==(((NodeToMatch->getValueType(i) == Res.getValueType() || NodeToMatch ->getValueType(i) == MVT::iPTR || Res.getValueType() == MVT ::iPTR || NodeToMatch->getValueType(i).getSizeInBits() == Res .getValueSizeInBits()) && "invalid replacement") ? static_cast <void> (0) : __assert_fail ("(NodeToMatch->getValueType(i) == Res.getValueType() || NodeToMatch->getValueType(i) == MVT::iPTR || Res.getValueType() == MVT::iPTR || NodeToMatch->getValueType(i).getSizeInBits() == Res.getValueSizeInBits()) && \"invalid replacement\"" , "/build/llvm-toolchain-snapshot-10~++20200112100611+7fa5290d5bd/llvm/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp" , 3595, __PRETTY_FUNCTION__)) | |||
| 3594 | Res.getValueSizeInBits()) &&(((NodeToMatch->getValueType(i) == Res.getValueType() || NodeToMatch ->getValueType(i) == MVT::iPTR || Res.getValueType() == MVT ::iPTR || NodeToMatch->getValueType(i).getSizeInBits() == Res .getValueSizeInBits()) && "invalid replacement") ? static_cast <void> (0) : __assert_fail ("(NodeToMatch->getValueType(i) == Res.getValueType() || NodeToMatch->getValueType(i) == MVT::iPTR || Res.getValueType() == MVT::iPTR || NodeToMatch->getValueType(i).getSizeInBits() == Res.getValueSizeInBits()) && \"invalid replacement\"" , "/build/llvm-toolchain-snapshot-10~++20200112100611+7fa5290d5bd/llvm/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp" , 3595, __PRETTY_FUNCTION__)) | |||
| 3595 | "invalid replacement")(((NodeToMatch->getValueType(i) == Res.getValueType() || NodeToMatch ->getValueType(i) == MVT::iPTR || Res.getValueType() == MVT ::iPTR || NodeToMatch->getValueType(i).getSizeInBits() == Res .getValueSizeInBits()) && "invalid replacement") ? static_cast <void> (0) : __assert_fail ("(NodeToMatch->getValueType(i) == Res.getValueType() || NodeToMatch->getValueType(i) == MVT::iPTR || Res.getValueType() == MVT::iPTR || NodeToMatch->getValueType(i).getSizeInBits() == Res.getValueSizeInBits()) && \"invalid replacement\"" , "/build/llvm-toolchain-snapshot-10~++20200112100611+7fa5290d5bd/llvm/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp" , 3595, __PRETTY_FUNCTION__)); | |||
| 3596 | ReplaceUses(SDValue(NodeToMatch, i), Res); | |||
| 3597 | } | |||
| 3598 | ||||
| 3599 | // Update chain uses. | |||
| 3600 | UpdateChains(NodeToMatch, InputChain, ChainNodesMatched, false); | |||
| 3601 | ||||
| 3602 | // If the root node defines glue, we need to update it to the glue result. | |||
| 3603 | // TODO: This never happens in our tests and I think it can be removed / | |||
| 3604 | // replaced with an assert, but if we do it this the way the change is | |||
| 3605 | // NFC. | |||
| 3606 | if (NodeToMatch->getValueType(NodeToMatch->getNumValues() - 1) == | |||
| 3607 | MVT::Glue && | |||
| 3608 | InputGlue.getNode()) | |||
| 3609 | ReplaceUses(SDValue(NodeToMatch, NodeToMatch->getNumValues() - 1), | |||
| 3610 | InputGlue); | |||
| 3611 | ||||
| 3612 | assert(NodeToMatch->use_empty() &&((NodeToMatch->use_empty() && "Didn't replace all uses of the node?" ) ? static_cast<void> (0) : __assert_fail ("NodeToMatch->use_empty() && \"Didn't replace all uses of the node?\"" , "/build/llvm-toolchain-snapshot-10~++20200112100611+7fa5290d5bd/llvm/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp" , 3613, __PRETTY_FUNCTION__)) | |||
| 3613 | "Didn't replace all uses of the node?")((NodeToMatch->use_empty() && "Didn't replace all uses of the node?" ) ? static_cast<void> (0) : __assert_fail ("NodeToMatch->use_empty() && \"Didn't replace all uses of the node?\"" , "/build/llvm-toolchain-snapshot-10~++20200112100611+7fa5290d5bd/llvm/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp" , 3613, __PRETTY_FUNCTION__)); | |||
| 3614 | CurDAG->RemoveDeadNode(NodeToMatch); | |||
| 3615 | ||||
| 3616 | return; | |||
| 3617 | } | |||
| 3618 | } | |||
| 3619 | ||||
| 3620 | // If the code reached this point, then the match failed. See if there is | |||
| 3621 | // another child to try in the current 'Scope', otherwise pop it until we | |||
| 3622 | // find a case to check. | |||
| 3623 | LLVM_DEBUG(dbgs() << " Match failed at index " << CurrentOpcodeIndexdo { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("isel")) { dbgs() << " Match failed at index " << CurrentOpcodeIndex << "\n"; } } while (false) | |||
| 3624 | << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("isel")) { dbgs() << " Match failed at index " << CurrentOpcodeIndex << "\n"; } } while (false); | |||
| 3625 | ++NumDAGIselRetries; | |||
| 3626 | while (true) { | |||
| 3627 | if (MatchScopes.empty()) { | |||
| 3628 | CannotYetSelect(NodeToMatch); | |||
| 3629 | return; | |||
| 3630 | } | |||
| 3631 | ||||
| 3632 | // Restore the interpreter state back to the point where the scope was | |||
| 3633 | // formed. | |||
| 3634 | MatchScope &LastScope = MatchScopes.back(); | |||
| 3635 | RecordedNodes.resize(LastScope.NumRecordedNodes); | |||
| 3636 | NodeStack.clear(); | |||
| 3637 | NodeStack.append(LastScope.NodeStack.begin(), LastScope.NodeStack.end()); | |||
| 3638 | N = NodeStack.back(); | |||
| 3639 | ||||
| 3640 | if (LastScope.NumMatchedMemRefs != MatchedMemRefs.size()) | |||
| 3641 | MatchedMemRefs.resize(LastScope.NumMatchedMemRefs); | |||
| 3642 | MatcherIndex = LastScope.FailIndex; | |||
| 3643 | ||||
| 3644 | LLVM_DEBUG(dbgs() << " Continuing at " << MatcherIndex << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("isel")) { dbgs() << " Continuing at " << MatcherIndex << "\n"; } } while (false); | |||
| 3645 | ||||
| 3646 | InputChain = LastScope.InputChain; | |||
| 3647 | InputGlue = LastScope.InputGlue; | |||
| 3648 | if (!LastScope.HasChainNodesMatched) | |||
| 3649 | ChainNodesMatched.clear(); | |||
| 3650 | ||||
| 3651 | // Check to see what the offset is at the new MatcherIndex. If it is zero | |||
| 3652 | // we have reached the end of this scope, otherwise we have another child | |||
| 3653 | // in the current scope to try. | |||
| 3654 | unsigned NumToSkip = MatcherTable[MatcherIndex++]; | |||
| 3655 | if (NumToSkip & 128) | |||
| 3656 | NumToSkip = GetVBR(NumToSkip, MatcherTable, MatcherIndex); | |||
| 3657 | ||||
| 3658 | // If we have another child in this scope to match, update FailIndex and | |||
| 3659 | // try it. | |||
| 3660 | if (NumToSkip != 0) { | |||
| 3661 | LastScope.FailIndex = MatcherIndex+NumToSkip; | |||
| 3662 | break; | |||
| 3663 | } | |||
| 3664 | ||||
| 3665 | // End of this scope, pop it and try the next child in the containing | |||
| 3666 | // scope. | |||
| 3667 | MatchScopes.pop_back(); | |||
| 3668 | } | |||
| 3669 | } | |||
| 3670 | } | |||
| 3671 | ||||
| 3672 | /// Return whether the node may raise an FP exception. | |||
| 3673 | bool SelectionDAGISel::mayRaiseFPException(SDNode *N) const { | |||
| 3674 | // For machine opcodes, consult the MCID flag. | |||
| 3675 | if (N->isMachineOpcode()) { | |||
| 3676 | const MCInstrDesc &MCID = TII->get(N->getMachineOpcode()); | |||
| 3677 | return MCID.mayRaiseFPException(); | |||
| 3678 | } | |||
| 3679 | ||||
| 3680 | // For ISD opcodes, only StrictFP opcodes may raise an FP | |||
| 3681 | // exception. | |||
| 3682 | if (N->isTargetOpcode()) | |||
| 3683 | return N->isTargetStrictFPOpcode(); | |||
| 3684 | return N->isStrictFPOpcode(); | |||
| 3685 | } | |||
| 3686 | ||||
| 3687 | bool SelectionDAGISel::isOrEquivalentToAdd(const SDNode *N) const { | |||
| 3688 | assert(N->getOpcode() == ISD::OR && "Unexpected opcode")((N->getOpcode() == ISD::OR && "Unexpected opcode" ) ? static_cast<void> (0) : __assert_fail ("N->getOpcode() == ISD::OR && \"Unexpected opcode\"" , "/build/llvm-toolchain-snapshot-10~++20200112100611+7fa5290d5bd/llvm/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp" , 3688, __PRETTY_FUNCTION__)); | |||
| 3689 | auto *C = dyn_cast<ConstantSDNode>(N->getOperand(1)); | |||
| 3690 | if (!C) | |||
| 3691 | return false; | |||
| 3692 | ||||
| 3693 | // Detect when "or" is used to add an offset to a stack object. | |||
| 3694 | if (auto *FN = dyn_cast<FrameIndexSDNode>(N->getOperand(0))) { | |||
| 3695 | MachineFrameInfo &MFI = MF->getFrameInfo(); | |||
| 3696 | unsigned A = MFI.getObjectAlignment(FN->getIndex()); | |||
| 3697 | assert(isPowerOf2_32(A) && "Unexpected alignment")((isPowerOf2_32(A) && "Unexpected alignment") ? static_cast <void> (0) : __assert_fail ("isPowerOf2_32(A) && \"Unexpected alignment\"" , "/build/llvm-toolchain-snapshot-10~++20200112100611+7fa5290d5bd/llvm/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp" , 3697, __PRETTY_FUNCTION__)); | |||
| 3698 | int32_t Off = C->getSExtValue(); | |||
| 3699 | // If the alleged offset fits in the zero bits guaranteed by | |||
| 3700 | // the alignment, then this or is really an add. | |||
| 3701 | return (Off >= 0) && (((A - 1) & Off) == unsigned(Off)); | |||
| 3702 | } | |||
| 3703 | return false; | |||
| 3704 | } | |||
| 3705 | ||||
| 3706 | void SelectionDAGISel::CannotYetSelect(SDNode *N) { | |||
| 3707 | std::string msg; | |||
| 3708 | raw_string_ostream Msg(msg); | |||
| 3709 | Msg << "Cannot select: "; | |||
| 3710 | ||||
| 3711 | if (N->getOpcode() != ISD::INTRINSIC_W_CHAIN && | |||
| 3712 | N->getOpcode() != ISD::INTRINSIC_WO_CHAIN && | |||
| 3713 | N->getOpcode() != ISD::INTRINSIC_VOID) { | |||
| 3714 | N->printrFull(Msg, CurDAG); | |||
| 3715 | Msg << "\nIn function: " << MF->getName(); | |||
| 3716 | } else { | |||
| 3717 | bool HasInputChain = N->getOperand(0).getValueType() == MVT::Other; | |||
| 3718 | unsigned iid = | |||
| 3719 | cast<ConstantSDNode>(N->getOperand(HasInputChain))->getZExtValue(); | |||
| 3720 | if (iid < Intrinsic::num_intrinsics) | |||
| 3721 | Msg << "intrinsic %" << Intrinsic::getName((Intrinsic::ID)iid, None); | |||
| 3722 | else if (const TargetIntrinsicInfo *TII = TM.getIntrinsicInfo()) | |||
| 3723 | Msg << "target intrinsic %" << TII->getName(iid); | |||
| 3724 | else | |||
| 3725 | Msg << "unknown intrinsic #" << iid; | |||
| 3726 | } | |||
| 3727 | report_fatal_error(Msg.str()); | |||
| 3728 | } | |||
| 3729 | ||||
| 3730 | char SelectionDAGISel::ID = 0; |