LCOV - code coverage report
Current view: top level - lib/CodeGen/SelectionDAG - FastISel.cpp (source / functions) Hit Total Coverage
Test: llvm-toolchain.info Lines: 921 1025 89.9 %
Date: 2018-06-17 00:07:59 Functions: 67 74 90.5 %
Legend: Lines: hit not hit

          Line data    Source code
       1             : //===- FastISel.cpp - Implementation of the FastISel class ----------------===//
       2             : //
       3             : //                     The LLVM Compiler Infrastructure
       4             : //
       5             : // This file is distributed under the University of Illinois Open Source
       6             : // License. See LICENSE.TXT for details.
       7             : //
       8             : //===----------------------------------------------------------------------===//
       9             : //
      10             : // This file contains the implementation of the FastISel class.
      11             : //
      12             : // "Fast" instruction selection is designed to emit very poor code quickly.
      13             : // Also, it is not designed to be able to do much lowering, so most illegal
      14             : // types (e.g. i64 on 32-bit targets) and operations are not supported.  It is
      15             : // also not intended to be able to do much optimization, except in a few cases
      16             : // where doing optimizations reduces overall compile time.  For example, folding
      17             : // constants into immediate fields is often done, because it's cheap and it
      18             : // reduces the number of instructions later phases have to examine.
      19             : //
      20             : // "Fast" instruction selection is able to fail gracefully and transfer
      21             : // control to the SelectionDAG selector for operations that it doesn't
      22             : // support.  In many cases, this allows us to avoid duplicating a lot of
      23             : // the complicated lowering logic that SelectionDAG currently has.
      24             : //
      25             : // The intended use for "fast" instruction selection is "-O0" mode
      26             : // compilation, where the quality of the generated code is irrelevant when
      27             : // weighed against the speed at which the code can be generated.  Also,
      28             : // at -O0, the LLVM optimizers are not running, and this makes the
      29             : // compile time of codegen a much higher portion of the overall compile
      30             : // time.  Despite its limitations, "fast" instruction selection is able to
      31             : // handle enough code on its own to provide noticeable overall speedups
      32             : // in -O0 compiles.
      33             : //
      34             : // Basic operations are supported in a target-independent way, by reading
      35             : // the same instruction descriptions that the SelectionDAG selector reads,
      36             : // and identifying simple arithmetic operations that can be directly selected
      37             : // from simple operators.  More complicated operations currently require
      38             : // target-specific code.
      39             : //
      40             : //===----------------------------------------------------------------------===//
      41             : 
      42             : #include "llvm/CodeGen/FastISel.h"
      43             : #include "llvm/ADT/APFloat.h"
      44             : #include "llvm/ADT/APSInt.h"
      45             : #include "llvm/ADT/DenseMap.h"
      46             : #include "llvm/ADT/Optional.h"
      47             : #include "llvm/ADT/SmallPtrSet.h"
      48             : #include "llvm/ADT/SmallString.h"
      49             : #include "llvm/ADT/SmallVector.h"
      50             : #include "llvm/ADT/Statistic.h"
      51             : #include "llvm/Analysis/BranchProbabilityInfo.h"
      52             : #include "llvm/Analysis/TargetLibraryInfo.h"
      53             : #include "llvm/CodeGen/Analysis.h"
      54             : #include "llvm/CodeGen/FunctionLoweringInfo.h"
      55             : #include "llvm/CodeGen/ISDOpcodes.h"
      56             : #include "llvm/CodeGen/MachineBasicBlock.h"
      57             : #include "llvm/CodeGen/MachineFrameInfo.h"
      58             : #include "llvm/CodeGen/MachineInstr.h"
      59             : #include "llvm/CodeGen/MachineInstrBuilder.h"
      60             : #include "llvm/CodeGen/MachineMemOperand.h"
      61             : #include "llvm/CodeGen/MachineModuleInfo.h"
      62             : #include "llvm/CodeGen/MachineOperand.h"
      63             : #include "llvm/CodeGen/MachineRegisterInfo.h"
      64             : #include "llvm/CodeGen/StackMaps.h"
      65             : #include "llvm/CodeGen/TargetInstrInfo.h"
      66             : #include "llvm/CodeGen/TargetLowering.h"
      67             : #include "llvm/CodeGen/TargetSubtargetInfo.h"
      68             : #include "llvm/CodeGen/ValueTypes.h"
      69             : #include "llvm/IR/Argument.h"
      70             : #include "llvm/IR/Attributes.h"
      71             : #include "llvm/IR/BasicBlock.h"
      72             : #include "llvm/IR/CallSite.h"
      73             : #include "llvm/IR/CallingConv.h"
      74             : #include "llvm/IR/Constant.h"
      75             : #include "llvm/IR/Constants.h"
      76             : #include "llvm/IR/DataLayout.h"
      77             : #include "llvm/IR/DebugInfo.h"
      78             : #include "llvm/IR/DebugLoc.h"
      79             : #include "llvm/IR/DerivedTypes.h"
      80             : #include "llvm/IR/Function.h"
      81             : #include "llvm/IR/GetElementPtrTypeIterator.h"
      82             : #include "llvm/IR/GlobalValue.h"
      83             : #include "llvm/IR/InlineAsm.h"
      84             : #include "llvm/IR/InstrTypes.h"
      85             : #include "llvm/IR/Instruction.h"
      86             : #include "llvm/IR/Instructions.h"
      87             : #include "llvm/IR/IntrinsicInst.h"
      88             : #include "llvm/IR/LLVMContext.h"
      89             : #include "llvm/IR/Mangler.h"
      90             : #include "llvm/IR/Metadata.h"
      91             : #include "llvm/IR/Operator.h"
      92             : #include "llvm/IR/Type.h"
      93             : #include "llvm/IR/User.h"
      94             : #include "llvm/IR/Value.h"
      95             : #include "llvm/MC/MCContext.h"
      96             : #include "llvm/MC/MCInstrDesc.h"
      97             : #include "llvm/MC/MCRegisterInfo.h"
      98             : #include "llvm/Support/Casting.h"
      99             : #include "llvm/Support/Debug.h"
     100             : #include "llvm/Support/ErrorHandling.h"
     101             : #include "llvm/Support/MachineValueType.h"
     102             : #include "llvm/Support/MathExtras.h"
     103             : #include "llvm/Support/raw_ostream.h"
     104             : #include "llvm/Target/TargetMachine.h"
     105             : #include "llvm/Target/TargetOptions.h"
     106             : #include <algorithm>
     107             : #include <cassert>
     108             : #include <cstdint>
     109             : #include <iterator>
     110             : #include <utility>
     111             : 
     112             : using namespace llvm;
     113             : 
     114             : #define DEBUG_TYPE "isel"
     115             : 
     116             : // FIXME: Remove this after the feature has proven reliable.
     117      101169 : static cl::opt<bool> SinkLocalValues("fast-isel-sink-local-values",
     118      202338 :                                      cl::init(true), cl::Hidden,
     119      303507 :                                      cl::desc("Sink local values in FastISel"));
     120             : 
     121             : STATISTIC(NumFastIselSuccessIndependent, "Number of insts selected by "
     122             :                                          "target-independent selector");
     123             : STATISTIC(NumFastIselSuccessTarget, "Number of insts selected by "
     124             :                                     "target-specific selector");
     125             : STATISTIC(NumFastIselDead, "Number of dead insts removed on failure");
     126             : 
     127             : /// Set the current block to which generated machine instructions will be
     128             : /// appended.
     129      102929 : void FastISel::startNewBlock() {
     130             :   assert(LocalValueMap.empty() &&
     131             :          "local values should be cleared after finishing a BB");
     132             : 
     133             :   // Instructions are appended to FuncInfo.MBB. If the basic block already
     134             :   // contains labels or copies, use the last instruction as the last local
     135             :   // value.
     136      102929 :   EmitStartPt = nullptr;
     137      205858 :   if (!FuncInfo.MBB->empty())
     138        6424 :     EmitStartPt = &FuncInfo.MBB->back();
     139      102929 :   LastLocalValue = EmitStartPt;
     140      102929 : }
     141             : 
     142             : /// Flush the local CSE map and sink anything we can.
     143      102926 : void FastISel::finishBasicBlock() { flushLocalValueMap(); }
     144             : 
     145       49900 : bool FastISel::lowerArguments() {
     146       49900 :   if (!FuncInfo.CanLowerReturn)
     147             :     // Fallback to SDISel argument lowering code to deal with sret pointer
     148             :     // parameter.
     149             :     return false;
     150             : 
     151       49881 :   if (!fastLowerArguments())
     152             :     return false;
     153             : 
     154             :   // Enter arguments into ValueMap for uses in non-entry BBs.
     155       99967 :   for (Function::const_arg_iterator I = FuncInfo.Fn->arg_begin(),
     156       41569 :                                     E = FuncInfo.Fn->arg_end();
     157       99967 :        I != E; ++I) {
     158       58398 :     DenseMap<const Value *, unsigned>::iterator VI = LocalValueMap.find(&*I);
     159             :     assert(VI != LocalValueMap.end() && "Missed an argument?");
     160      116796 :     FuncInfo.ValueMap[&*I] = VI->second;
     161             :   }
     162             :   return true;
     163             : }
     164             : 
     165             : /// Return the defined register if this instruction defines exactly one
     166             : /// virtual register and uses no other virtual registers. Otherwise return 0.
     167       36901 : static unsigned findSinkableLocalRegDef(MachineInstr &MI) {
     168             :   unsigned RegDef = 0;
     169      276853 :   for (const MachineOperand &MO : MI.operands()) {
     170      127170 :     if (!MO.isReg())
     171       56972 :       continue;
     172       70198 :     if (MO.isDef()) {
     173       40220 :       if (RegDef)
     174             :         return 0;
     175       36901 :       RegDef = MO.getReg();
     176       59956 :     } else if (TargetRegisterInfo::isVirtualRegister(MO.getReg())) {
     177             :       // This is another use of a vreg. Don't try to sink it.
     178             :       return 0;
     179             :     }
     180             :   }
     181             :   return RegDef;
     182             : }
     183             : 
     184      163599 : void FastISel::flushLocalValueMap() {
     185             :   // Try to sink local values down to their first use so that we can give them a
     186             :   // better debug location. This has the side effect of shrinking local value
     187             :   // live ranges, which helps out fast regalloc.
     188      163599 :   if (SinkLocalValues && LastLocalValue != EmitStartPt) {
     189             :     // Sink local value materialization instructions between EmitStartPt and
     190             :     // LastLocalValue. Visit them bottom-up, starting from LastLocalValue, to
     191             :     // avoid inserting into the range that we're iterating over.
     192             :     MachineBasicBlock::reverse_iterator RE =
     193             :         EmitStartPt ? MachineBasicBlock::reverse_iterator(EmitStartPt)
     194       27425 :                     : FuncInfo.MBB->rend();
     195             :     MachineBasicBlock::reverse_iterator RI(LastLocalValue);
     196             : 
     197             :     InstOrderMap OrderMap;
     198       65741 :     for (; RI != RE;) {
     199             :       MachineInstr &LocalMI = *RI;
     200             :       ++RI;
     201       38316 :       bool Store = true;
     202       38316 :       if (!LocalMI.isSafeToMove(nullptr, Store))
     203       10024 :         continue;
     204       36901 :       unsigned DefReg = findSinkableLocalRegDef(LocalMI);
     205       36901 :       if (DefReg == 0)
     206        7194 :         continue;
     207             : 
     208       29707 :       sinkLocalValueMaterialization(LocalMI, DefReg, OrderMap);
     209             :     }
     210             :   }
     211             : 
     212      163599 :   LocalValueMap.clear();
     213      163599 :   LastLocalValue = EmitStartPt;
     214      163599 :   recomputeInsertPt();
     215      163599 :   SavedInsertPt = FuncInfo.InsertPt;
     216      163599 :   LastFlushPoint = FuncInfo.InsertPt;
     217      163599 : }
     218             : 
     219             : static bool isRegUsedByPhiNodes(unsigned DefReg,
     220             :                                 FunctionLoweringInfo &FuncInfo) {
     221       29473 :   for (auto &P : FuncInfo.PHINodesToUpdate)
     222         518 :     if (P.second == DefReg)
     223             :       return true;
     224             :   return false;
     225             : }
     226             : 
     227             : /// Build a map of instruction orders. Return the first terminator and its
     228             : /// order. Consider EH_LABEL instructions to be terminators as well, since local
     229             : /// values for phis after invokes must be materialized before the call.
     230       23951 : void FastISel::InstOrderMap::initialize(
     231             :     MachineBasicBlock *MBB, MachineBasicBlock::iterator LastFlushPoint) {
     232             :   unsigned Order = 0;
     233      252997 :   for (MachineInstr &I : *MBB) {
     234      444741 :     if (!FirstTerminator &&
     235      216681 :         (I.isTerminator() || (I.isEHLabel() && &I != &MBB->front()))) {
     236        6618 :       FirstTerminator = &I;
     237        6618 :       FirstTerminatorOrder = Order;
     238             :     }
     239      444794 :     Orders[&I] = Order++;
     240             : 
     241             :     // We don't need to order instructions past the last flush point.
     242      444794 :     if (I.getIterator() == LastFlushPoint)
     243             :       break;
     244             :   }
     245       23951 : }
     246             : 
     247       29707 : void FastISel::sinkLocalValueMaterialization(MachineInstr &LocalMI,
     248             :                                              unsigned DefReg,
     249             :                                              InstOrderMap &OrderMap) {
     250             :   // If this register is used by a register fixup, MRI will not contain all
     251             :   // the uses until after register fixups, so don't attempt to sink or DCE
     252             :   // this instruction. Register fixups typically come from no-op cast
     253             :   // instructions, which replace the cast instruction vreg with the local
     254             :   // value vreg.
     255       29707 :   if (FuncInfo.RegsWithFixups.count(DefReg))
     256         697 :     return;
     257             : 
     258             :   // We can DCE this instruction if there are no uses and it wasn't a
     259             :   // materialized for a successor PHI node.
     260       29132 :   bool UsedByPHI = isRegUsedByPhiNodes(DefReg, FuncInfo);
     261       58087 :   if (!UsedByPHI && MRI.use_nodbg_empty(DefReg)) {
     262         122 :     if (EmitStartPt == &LocalMI)
     263           0 :       EmitStartPt = EmitStartPt->getPrevNode();
     264             :     LLVM_DEBUG(dbgs() << "removing dead local value materialization "
     265             :                       << LocalMI);
     266         122 :     OrderMap.Orders.erase(&LocalMI);
     267         122 :     LocalMI.eraseFromParent();
     268         122 :     return;
     269             :   }
     270             : 
     271             :   // Number the instructions if we haven't yet so we can efficiently find the
     272             :   // earliest use.
     273       29010 :   if (OrderMap.Orders.empty())
     274       23951 :     OrderMap.initialize(FuncInfo.MBB, LastFlushPoint);
     275             : 
     276             :   // Find the first user in the BB.
     277             :   MachineInstr *FirstUser = nullptr;
     278             :   unsigned FirstOrder = std::numeric_limits<unsigned>::max();
     279      145810 :   for (MachineInstr &UseInst : MRI.use_nodbg_instructions(DefReg)) {
     280       29390 :     auto I = OrderMap.Orders.find(&UseInst);
     281             :     assert(I != OrderMap.Orders.end() &&
     282             :            "local value used by instruction outside local region");
     283       29390 :     unsigned UseOrder = I->second;
     284       29390 :     if (UseOrder < FirstOrder) {
     285             :       FirstOrder = UseOrder;
     286             :       FirstUser = &UseInst;
     287             :     }
     288             :   }
     289             : 
     290             :   // The insertion point will be the first terminator or the first user,
     291             :   // whichever came first. If there was no terminator, this must be a
     292             :   // fallthrough block and the insertion point is the end of the block.
     293             :   MachineBasicBlock::instr_iterator SinkPos;
     294       29010 :   if (UsedByPHI && OrderMap.FirstTerminatorOrder < FirstOrder) {
     295             :     FirstOrder = OrderMap.FirstTerminatorOrder;
     296         173 :     SinkPos = OrderMap.FirstTerminator->getIterator();
     297       28837 :   } else if (FirstUser) {
     298       28833 :     SinkPos = FirstUser->getIterator();
     299             :   } else {
     300             :     assert(UsedByPHI && "must be users if not used by a phi");
     301           4 :     SinkPos = FuncInfo.MBB->instr_end();
     302             :   }
     303             : 
     304             :   // Collect all DBG_VALUEs before the new insertion position so that we can
     305             :   // sink them.
     306             :   SmallVector<MachineInstr *, 1> DbgValues;
     307      147886 :   for (MachineInstr &DbgVal : MRI.use_instructions(DefReg)) {
     308       30428 :     if (!DbgVal.isDebugValue())
     309       29390 :       continue;
     310        2076 :     unsigned UseOrder = OrderMap.Orders[&DbgVal];
     311        1038 :     if (UseOrder < FirstOrder)
     312        1038 :       DbgValues.push_back(&DbgVal);
     313             :   }
     314             : 
     315             :   // Sink LocalMI before SinkPos and assign it the same DebugLoc.
     316             :   LLVM_DEBUG(dbgs() << "sinking local value to first use " << LocalMI);
     317       29010 :   FuncInfo.MBB->remove(&LocalMI);
     318       29010 :   FuncInfo.MBB->insert(SinkPos, &LocalMI);
     319       58020 :   if (SinkPos != FuncInfo.MBB->end())
     320       29006 :     LocalMI.setDebugLoc(SinkPos->getDebugLoc());
     321             : 
     322             :   // Sink any debug values that we've collected.
     323       31086 :   for (MachineInstr *DI : DbgValues) {
     324        1038 :     FuncInfo.MBB->remove(DI);
     325        1038 :     FuncInfo.MBB->insert(SinkPos, DI);
     326             :   }
     327             : }
     328             : 
     329      135138 : bool FastISel::hasTrivialKill(const Value *V) {
     330             :   // Don't consider constants or arguments to have trivial kills.
     331             :   const Instruction *I = dyn_cast<Instruction>(V);
     332             :   if (!I)
     333             :     return false;
     334             : 
     335             :   // No-op casts are trivially coalesced by fast-isel.
     336             :   if (const auto *Cast = dyn_cast<CastInst>(I))
     337       11622 :     if (Cast->isNoopCast(DL) && !hasTrivialKill(Cast->getOperand(0)))
     338             :       return false;
     339             : 
     340             :   // Even the value might have only one use in the LLVM IR, it is possible that
     341             :   // FastISel might fold the use into another instruction and now there is more
     342             :   // than one use at the Machine Instruction level.
     343       70303 :   unsigned Reg = lookUpRegForValue(V);
     344      135315 :   if (Reg && !MRI.use_empty(Reg))
     345             :     return false;
     346             : 
     347             :   // GEPs with all zero indices are trivially coalesced by fast-isel.
     348             :   if (const auto *GEP = dyn_cast<GetElementPtrInst>(I))
     349        6929 :     if (GEP->hasAllZeroIndices() && !hasTrivialKill(GEP->getOperand(0)))
     350             :       return false;
     351             : 
     352             :   // Only instructions with a single use in the same basic block are considered
     353             :   // to have trivial kills.
     354       59918 :   return I->hasOneUse() &&
     355       58229 :          !(I->getOpcode() == Instruction::BitCast ||
     356             :            I->getOpcode() == Instruction::PtrToInt ||
     357      120386 :            I->getOpcode() == Instruction::IntToPtr) &&
     358       56767 :          cast<Instruction>(*I->user_begin())->getParent() == I->getParent();
     359             : }
     360             : 
     361      354296 : unsigned FastISel::getRegForValue(const Value *V) {
     362      354296 :   EVT RealVT = TLI.getValueType(DL, V->getType(), /*AllowUnknown=*/true);
     363             :   // Don't handle non-simple values in FastISel.
     364      354296 :   if (!RealVT.isSimple())
     365             :     return 0;
     366             : 
     367             :   // Ignore illegal types. We must do this before looking up the value
     368             :   // in ValueMap because Arguments are given virtual registers regardless
     369             :   // of whether FastISel can handle them.
     370             :   MVT VT = RealVT.getSimpleVT();
     371      354286 :   if (!TLI.isTypeLegal(VT)) {
     372             :     // Handle integer promotions, though, because they're common and easy.
     373        7023 :     if (VT == MVT::i1 || VT == MVT::i8 || VT == MVT::i16)
     374        6304 :       VT = TLI.getTypeToTransformTo(V->getContext(), VT).getSimpleVT();
     375             :     else
     376             :       return 0;
     377             :   }
     378             : 
     379             :   // Look up the value to see if we already have a register for it.
     380      353567 :   unsigned Reg = lookUpRegForValue(V);
     381      353567 :   if (Reg)
     382             :     return Reg;
     383             : 
     384             :   // In bottom-up mode, just create the virtual register which will be used
     385             :   // to hold the value. It will be materialized later.
     386      202668 :   if (isa<Instruction>(V) &&
     387             :       (!isa<AllocaInst>(V) ||
     388       12482 :        !FuncInfo.StaticAllocaMap.count(cast<AllocaInst>(V))))
     389      159211 :     return FuncInfo.InitializeRegForValue(V);
     390             : 
     391       43457 :   SavePoint SaveInsertPt = enterLocalValueArea();
     392             : 
     393             :   // Materialize the value in a register. Emit any instructions in the
     394             :   // local value area.
     395       43457 :   Reg = materializeRegForValue(V, VT);
     396             : 
     397       86914 :   leaveLocalValueArea(SaveInsertPt);
     398             : 
     399             :   return Reg;
     400             : }
     401             : 
     402       22036 : unsigned FastISel::materializeConstant(const Value *V, MVT VT) {
     403             :   unsigned Reg = 0;
     404             :   if (const auto *CI = dyn_cast<ConstantInt>(V)) {
     405         118 :     if (CI->getValue().getActiveBits() <= 64)
     406         236 :       Reg = fastEmit_i(VT, VT, ISD::Constant, CI->getZExtValue());
     407             :   } else if (isa<AllocaInst>(V))
     408       12468 :     Reg = fastMaterializeAlloca(cast<AllocaInst>(V));
     409        9450 :   else if (isa<ConstantPointerNull>(V))
     410             :     // Translate this as an integer zero so that it can be
     411             :     // local-CSE'd with actual integer zeros.
     412        1098 :     Reg = getRegForValue(
     413        1098 :         Constant::getNullValue(DL.getIntPtrType(V->getContext())));
     414             :   else if (const auto *CF = dyn_cast<ConstantFP>(V)) {
     415           5 :     if (CF->isNullValue())
     416           1 :       Reg = fastMaterializeFloatZero(CF);
     417             :     else
     418             :       // Try to emit the constant directly.
     419           4 :       Reg = fastEmit_f(VT, VT, ISD::ConstantFP, CF);
     420             : 
     421           5 :     if (!Reg) {
     422             :       // Try to emit the constant by using an integer constant with a cast.
     423             :       const APFloat &Flt = CF->getValueAPF();
     424           1 :       EVT IntVT = TLI.getPointerTy(DL);
     425           1 :       uint32_t IntBitWidth = IntVT.getSizeInBits();
     426           1 :       APSInt SIntVal(IntBitWidth, /*isUnsigned=*/false);
     427             :       bool isExact;
     428           1 :       (void)Flt.convertToInteger(SIntVal, APFloat::rmTowardZero, &isExact);
     429           1 :       if (isExact) {
     430             :         unsigned IntegerReg =
     431           1 :             getRegForValue(ConstantInt::get(V->getContext(), SIntVal));
     432           1 :         if (IntegerReg != 0)
     433           1 :           Reg = fastEmit_r(IntVT.getSimpleVT(), VT, ISD::SINT_TO_FP, IntegerReg,
     434           1 :                            /*Kill=*/false);
     435             :       }
     436             :     }
     437             :   } else if (const auto *Op = dyn_cast<Operator>(V)) {
     438        8007 :     if (!selectOperator(Op, Op->getOpcode()))
     439         108 :       if (!isa<Instruction>(Op) ||
     440           0 :           !fastSelectInstruction(cast<Instruction>(Op)))
     441             :         return 0;
     442        7899 :     Reg = lookUpRegForValue(Op);
     443         340 :   } else if (isa<UndefValue>(V)) {
     444         175 :     Reg = createResultReg(TLI.getRegClassFor(VT));
     445         175 :     BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc,
     446         350 :             TII.get(TargetOpcode::IMPLICIT_DEF), Reg);
     447             :   }
     448             :   return Reg;
     449             : }
     450             : 
     451             : /// Helper for getRegForValue. This function is called when the value isn't
     452             : /// already available in a register and must be materialized with new
     453             : /// instructions.
     454       43457 : unsigned FastISel::materializeRegForValue(const Value *V, MVT VT) {
     455             :   unsigned Reg = 0;
     456             :   // Give the target-specific code a try first.
     457       86914 :   if (isa<Constant>(V))
     458       30989 :     Reg = fastMaterializeConstant(cast<Constant>(V));
     459             : 
     460             :   // If target-specific code couldn't or didn't want to handle the value, then
     461             :   // give target-independent code a try.
     462       30989 :   if (!Reg)
     463       22036 :     Reg = materializeConstant(V, VT);
     464             : 
     465             :   // Don't cache constant materializations in the general ValueMap.
     466             :   // To do so would require tracking what uses they dominate.
     467       43457 :   if (Reg) {
     468       86366 :     LocalValueMap[V] = Reg;
     469       43183 :     LastLocalValue = MRI.getVRegDef(Reg);
     470             :   }
     471       43457 :   return Reg;
     472             : }
     473             : 
     474      489790 : unsigned FastISel::lookUpRegForValue(const Value *V) {
     475             :   // Look up the value to see if we already have a register for it. We
     476             :   // cache values defined by Instructions across blocks, and other values
     477             :   // only locally. This is because Instructions already have the SSA
     478             :   // def-dominates-use requirement enforced.
     479      489790 :   DenseMap<const Value *, unsigned>::iterator I = FuncInfo.ValueMap.find(V);
     480      979580 :   if (I != FuncInfo.ValueMap.end())
     481      212290 :     return I->second;
     482      555000 :   return LocalValueMap[V];
     483             : }
     484             : 
     485      222864 : void FastISel::updateValueMap(const Value *I, unsigned Reg, unsigned NumRegs) {
     486      445728 :   if (!isa<Instruction>(I)) {
     487      132594 :     LocalValueMap[I] = Reg;
     488       66297 :     return;
     489             :   }
     490             : 
     491      156567 :   unsigned &AssignedReg = FuncInfo.ValueMap[I];
     492      156567 :   if (AssignedReg == 0)
     493             :     // Use the new register.
     494        7909 :     AssignedReg = Reg;
     495      148658 :   else if (Reg != AssignedReg) {
     496             :     // Arrange for uses of AssignedReg to be replaced by uses of Reg.
     497      446486 :     for (unsigned i = 0; i < NumRegs; i++) {
     498      297828 :       FuncInfo.RegFixups[AssignedReg + i] = Reg + i;
     499      297828 :       FuncInfo.RegsWithFixups.insert(Reg + i);
     500             :     }
     501             : 
     502      148658 :     AssignedReg = Reg;
     503             :   }
     504             : }
     505             : 
     506        1499 : std::pair<unsigned, bool> FastISel::getRegForGEPIndex(const Value *Idx) {
     507        1499 :   unsigned IdxN = getRegForValue(Idx);
     508        1499 :   if (IdxN == 0)
     509             :     // Unhandled operand. Halt "fast" selection and bail.
     510           9 :     return std::pair<unsigned, bool>(0, false);
     511             : 
     512        1490 :   bool IdxNIsKill = hasTrivialKill(Idx);
     513             : 
     514             :   // If the index is smaller or larger than intptr_t, truncate or extend it.
     515        1490 :   MVT PtrVT = TLI.getPointerTy(DL);
     516        1490 :   EVT IdxVT = EVT::getEVT(Idx->getType(), /*HandleUnknown=*/false);
     517        1490 :   if (IdxVT.bitsLT(PtrVT)) {
     518          48 :     IdxN = fastEmit_r(IdxVT.getSimpleVT(), PtrVT, ISD::SIGN_EXTEND, IdxN,
     519          48 :                       IdxNIsKill);
     520             :     IdxNIsKill = true;
     521        1442 :   } else if (IdxVT.bitsGT(PtrVT)) {
     522             :     IdxN =
     523           9 :         fastEmit_r(IdxVT.getSimpleVT(), PtrVT, ISD::TRUNCATE, IdxN, IdxNIsKill);
     524             :     IdxNIsKill = true;
     525             :   }
     526        1490 :   return std::pair<unsigned, bool>(IdxN, IdxNIsKill);
     527             : }
     528             : 
     529     1027834 : void FastISel::recomputeInsertPt() {
     530     1027834 :   if (getLastLocalValue()) {
     531      703910 :     FuncInfo.InsertPt = getLastLocalValue();
     532     1407820 :     FuncInfo.MBB = FuncInfo.InsertPt->getParent();
     533      703910 :     ++FuncInfo.InsertPt;
     534             :   } else
     535      323924 :     FuncInfo.InsertPt = FuncInfo.MBB->getFirstNonPHI();
     536             : 
     537             :   // Now skip past any EH_LABELs, which must remain at the beginning.
     538     2839593 :   while (FuncInfo.InsertPt != FuncInfo.MBB->end() &&
     539      779677 :          FuncInfo.InsertPt->getOpcode() == TargetOpcode::EH_LABEL)
     540             :     ++FuncInfo.InsertPt;
     541     1027834 : }
     542             : 
     543         226 : void FastISel::removeDeadCode(MachineBasicBlock::iterator I,
     544             :                               MachineBasicBlock::iterator E) {
     545             :   assert(I.isValid() && E.isValid() && std::distance(I, E) > 0 &&
     546             :          "Invalid iterator!");
     547         577 :   while (I != E) {
     548             :     MachineInstr *Dead = &*I;
     549             :     ++I;
     550         351 :     Dead->eraseFromParent();
     551             :     ++NumFastIselDead;
     552             :   }
     553         226 :   recomputeInsertPt();
     554         226 : }
     555             : 
     556       43610 : FastISel::SavePoint FastISel::enterLocalValueArea() {
     557       43610 :   MachineBasicBlock::iterator OldInsertPt = FuncInfo.InsertPt;
     558             :   DebugLoc OldDL = DbgLoc;
     559       43610 :   recomputeInsertPt();
     560       87220 :   DbgLoc = DebugLoc();
     561       43610 :   SavePoint SP = {OldInsertPt, OldDL};
     562       43610 :   return SP;
     563             : }
     564             : 
     565       43610 : void FastISel::leaveLocalValueArea(SavePoint OldInsertPt) {
     566       87220 :   if (FuncInfo.InsertPt != FuncInfo.MBB->begin())
     567       43449 :     LastLocalValue = &*std::prev(FuncInfo.InsertPt);
     568             : 
     569             :   // Restore the previous insert position.
     570       43610 :   FuncInfo.InsertPt = OldInsertPt.InsertPt;
     571             :   DbgLoc = OldInsertPt.DL;
     572       43610 : }
     573             : 
     574        9423 : bool FastISel::selectBinaryOp(const User *I, unsigned ISDOpcode) {
     575        9423 :   EVT VT = EVT::getEVT(I->getType(), /*HandleUnknown=*/true);
     576        9423 :   if (VT == MVT::Other || !VT.isSimple())
     577             :     // Unhandled type. Halt "fast" selection and bail.
     578             :     return false;
     579             : 
     580             :   // We only handle legal types. For example, on x86-32 the instruction
     581             :   // selector contains all of the 64-bit instructions from x86-64,
     582             :   // under the assumption that i64 won't be used if the target doesn't
     583             :   // support it.
     584        9423 :   if (!TLI.isTypeLegal(VT)) {
     585             :     // MVT::i1 is special. Allow AND, OR, or XOR because they
     586             :     // don't require additional zeroing, which makes them easy.
     587         386 :     if (VT == MVT::i1 && (ISDOpcode == ISD::AND || ISDOpcode == ISD::OR ||
     588             :                           ISDOpcode == ISD::XOR))
     589         379 :       VT = TLI.getTypeToTransformTo(I->getContext(), VT);
     590             :     else
     591             :       return false;
     592             :   }
     593             : 
     594             :   // Check if the first operand is a constant, and handle it as "ri".  At -O0,
     595             :   // we don't have anything that canonicalizes operand order.
     596             :   if (const auto *CI = dyn_cast<ConstantInt>(I->getOperand(0)))
     597         500 :     if (isa<Instruction>(I) && cast<Instruction>(I)->isCommutative()) {
     598         352 :       unsigned Op1 = getRegForValue(I->getOperand(1));
     599         352 :       if (!Op1)
     600             :         return false;
     601         352 :       bool Op1IsKill = hasTrivialKill(I->getOperand(1));
     602             : 
     603             :       unsigned ResultReg =
     604         352 :           fastEmit_ri_(VT.getSimpleVT(), ISDOpcode, Op1, Op1IsKill,
     605         352 :                        CI->getZExtValue(), VT.getSimpleVT());
     606         352 :       if (!ResultReg)
     607             :         return false;
     608             : 
     609             :       // We successfully emitted code for the given LLVM Instruction.
     610         352 :       updateValueMap(I, ResultReg);
     611         352 :       return true;
     612             :     }
     613             : 
     614        8958 :   unsigned Op0 = getRegForValue(I->getOperand(0));
     615        8958 :   if (!Op0) // Unhandled operand. Halt "fast" selection and bail.
     616             :     return false;
     617        8948 :   bool Op0IsKill = hasTrivialKill(I->getOperand(0));
     618             : 
     619             :   // Check if the second operand is a constant and handle it appropriately.
     620             :   if (const auto *CI = dyn_cast<ConstantInt>(I->getOperand(1))) {
     621        4411 :     uint64_t Imm = CI->getSExtValue();
     622             : 
     623             :     // Transform "sdiv exact X, 8" -> "sra X, 3".
     624         781 :     if (ISDOpcode == ISD::SDIV && isa<BinaryOperator>(I) &&
     625        5192 :         cast<BinaryOperator>(I)->isExact() && isPowerOf2_64(Imm)) {
     626         691 :       Imm = Log2_64(Imm);
     627             :       ISDOpcode = ISD::SRA;
     628             :     }
     629             : 
     630             :     // Transform "urem x, pow2" -> "and x, pow2-1".
     631        3720 :     if (ISDOpcode == ISD::UREM && isa<BinaryOperator>(I) &&
     632             :         isPowerOf2_64(Imm)) {
     633             :       --Imm;
     634             :       ISDOpcode = ISD::AND;
     635             :     }
     636             : 
     637        4411 :     unsigned ResultReg = fastEmit_ri_(VT.getSimpleVT(), ISDOpcode, Op0,
     638        4411 :                                       Op0IsKill, Imm, VT.getSimpleVT());
     639        4411 :     if (!ResultReg)
     640             :       return false;
     641             : 
     642             :     // We successfully emitted code for the given LLVM Instruction.
     643        4181 :     updateValueMap(I, ResultReg);
     644        4181 :     return true;
     645             :   }
     646             : 
     647        4537 :   unsigned Op1 = getRegForValue(I->getOperand(1));
     648        4537 :   if (!Op1) // Unhandled operand. Halt "fast" selection and bail.
     649             :     return false;
     650        4500 :   bool Op1IsKill = hasTrivialKill(I->getOperand(1));
     651             : 
     652             :   // Now we have both operands in registers. Emit the instruction.
     653        4500 :   unsigned ResultReg = fastEmit_rr(VT.getSimpleVT(), VT.getSimpleVT(),
     654        9000 :                                    ISDOpcode, Op0, Op0IsKill, Op1, Op1IsKill);
     655        4500 :   if (!ResultReg)
     656             :     // Target-specific code wasn't able to find a machine opcode for
     657             :     // the given ISD opcode and type. Halt "fast" selection and bail.
     658             :     return false;
     659             : 
     660             :   // We successfully emitted code for the given LLVM Instruction.
     661        4015 :   updateValueMap(I, ResultReg);
     662        4015 :   return true;
     663             : }
     664             : 
     665       17345 : bool FastISel::selectGetElementPtr(const User *I) {
     666       17345 :   unsigned N = getRegForValue(I->getOperand(0));
     667       17345 :   if (!N) // Unhandled operand. Halt "fast" selection and bail.
     668             :     return false;
     669       17312 :   bool NIsKill = hasTrivialKill(I->getOperand(0));
     670             : 
     671             :   // Keep a running tab of the total offset to coalesce multiple N = N + Offset
     672             :   // into a single N = N + TotalOffset.
     673             :   uint64_t TotalOffs = 0;
     674             :   // FIXME: What's a good SWAG number for MaxOffs?
     675             :   uint64_t MaxOffs = 2048;
     676       34624 :   MVT VT = TLI.getPointerTy(DL);
     677       50205 :   for (gep_type_iterator GTI = gep_type_begin(I), E = gep_type_end(I);
     678       50205 :        GTI != E; ++GTI) {
     679             :     const Value *Idx = GTI.getOperand();
     680        8663 :     if (StructType *StTy = GTI.getStructTypeOrNull()) {
     681             :       uint64_t Field = cast<ConstantInt>(Idx)->getZExtValue();
     682        8663 :       if (Field) {
     683             :         // N = N + Offset
     684        2816 :         TotalOffs += DL.getStructLayout(StTy)->getElementOffset(Field);
     685        1408 :         if (TotalOffs >= MaxOffs) {
     686           0 :           N = fastEmit_ri_(VT, ISD::ADD, N, NIsKill, TotalOffs, VT);
     687           0 :           if (!N) // Unhandled operand. Halt "fast" selection and bail.
     688          17 :             return false;
     689             :           NIsKill = true;
     690             :           TotalOffs = 0;
     691             :         }
     692             :       }
     693             :     } else {
     694       24247 :       Type *Ty = GTI.getIndexedType();
     695             : 
     696             :       // If this is a constant subscript, handle it quickly.
     697             :       if (const auto *CI = dyn_cast<ConstantInt>(Idx)) {
     698       23374 :         if (CI->isZero())
     699       44841 :           continue;
     700             :         // N = N + Offset
     701        5721 :         uint64_t IdxN = CI->getValue().sextOrTrunc(64).getSExtValue();
     702        1907 :         TotalOffs += DL.getTypeAllocSize(Ty) * IdxN;
     703        1907 :         if (TotalOffs >= MaxOffs) {
     704         109 :           N = fastEmit_ri_(VT, ISD::ADD, N, NIsKill, TotalOffs, VT);
     705         109 :           if (!N) // Unhandled operand. Halt "fast" selection and bail.
     706          17 :             return false;
     707             :           NIsKill = true;
     708             :           TotalOffs = 0;
     709             :         }
     710        1907 :         continue;
     711             :       }
     712         873 :       if (TotalOffs) {
     713           0 :         N = fastEmit_ri_(VT, ISD::ADD, N, NIsKill, TotalOffs, VT);
     714           0 :         if (!N) // Unhandled operand. Halt "fast" selection and bail.
     715             :           return false;
     716             :         NIsKill = true;
     717             :         TotalOffs = 0;
     718             :       }
     719             : 
     720             :       // N = N + Idx * ElementSize;
     721         873 :       uint64_t ElementSize = DL.getTypeAllocSize(Ty);
     722         873 :       std::pair<unsigned, bool> Pair = getRegForGEPIndex(Idx);
     723         873 :       unsigned IdxN = Pair.first;
     724         873 :       bool IdxNIsKill = Pair.second;
     725         873 :       if (!IdxN) // Unhandled operand. Halt "fast" selection and bail.
     726             :         return false;
     727             : 
     728         858 :       if (ElementSize != 1) {
     729         410 :         IdxN = fastEmit_ri_(VT, ISD::MUL, IdxN, IdxNIsKill, ElementSize, VT);
     730         410 :         if (!IdxN) // Unhandled operand. Halt "fast" selection and bail.
     731             :           return false;
     732             :         IdxNIsKill = true;
     733             :       }
     734         856 :       N = fastEmit_rr(VT, VT, ISD::ADD, N, NIsKill, IdxN, IdxNIsKill);
     735         856 :       if (!N) // Unhandled operand. Halt "fast" selection and bail.
     736             :         return false;
     737             :     }
     738             :   }
     739       17295 :   if (TotalOffs) {
     740        3115 :     N = fastEmit_ri_(VT, ISD::ADD, N, NIsKill, TotalOffs, VT);
     741        3115 :     if (!N) // Unhandled operand. Halt "fast" selection and bail.
     742             :       return false;
     743             :   }
     744             : 
     745             :   // We successfully emitted code for the given LLVM Instruction.
     746       17295 :   updateValueMap(I, N);
     747       17295 :   return true;
     748             : }
     749             : 
     750          65 : bool FastISel::addStackMapLiveVars(SmallVectorImpl<MachineOperand> &Ops,
     751             :                                    const CallInst *CI, unsigned StartIdx) {
     752         368 :   for (unsigned i = StartIdx, e = CI->getNumArgOperands(); i != e; ++i) {
     753             :     Value *Val = CI->getArgOperand(i);
     754             :     // Check for constants and encode them with a StackMaps::ConstantOp prefix.
     755             :     if (const auto *C = dyn_cast<ConstantInt>(Val)) {
     756          36 :       Ops.push_back(MachineOperand::CreateImm(StackMaps::ConstantOp));
     757          18 :       Ops.push_back(MachineOperand::CreateImm(C->getSExtValue()));
     758         101 :     } else if (isa<ConstantPointerNull>(Val)) {
     759           0 :       Ops.push_back(MachineOperand::CreateImm(StackMaps::ConstantOp));
     760           0 :       Ops.push_back(MachineOperand::CreateImm(0));
     761             :     } else if (auto *AI = dyn_cast<AllocaInst>(Val)) {
     762             :       // Values coming from a stack location also require a special encoding,
     763             :       // but that is added later on by the target specific frame index
     764             :       // elimination implementation.
     765           8 :       auto SI = FuncInfo.StaticAllocaMap.find(AI);
     766          16 :       if (SI != FuncInfo.StaticAllocaMap.end())
     767          16 :         Ops.push_back(MachineOperand::CreateFI(SI->second));
     768             :       else
     769           0 :         return false;
     770             :     } else {
     771          93 :       unsigned Reg = getRegForValue(Val);
     772          93 :       if (!Reg)
     773             :         return false;
     774         186 :       Ops.push_back(MachineOperand::CreateReg(Reg, /*IsDef=*/false));
     775             :     }
     776             :   }
     777             :   return true;
     778             : }
     779             : 
     780          26 : bool FastISel::selectStackmap(const CallInst *I) {
     781             :   // void @llvm.experimental.stackmap(i64 <id>, i32 <numShadowBytes>,
     782             :   //                                  [live variables...])
     783             :   assert(I->getCalledFunction()->getReturnType()->isVoidTy() &&
     784             :          "Stackmap cannot return a value.");
     785             : 
     786             :   // The stackmap intrinsic only records the live variables (the arguments
     787             :   // passed to it) and emits NOPS (if requested). Unlike the patchpoint
     788             :   // intrinsic, this won't be lowered to a function call. This means we don't
     789             :   // have to worry about calling conventions and target-specific lowering code.
     790             :   // Instead we perform the call lowering right here.
     791             :   //
     792             :   // CALLSEQ_START(0, 0...)
     793             :   // STACKMAP(id, nbytes, ...)
     794             :   // CALLSEQ_END(0, 0)
     795             :   //
     796             :   SmallVector<MachineOperand, 32> Ops;
     797             : 
     798             :   // Add the <id> and <numBytes> constants.
     799             :   assert(isa<ConstantInt>(I->getOperand(PatchPointOpers::IDPos)) &&
     800             :          "Expected a constant integer.");
     801          26 :   const auto *ID = cast<ConstantInt>(I->getOperand(PatchPointOpers::IDPos));
     802          52 :   Ops.push_back(MachineOperand::CreateImm(ID->getZExtValue()));
     803             : 
     804             :   assert(isa<ConstantInt>(I->getOperand(PatchPointOpers::NBytesPos)) &&
     805             :          "Expected a constant integer.");
     806             :   const auto *NumBytes =
     807             :       cast<ConstantInt>(I->getOperand(PatchPointOpers::NBytesPos));
     808          52 :   Ops.push_back(MachineOperand::CreateImm(NumBytes->getZExtValue()));
     809             : 
     810             :   // Push live variables for the stack map (skipping the first two arguments
     811             :   // <id> and <numBytes>).
     812          26 :   if (!addStackMapLiveVars(Ops, I, 2))
     813             :     return false;
     814             : 
     815             :   // We are not adding any register mask info here, because the stackmap doesn't
     816             :   // clobber anything.
     817             : 
     818             :   // Add scratch registers as implicit def and early clobber.
     819             :   CallingConv::ID CC = I->getCallingConv();
     820          26 :   const MCPhysReg *ScratchRegs = TLI.getScratchRegisters(CC);
     821         138 :   for (unsigned i = 0; ScratchRegs[i]; ++i)
     822          56 :     Ops.push_back(MachineOperand::CreateReg(
     823             :         ScratchRegs[i], /*IsDef=*/true, /*IsImp=*/true, /*IsKill=*/false,
     824          56 :         /*IsDead=*/false, /*IsUndef=*/false, /*IsEarlyClobber=*/true));
     825             : 
     826             :   // Issue CALLSEQ_START
     827          26 :   unsigned AdjStackDown = TII.getCallFrameSetupOpcode();
     828             :   auto Builder =
     829          52 :       BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc, TII.get(AdjStackDown));
     830          26 :   const MCInstrDesc &MCID = Builder.getInstr()->getDesc();
     831         178 :   for (unsigned I = 0, E = MCID.getNumOperands(); I < E; ++I)
     832             :     Builder.addImm(0);
     833             : 
     834             :   // Issue STACKMAP.
     835          26 :   MachineInstrBuilder MIB = BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc,
     836          52 :                                     TII.get(TargetOpcode::STACKMAP));
     837         436 :   for (auto const &MO : Ops)
     838             :     MIB.add(MO);
     839             : 
     840             :   // Issue CALLSEQ_END
     841          26 :   unsigned AdjStackUp = TII.getCallFrameDestroyOpcode();
     842          52 :   BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc, TII.get(AdjStackUp))
     843             :       .addImm(0)
     844             :       .addImm(0);
     845             : 
     846             :   // Inform the Frame Information that we have a stackmap in this function.
     847          26 :   FuncInfo.MF->getFrameInfo().setHasStackMap();
     848             : 
     849          26 :   return true;
     850             : }
     851             : 
     852             : /// Lower an argument list according to the target calling convention.
     853             : ///
     854             : /// This is a helper for lowering intrinsics that follow a target calling
     855             : /// convention or require stack pointer adjustment. Only a subset of the
     856             : /// intrinsic's operands need to participate in the calling convention.
     857          39 : bool FastISel::lowerCallOperands(const CallInst *CI, unsigned ArgIdx,
     858             :                                  unsigned NumArgs, const Value *Callee,
     859             :                                  bool ForceRetVoidTy, CallLoweringInfo &CLI) {
     860             :   ArgListTy Args;
     861          39 :   Args.reserve(NumArgs);
     862             : 
     863             :   // Populate the argument list.
     864             :   ImmutableCallSite CS(CI);
     865         154 :   for (unsigned ArgI = ArgIdx, ArgE = ArgIdx + NumArgs; ArgI != ArgE; ++ArgI) {
     866         115 :     Value *V = CI->getOperand(ArgI);
     867             : 
     868             :     assert(!V->getType()->isEmptyTy() && "Empty type passed to intrinsic.");
     869             : 
     870             :     ArgListEntry Entry;
     871         115 :     Entry.Val = V;
     872         115 :     Entry.Ty = V->getType();
     873         115 :     Entry.setAttributes(&CS, ArgI);
     874         115 :     Args.push_back(Entry);
     875             :   }
     876             : 
     877          39 :   Type *RetTy = ForceRetVoidTy ? Type::getVoidTy(CI->getType()->getContext())
     878             :                                : CI->getType();
     879             :   CLI.setCallee(CI->getCallingConv(), RetTy, Callee, std::move(Args), NumArgs);
     880             : 
     881          78 :   return lowerCallTo(CLI);
     882             : }
     883             : 
     884          16 : FastISel::CallLoweringInfo &FastISel::CallLoweringInfo::setCallee(
     885             :     const DataLayout &DL, MCContext &Ctx, CallingConv::ID CC, Type *ResultTy,
     886             :     StringRef Target, ArgListTy &&ArgsList, unsigned FixedArgs) {
     887             :   SmallString<32> MangledName;
     888          16 :   Mangler::getNameWithPrefix(MangledName, Target, DL);
     889          16 :   MCSymbol *Sym = Ctx.getOrCreateSymbol(MangledName);
     890          16 :   return setCallee(CC, ResultTy, Sym, std::move(ArgsList), FixedArgs);
     891             : }
     892             : 
     893          39 : bool FastISel::selectPatchpoint(const CallInst *I) {
     894             :   // void|i64 @llvm.experimental.patchpoint.void|i64(i64 <id>,
     895             :   //                                                 i32 <numBytes>,
     896             :   //                                                 i8* <target>,
     897             :   //                                                 i32 <numArgs>,
     898             :   //                                                 [Args...],
     899             :   //                                                 [live variables...])
     900             :   CallingConv::ID CC = I->getCallingConv();
     901          39 :   bool IsAnyRegCC = CC == CallingConv::AnyReg;
     902          78 :   bool HasDef = !I->getType()->isVoidTy();
     903          39 :   Value *Callee = I->getOperand(PatchPointOpers::TargetPos)->stripPointerCasts();
     904             : 
     905             :   // Get the real number of arguments participating in the call <numArgs>
     906             :   assert(isa<ConstantInt>(I->getOperand(PatchPointOpers::NArgPos)) &&
     907             :          "Expected a constant integer.");
     908             :   const auto *NumArgsVal =
     909             :       cast<ConstantInt>(I->getOperand(PatchPointOpers::NArgPos));
     910          39 :   unsigned NumArgs = NumArgsVal->getZExtValue();
     911             : 
     912             :   // Skip the four meta args: <id>, <numNopBytes>, <target>, <numArgs>
     913             :   // This includes all meta-operands up to but not including CC.
     914             :   unsigned NumMetaOpers = PatchPointOpers::CCPos;
     915             :   assert(I->getNumArgOperands() >= NumMetaOpers + NumArgs &&
     916             :          "Not enough arguments provided to the patchpoint intrinsic");
     917             : 
     918             :   // For AnyRegCC the arguments are lowered later on manually.
     919          39 :   unsigned NumCallArgs = IsAnyRegCC ? 0 : NumArgs;
     920          78 :   CallLoweringInfo CLI;
     921             :   CLI.setIsPatchPoint();
     922          39 :   if (!lowerCallOperands(I, NumMetaOpers, NumCallArgs, Callee, IsAnyRegCC, CLI))
     923             :     return false;
     924             : 
     925             :   assert(CLI.Call && "No call instruction specified.");
     926             : 
     927             :   SmallVector<MachineOperand, 32> Ops;
     928             : 
     929             :   // Add an explicit result reg if we use the anyreg calling convention.
     930          39 :   if (IsAnyRegCC && HasDef) {
     931             :     assert(CLI.NumResultRegs == 0 && "Unexpected result register.");
     932           0 :     CLI.ResultReg = createResultReg(TLI.getRegClassFor(MVT::i64));
     933           0 :     CLI.NumResultRegs = 1;
     934           0 :     Ops.push_back(MachineOperand::CreateReg(CLI.ResultReg, /*IsDef=*/true));
     935             :   }
     936             : 
     937             :   // Add the <id> and <numBytes> constants.
     938             :   assert(isa<ConstantInt>(I->getOperand(PatchPointOpers::IDPos)) &&
     939             :          "Expected a constant integer.");
     940             :   const auto *ID = cast<ConstantInt>(I->getOperand(PatchPointOpers::IDPos));
     941          78 :   Ops.push_back(MachineOperand::CreateImm(ID->getZExtValue()));
     942             : 
     943             :   assert(isa<ConstantInt>(I->getOperand(PatchPointOpers::NBytesPos)) &&
     944             :          "Expected a constant integer.");
     945             :   const auto *NumBytes =
     946             :       cast<ConstantInt>(I->getOperand(PatchPointOpers::NBytesPos));
     947          78 :   Ops.push_back(MachineOperand::CreateImm(NumBytes->getZExtValue()));
     948             : 
     949             :   // Add the call target.
     950             :   if (const auto *C = dyn_cast<IntToPtrInst>(Callee)) {
     951             :     uint64_t CalleeConstAddr =
     952             :       cast<ConstantInt>(C->getOperand(0))->getZExtValue();
     953          36 :     Ops.push_back(MachineOperand::CreateImm(CalleeConstAddr));
     954             :   } else if (const auto *C = dyn_cast<ConstantExpr>(Callee)) {
     955           4 :     if (C->getOpcode() == Instruction::IntToPtr) {
     956             :       uint64_t CalleeConstAddr =
     957             :         cast<ConstantInt>(C->getOperand(0))->getZExtValue();
     958           8 :       Ops.push_back(MachineOperand::CreateImm(CalleeConstAddr));
     959             :     } else
     960           0 :       llvm_unreachable("Unsupported ConstantExpr.");
     961             :   } else if (const auto *GV = dyn_cast<GlobalValue>(Callee)) {
     962           4 :     Ops.push_back(MachineOperand::CreateGA(GV, 0));
     963          13 :   } else if (isa<ConstantPointerNull>(Callee))
     964          13 :     Ops.push_back(MachineOperand::CreateImm(0));
     965             :   else
     966           0 :     llvm_unreachable("Unsupported callee address.");
     967             : 
     968             :   // Adjust <numArgs> to account for any arguments that have been passed on
     969             :   // the stack instead.
     970          77 :   unsigned NumCallRegArgs = IsAnyRegCC ? NumArgs : CLI.OutRegs.size();
     971          78 :   Ops.push_back(MachineOperand::CreateImm(NumCallRegArgs));
     972             : 
     973             :   // Add the calling convention
     974          78 :   Ops.push_back(MachineOperand::CreateImm((unsigned)CC));
     975             : 
     976             :   // Add the arguments we omitted previously. The register allocator should
     977             :   // place these in any free register.
     978          39 :   if (IsAnyRegCC) {
     979           3 :     for (unsigned i = NumMetaOpers, e = NumMetaOpers + NumArgs; i != e; ++i) {
     980           2 :       unsigned Reg = getRegForValue(I->getArgOperand(i));
     981           2 :       if (!Reg)
     982             :         return false;
     983           2 :       Ops.push_back(MachineOperand::CreateReg(Reg, /*IsDef=*/false));
     984             :     }
     985             :   }
     986             : 
     987             :   // Push the arguments from the call instruction.
     988         141 :   for (auto Reg : CLI.OutRegs)
     989          51 :     Ops.push_back(MachineOperand::CreateReg(Reg, /*IsDef=*/false));
     990             : 
     991             :   // Push live variables for the stack map.
     992          39 :   if (!addStackMapLiveVars(Ops, I, NumMetaOpers + NumArgs))
     993             :     return false;
     994             : 
     995             :   // Push the register mask info.
     996          39 :   Ops.push_back(MachineOperand::CreateRegMask(
     997          39 :       TRI.getCallPreservedMask(*FuncInfo.MF, CC)));
     998             : 
     999             :   // Add scratch registers as implicit def and early clobber.
    1000          39 :   const MCPhysReg *ScratchRegs = TLI.getScratchRegisters(CC);
    1001         225 :   for (unsigned i = 0; ScratchRegs[i]; ++i)
    1002          93 :     Ops.push_back(MachineOperand::CreateReg(
    1003             :         ScratchRegs[i], /*IsDef=*/true, /*IsImp=*/true, /*IsKill=*/false,
    1004          93 :         /*IsDead=*/false, /*IsUndef=*/false, /*IsEarlyClobber=*/true));
    1005             : 
    1006             :   // Add implicit defs (return values).
    1007          81 :   for (auto Reg : CLI.InRegs)
    1008          21 :     Ops.push_back(MachineOperand::CreateReg(Reg, /*IsDef=*/true,
    1009             :                                             /*IsImpl=*/true));
    1010             : 
    1011             :   // Insert the patchpoint instruction before the call generated by the target.
    1012          39 :   MachineInstrBuilder MIB = BuildMI(*FuncInfo.MBB, CLI.Call, DbgLoc,
    1013          78 :                                     TII.get(TargetOpcode::PATCHPOINT));
    1014             : 
    1015         921 :   for (auto &MO : Ops)
    1016             :     MIB.add(MO);
    1017             : 
    1018          78 :   MIB->setPhysRegsDeadExcept(CLI.InRegs, TRI);
    1019             : 
    1020             :   // Delete the original call instruction.
    1021          39 :   CLI.Call->eraseFromParent();
    1022             : 
    1023             :   // Inform the Frame Information that we have a patchpoint in this function.
    1024          39 :   FuncInfo.MF->getFrameInfo().setHasPatchPoint();
    1025             : 
    1026          39 :   if (CLI.NumResultRegs)
    1027          21 :     updateValueMap(I, CLI.ResultReg, CLI.NumResultRegs);
    1028             :   return true;
    1029             : }
    1030             : 
    1031           0 : bool FastISel::selectXRayCustomEvent(const CallInst *I) {
    1032           0 :   const auto &Triple = TM.getTargetTriple();
    1033           0 :   if (Triple.getArch() != Triple::x86_64 || !Triple.isOSLinux())
    1034             :     return true; // don't do anything to this instruction.
    1035             :   SmallVector<MachineOperand, 8> Ops;
    1036           0 :   Ops.push_back(MachineOperand::CreateReg(getRegForValue(I->getArgOperand(0)),
    1037           0 :                                           /*IsDef=*/false));
    1038           0 :   Ops.push_back(MachineOperand::CreateReg(getRegForValue(I->getArgOperand(1)),
    1039           0 :                                           /*IsDef=*/false));
    1040             :   MachineInstrBuilder MIB =
    1041           0 :       BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc,
    1042           0 :               TII.get(TargetOpcode::PATCHABLE_EVENT_CALL));
    1043           0 :   for (auto &MO : Ops)
    1044             :     MIB.add(MO);
    1045             : 
    1046             :   // Insert the Patchable Event Call instruction, that gets lowered properly.
    1047             :   return true;
    1048             : }
    1049             : 
    1050           0 : bool FastISel::selectXRayTypedEvent(const CallInst *I) {
    1051           0 :   const auto &Triple = TM.getTargetTriple();
    1052           0 :   if (Triple.getArch() != Triple::x86_64 || !Triple.isOSLinux())
    1053             :     return true; // don't do anything to this instruction.
    1054             :   SmallVector<MachineOperand, 8> Ops;
    1055           0 :   Ops.push_back(MachineOperand::CreateReg(getRegForValue(I->getArgOperand(0)),
    1056           0 :                                           /*IsDef=*/false));
    1057           0 :   Ops.push_back(MachineOperand::CreateReg(getRegForValue(I->getArgOperand(1)),
    1058           0 :                                           /*IsDef=*/false));
    1059           0 :   Ops.push_back(MachineOperand::CreateReg(getRegForValue(I->getArgOperand(2)),
    1060           0 :                                           /*IsDef=*/false));
    1061             :   MachineInstrBuilder MIB =
    1062           0 :       BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc,
    1063           0 :               TII.get(TargetOpcode::PATCHABLE_TYPED_EVENT_CALL));
    1064           0 :   for (auto &MO : Ops)
    1065             :     MIB.add(MO);
    1066             : 
    1067             :   // Insert the Patchable Typed Event Call instruction, that gets lowered properly.
    1068             :   return true;
    1069             : }
    1070             : 
    1071             : /// Returns an AttributeList representing the attributes applied to the return
    1072             : /// value of the given call.
    1073       60783 : static AttributeList getReturnAttrs(FastISel::CallLoweringInfo &CLI) {
    1074             :   SmallVector<Attribute::AttrKind, 2> Attrs;
    1075       60783 :   if (CLI.RetSExt)
    1076          42 :     Attrs.push_back(Attribute::SExt);
    1077       60783 :   if (CLI.RetZExt)
    1078        1245 :     Attrs.push_back(Attribute::ZExt);
    1079       60783 :   if (CLI.IsInReg)
    1080           0 :     Attrs.push_back(Attribute::InReg);
    1081             : 
    1082       60783 :   return AttributeList::get(CLI.RetTy->getContext(), AttributeList::ReturnIndex,
    1083      121566 :                             Attrs);
    1084             : }
    1085             : 
    1086         176 : bool FastISel::lowerCallTo(const CallInst *CI, const char *SymName,
    1087             :                            unsigned NumArgs) {
    1088         176 :   MCContext &Ctx = MF->getContext();
    1089             :   SmallString<32> MangledName;
    1090         352 :   Mangler::getNameWithPrefix(MangledName, SymName, DL);
    1091         176 :   MCSymbol *Sym = Ctx.getOrCreateSymbol(MangledName);
    1092         352 :   return lowerCallTo(CI, Sym, NumArgs);
    1093             : }
    1094             : 
    1095         176 : bool FastISel::lowerCallTo(const CallInst *CI, MCSymbol *Symbol,
    1096             :                            unsigned NumArgs) {
    1097             :   ImmutableCallSite CS(CI);
    1098             : 
    1099             :   FunctionType *FTy = CS.getFunctionType();
    1100             :   Type *RetTy = CS.getType();
    1101             : 
    1102             :   ArgListTy Args;
    1103         176 :   Args.reserve(NumArgs);
    1104             : 
    1105             :   // Populate the argument list.
    1106             :   // Attributes for args start at offset 1, after the return attribute.
    1107        1232 :   for (unsigned ArgI = 0; ArgI != NumArgs; ++ArgI) {
    1108         528 :     Value *V = CI->getOperand(ArgI);
    1109             : 
    1110             :     assert(!V->getType()->isEmptyTy() && "Empty type passed to intrinsic.");
    1111             : 
    1112             :     ArgListEntry Entry;
    1113         528 :     Entry.Val = V;
    1114         528 :     Entry.Ty = V->getType();
    1115         528 :     Entry.setAttributes(&CS, ArgI);
    1116         528 :     Args.push_back(Entry);
    1117             :   }
    1118         352 :   TLI.markLibCallAttributes(MF, CS.getCallingConv(), Args);
    1119             : 
    1120         352 :   CallLoweringInfo CLI;
    1121         176 :   CLI.setCallee(RetTy, FTy, Symbol, std::move(Args), CS, NumArgs);
    1122             : 
    1123         352 :   return lowerCallTo(CLI);
    1124             : }
    1125             : 
    1126       60783 : bool FastISel::lowerCallTo(CallLoweringInfo &CLI) {
    1127             :   // Handle the incoming return values from the call.
    1128             :   CLI.clearIns();
    1129             :   SmallVector<EVT, 4> RetTys;
    1130       60783 :   ComputeValueVTs(TLI, DL, CLI.RetTy, RetTys);
    1131             : 
    1132             :   SmallVector<ISD::OutputArg, 4> Outs;
    1133       60783 :   GetReturnInfo(CLI.RetTy, getReturnAttrs(CLI), Outs, TLI, DL);
    1134             : 
    1135      243132 :   bool CanLowerReturn = TLI.CanLowerReturn(
    1136      243132 :       CLI.CallConv, *FuncInfo.MF, CLI.IsVarArg, Outs, CLI.RetTy->getContext());
    1137             : 
    1138             :   // FIXME: sret demotion isn't supported yet - bail out.
    1139       60783 :   if (!CanLowerReturn)
    1140             :     return false;
    1141             : 
    1142       95452 :   for (unsigned I = 0, E = RetTys.size(); I != E; ++I) {
    1143       69346 :     EVT VT = RetTys[I];
    1144       34673 :     MVT RegisterVT = TLI.getRegisterType(CLI.RetTy->getContext(), VT);
    1145       34673 :     unsigned NumRegs = TLI.getNumRegisters(CLI.RetTy->getContext(), VT);
    1146      104031 :     for (unsigned i = 0; i != NumRegs; ++i) {
    1147             :       ISD::InputArg MyFlags;
    1148       34679 :       MyFlags.VT = RegisterVT;
    1149       34679 :       MyFlags.ArgVT = VT;
    1150       34679 :       MyFlags.Used = CLI.IsReturnValueUsed;
    1151       34679 :       if (CLI.RetSExt)
    1152             :         MyFlags.Flags.setSExt();
    1153       34679 :       if (CLI.RetZExt)
    1154             :         MyFlags.Flags.setZExt();
    1155       34679 :       if (CLI.IsInReg)
    1156             :         MyFlags.Flags.setInReg();
    1157       34679 :       CLI.Ins.push_back(MyFlags);
    1158             :     }
    1159             :   }
    1160             : 
    1161             :   // Handle all of the outgoing arguments.
    1162             :   CLI.clearOuts();
    1163      143681 :   for (auto &Arg : CLI.getArgs()) {
    1164       82902 :     Type *FinalType = Arg.Ty;
    1165       82902 :     if (Arg.IsByVal)
    1166          43 :       FinalType = cast<PointerType>(Arg.Ty)->getElementType();
    1167      165804 :     bool NeedsRegBlock = TLI.functionArgumentNeedsConsecutiveRegisters(
    1168      165804 :         FinalType, CLI.CallConv, CLI.IsVarArg);
    1169             : 
    1170             :     ISD::ArgFlagsTy Flags;
    1171       82902 :     if (Arg.IsZExt)
    1172             :       Flags.setZExt();
    1173       82902 :     if (Arg.IsSExt)
    1174             :       Flags.setSExt();
    1175       82902 :     if (Arg.IsInReg)
    1176             :       Flags.setInReg();
    1177       82902 :     if (Arg.IsSRet)
    1178             :       Flags.setSRet();
    1179       82902 :     if (Arg.IsSwiftSelf)
    1180             :       Flags.setSwiftSelf();
    1181       82902 :     if (Arg.IsSwiftError)
    1182             :       Flags.setSwiftError();
    1183       82902 :     if (Arg.IsByVal)
    1184             :       Flags.setByVal();
    1185       82902 :     if (Arg.IsInAlloca) {
    1186             :       Flags.setInAlloca();
    1187             :       // Set the byval flag for CCAssignFn callbacks that don't know about
    1188             :       // inalloca. This way we can know how many bytes we should've allocated
    1189             :       // and how many bytes a callee cleanup function will pop.  If we port
    1190             :       // inalloca to more targets, we'll have to add custom inalloca handling in
    1191             :       // the various CC lowering callbacks.
    1192             :       Flags.setByVal();
    1193             :     }
    1194       82902 :     if (Arg.IsByVal || Arg.IsInAlloca) {
    1195          49 :       PointerType *Ty = cast<PointerType>(Arg.Ty);
    1196          49 :       Type *ElementTy = Ty->getElementType();
    1197          49 :       unsigned FrameSize = DL.getTypeAllocSize(ElementTy);
    1198             :       // For ByVal, alignment should come from FE. BE will guess if this info is
    1199             :       // not there, but there are cases it cannot get right.
    1200          49 :       unsigned FrameAlign = Arg.Alignment;
    1201          49 :       if (!FrameAlign)
    1202          11 :         FrameAlign = TLI.getByValTypeAlignment(ElementTy, DL);
    1203             :       Flags.setByValSize(FrameSize);
    1204             :       Flags.setByValAlign(FrameAlign);
    1205             :     }
    1206       82902 :     if (Arg.IsNest)
    1207             :       Flags.setNest();
    1208       82902 :     if (NeedsRegBlock)
    1209             :       Flags.setInConsecutiveRegs();
    1210       82902 :     unsigned OriginalAlignment = DL.getABITypeAlignment(Arg.Ty);
    1211             :     Flags.setOrigAlign(OriginalAlignment);
    1212             : 
    1213       82902 :     CLI.OutVals.push_back(Arg.Val);
    1214       82902 :     CLI.OutFlags.push_back(Flags);
    1215             :   }
    1216             : 
    1217       60779 :   if (!fastLowerCall(CLI))
    1218             :     return false;
    1219             : 
    1220             :   // Set all unused physreg defs as dead.
    1221             :   assert(CLI.Call && "No call instruction specified.");
    1222      119606 :   CLI.Call->setPhysRegsDeadExcept(CLI.InRegs, TRI);
    1223             : 
    1224       59803 :   if (CLI.NumResultRegs && CLI.CS)
    1225       67930 :     updateValueMap(CLI.CS->getInstruction(), CLI.ResultReg, CLI.NumResultRegs);
    1226             : 
    1227             :   return true;
    1228             : }
    1229             : 
    1230       60552 : bool FastISel::lowerCall(const CallInst *CI) {
    1231             :   ImmutableCallSite CS(CI);
    1232             : 
    1233             :   FunctionType *FuncTy = CS.getFunctionType();
    1234             :   Type *RetTy = CS.getType();
    1235             : 
    1236             :   ArgListTy Args;
    1237             :   ArgListEntry Entry;
    1238       60552 :   Args.reserve(CS.arg_size());
    1239             : 
    1240      142790 :   for (ImmutableCallSite::arg_iterator i = CS.arg_begin(), e = CS.arg_end();
    1241      142790 :        i != e; ++i) {
    1242       82238 :     Value *V = *i;
    1243             : 
    1244             :     // Skip empty types
    1245       82238 :     if (V->getType()->isEmptyTy())
    1246           0 :       continue;
    1247             : 
    1248       82238 :     Entry.Val = V;
    1249       82238 :     Entry.Ty = V->getType();
    1250             : 
    1251             :     // Skip the first return-type Attribute to get to params.
    1252      164476 :     Entry.setAttributes(&CS, i - CS.arg_begin());
    1253       82238 :     Args.push_back(Entry);
    1254             :   }
    1255             : 
    1256             :   // Check if target-independent constraints permit a tail call here.
    1257             :   // Target-dependent constraints are checked within fastLowerCall.
    1258             :   bool IsTailCall = CI->isTailCall();
    1259       60552 :   if (IsTailCall && !isInTailCallPosition(CS, TM))
    1260             :     IsTailCall = false;
    1261             : 
    1262      121104 :   CallLoweringInfo CLI;
    1263       60552 :   CLI.setCallee(RetTy, FuncTy, CI->getCalledValue(), std::move(Args), CS)
    1264             :       .setTailCall(IsTailCall);
    1265             : 
    1266      121104 :   return lowerCallTo(CLI);
    1267             : }
    1268             : 
    1269      124075 : bool FastISel::selectCall(const User *I) {
    1270             :   const CallInst *Call = cast<CallInst>(I);
    1271             : 
    1272             :   // Handle simple inline asms.
    1273             :   if (const InlineAsm *IA = dyn_cast<InlineAsm>(Call->getCalledValue())) {
    1274             :     // If the inline asm has side effects, then make sure that no local value
    1275             :     // lives across by flushing the local value map.
    1276         136 :     if (IA->hasSideEffects())
    1277         121 :       flushLocalValueMap();
    1278             : 
    1279             :     // Don't attempt to handle constraints.
    1280         136 :     if (!IA->getConstraintString().empty())
    1281             :       return false;
    1282             : 
    1283             :     unsigned ExtraInfo = 0;
    1284          10 :     if (IA->hasSideEffects())
    1285             :       ExtraInfo |= InlineAsm::Extra_HasSideEffects;
    1286          10 :     if (IA->isAlignStack())
    1287           0 :       ExtraInfo |= InlineAsm::Extra_IsAlignStack;
    1288             : 
    1289          10 :     BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc,
    1290          20 :             TII.get(TargetOpcode::INLINEASM))
    1291             :         .addExternalSymbol(IA->getAsmString().c_str())
    1292          10 :         .addImm(ExtraInfo);
    1293          10 :     return true;
    1294             :   }
    1295             : 
    1296      123939 :   MachineModuleInfo &MMI = FuncInfo.MF->getMMI();
    1297      123939 :   computeUsesVAFloatArgument(*Call, MMI);
    1298             : 
    1299             :   // Handle intrinsic function calls.
    1300             :   if (const auto *II = dyn_cast<IntrinsicInst>(Call))
    1301       63387 :     return selectIntrinsicCall(II);
    1302             : 
    1303             :   // Usually, it does not make sense to initialize a value,
    1304             :   // make an unrelated function call and use the value, because
    1305             :   // it tends to be spilled on the stack. So, we move the pointer
    1306             :   // to the last local value to the beginning of the block, so that
    1307             :   // all the values which have already been materialized,
    1308             :   // appear after the call. It also makes sense to skip intrinsics
    1309             :   // since they tend to be inlined.
    1310       60552 :   flushLocalValueMap();
    1311             : 
    1312       60552 :   return lowerCall(Call);
    1313             : }
    1314             : 
    1315       63387 : bool FastISel::selectIntrinsicCall(const IntrinsicInst *II) {
    1316       63387 :   switch (II->getIntrinsicID()) {
    1317             :   default:
    1318             :     break;
    1319             :   // At -O0 we don't care about the lifetime intrinsics.
    1320             :   case Intrinsic::lifetime_start:
    1321             :   case Intrinsic::lifetime_end:
    1322             :   // The donothing intrinsic does, well, nothing.
    1323             :   case Intrinsic::donothing:
    1324             :   // Neither does the sideeffect intrinsic.
    1325             :   case Intrinsic::sideeffect:
    1326             :   // Neither does the assume intrinsic; it's also OK not to codegen its operand.
    1327             :   case Intrinsic::assume:
    1328             :     return true;
    1329             :   case Intrinsic::dbg_declare: {
    1330             :     const DbgDeclareInst *DI = cast<DbgDeclareInst>(II);
    1331             :     assert(DI->getVariable() && "Missing variable");
    1332       57779 :     if (!FuncInfo.MF->getMMI().hasDebugInfo()) {
    1333             :       LLVM_DEBUG(dbgs() << "Dropping debug info for " << *DI << "\n");
    1334             :       return true;
    1335             :     }
    1336             : 
    1337             :     const Value *Address = DI->getAddress();
    1338      115558 :     if (!Address || isa<UndefValue>(Address)) {
    1339             :       LLVM_DEBUG(dbgs() << "Dropping debug info for " << *DI << "\n");
    1340             :       return true;
    1341             :     }
    1342             : 
    1343             :     // Byval arguments with frame indices were already handled after argument
    1344             :     // lowering and before isel.
    1345             :     const auto *Arg =
    1346       57776 :         dyn_cast<Argument>(Address->stripInBoundsConstantOffsets());
    1347         117 :     if (Arg && FuncInfo.getArgumentFrameIndex(Arg) != INT_MAX)
    1348             :       return true;
    1349             : 
    1350             :     Optional<MachineOperand> Op;
    1351       57742 :     if (unsigned Reg = lookUpRegForValue(Address))
    1352             :       Op = MachineOperand::CreateReg(Reg, false);
    1353             : 
    1354             :     // If we have a VLA that has a "use" in a metadata node that's then used
    1355             :     // here but it has no other uses, then we have a problem. E.g.,
    1356             :     //
    1357             :     //   int foo (const int *x) {
    1358             :     //     char a[*x];
    1359             :     //     return 0;
    1360             :     //   }
    1361             :     //
    1362             :     // If we assign 'a' a vreg and fast isel later on has to use the selection
    1363             :     // DAG isel, it will want to copy the value to the vreg. However, there are
    1364             :     // no uses, which goes counter to what selection DAG isel expects.
    1365      113556 :     if (!Op && !Address->use_empty() && isa<Instruction>(Address) &&
    1366             :         (!isa<AllocaInst>(Address) ||
    1367       55813 :          !FuncInfo.StaticAllocaMap.count(cast<AllocaInst>(Address))))
    1368           1 :       Op = MachineOperand::CreateReg(FuncInfo.InitializeRegForValue(Address),
    1369           1 :                                      false);
    1370             : 
    1371       57742 :     if (Op) {
    1372             :       assert(DI->getVariable()->isValidLocationForIntrinsic(DbgLoc) &&
    1373             :              "Expected inlined-at fields to agree");
    1374        1278 :       if (Op->isReg()) {
    1375             :         Op->setIsDebug(true);
    1376             :         // A dbg.declare describes the address of a source variable, so lower it
    1377             :         // into an indirect DBG_VALUE.
    1378        1278 :         BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc,
    1379        1278 :                 TII.get(TargetOpcode::DBG_VALUE), /*IsIndirect*/ true,
    1380        3834 :                 Op->getReg(), DI->getVariable(), DI->getExpression());
    1381             :       } else
    1382           0 :         BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc,
    1383           0 :                 TII.get(TargetOpcode::DBG_VALUE))
    1384             :             .add(*Op)
    1385             :             .addImm(0)
    1386             :             .addMetadata(DI->getVariable())
    1387             :             .addMetadata(DI->getExpression());
    1388             :     } else {
    1389             :       // We can't yet handle anything else here because it would require
    1390             :       // generating code, thus altering codegen because of debug info.
    1391             :       LLVM_DEBUG(dbgs() << "Dropping debug info for " << *DI << "\n");
    1392             :     }
    1393             :     return true;
    1394             :   }
    1395             :   case Intrinsic::dbg_value: {
    1396             :     // This form of DBG_VALUE is target-independent.
    1397             :     const DbgValueInst *DI = cast<DbgValueInst>(II);
    1398          65 :     const MCInstrDesc &II = TII.get(TargetOpcode::DBG_VALUE);
    1399             :     const Value *V = DI->getValue();
    1400             :     assert(DI->getVariable()->isValidLocationForIntrinsic(DbgLoc) &&
    1401             :            "Expected inlined-at fields to agree");
    1402          65 :     if (!V) {
    1403             :       // Currently the optimizer can produce this; insert an undef to
    1404             :       // help debugging.  Probably the optimizer should not do this.
    1405           0 :       BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc, II, false, 0U,
    1406           0 :               DI->getVariable(), DI->getExpression());
    1407             :     } else if (const auto *CI = dyn_cast<ConstantInt>(V)) {
    1408          14 :       if (CI->getBitWidth() > 64)
    1409           0 :         BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc, II)
    1410             :             .addCImm(CI)
    1411             :             .addImm(0U)
    1412             :             .addMetadata(DI->getVariable())
    1413             :             .addMetadata(DI->getExpression());
    1414             :       else
    1415          14 :         BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc, II)
    1416          14 :             .addImm(CI->getZExtValue())
    1417             :             .addImm(0U)
    1418             :             .addMetadata(DI->getVariable())
    1419             :             .addMetadata(DI->getExpression());
    1420             :     } else if (const auto *CF = dyn_cast<ConstantFP>(V)) {
    1421           0 :       BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc, II)
    1422             :           .addFPImm(CF)
    1423             :           .addImm(0U)
    1424             :           .addMetadata(DI->getVariable())
    1425             :           .addMetadata(DI->getExpression());
    1426          51 :     } else if (unsigned Reg = lookUpRegForValue(V)) {
    1427             :       // FIXME: This does not handle register-indirect values at offset 0.
    1428             :       bool IsIndirect = false;
    1429          36 :       BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc, II, IsIndirect, Reg,
    1430          36 :               DI->getVariable(), DI->getExpression());
    1431             :     } else {
    1432             :       // We can't yet handle anything else here because it would require
    1433             :       // generating code, thus altering codegen because of debug info.
    1434             :       LLVM_DEBUG(dbgs() << "Dropping debug info for " << *DI << "\n");
    1435             :     }
    1436             :     return true;
    1437             :   }
    1438           3 :   case Intrinsic::objectsize: {
    1439           3 :     ConstantInt *CI = cast<ConstantInt>(II->getArgOperand(1));
    1440           3 :     unsigned long long Res = CI->isZero() ? -1ULL : 0;
    1441           3 :     Constant *ResCI = ConstantInt::get(II->getType(), Res);
    1442           3 :     unsigned ResultReg = getRegForValue(ResCI);
    1443           3 :     if (!ResultReg)
    1444             :       return false;
    1445           3 :     updateValueMap(II, ResultReg);
    1446           3 :     return true;
    1447             :   }
    1448           7 :   case Intrinsic::launder_invariant_group:
    1449             :   case Intrinsic::expect: {
    1450          14 :     unsigned ResultReg = getRegForValue(II->getArgOperand(0));
    1451           7 :     if (!ResultReg)
    1452             :       return false;
    1453           7 :     updateValueMap(II, ResultReg);
    1454           7 :     return true;
    1455             :   }
    1456          26 :   case Intrinsic::experimental_stackmap:
    1457          26 :     return selectStackmap(II);
    1458          39 :   case Intrinsic::experimental_patchpoint_void:
    1459             :   case Intrinsic::experimental_patchpoint_i64:
    1460          39 :     return selectPatchpoint(II);
    1461             : 
    1462           0 :   case Intrinsic::xray_customevent:
    1463           0 :     return selectXRayCustomEvent(II);
    1464           0 :   case Intrinsic::xray_typedevent:
    1465           0 :     return selectXRayTypedEvent(II);
    1466             :   }
    1467             : 
    1468        5454 :   return fastLowerIntrinsicCall(II);
    1469             : }
    1470             : 
    1471        2881 : bool FastISel::selectCast(const User *I, unsigned Opcode) {
    1472        5762 :   EVT SrcVT = TLI.getValueType(DL, I->getOperand(0)->getType());
    1473        2881 :   EVT DstVT = TLI.getValueType(DL, I->getType());
    1474             : 
    1475        5760 :   if (SrcVT == MVT::Other || !SrcVT.isSimple() || DstVT == MVT::Other ||
    1476             :       !DstVT.isSimple())
    1477             :     // Unhandled type. Halt "fast" selection and bail.
    1478             :     return false;
    1479             : 
    1480             :   // Check if the destination type is legal.
    1481        2879 :   if (!TLI.isTypeLegal(DstVT))
    1482             :     return false;
    1483             : 
    1484             :   // Check if the source operand is legal.
    1485             :   if (!TLI.isTypeLegal(SrcVT))
    1486             :     return false;
    1487             : 
    1488        1874 :   unsigned InputReg = getRegForValue(I->getOperand(0));
    1489        1874 :   if (!InputReg)
    1490             :     // Unhandled operand.  Halt "fast" selection and bail.
    1491             :     return false;
    1492             : 
    1493        1871 :   bool InputRegIsKill = hasTrivialKill(I->getOperand(0));
    1494             : 
    1495        1871 :   unsigned ResultReg = fastEmit_r(SrcVT.getSimpleVT(), DstVT.getSimpleVT(),
    1496        3742 :                                   Opcode, InputReg, InputRegIsKill);
    1497        1871 :   if (!ResultReg)
    1498             :     return false;
    1499             : 
    1500        1716 :   updateValueMap(I, ResultReg);
    1501        1716 :   return true;
    1502             : }
    1503             : 
    1504       18257 : bool FastISel::selectBitCast(const User *I) {
    1505             :   // If the bitcast doesn't change the type, just use the operand value.
    1506       36514 :   if (I->getType() == I->getOperand(0)->getType()) {
    1507           3 :     unsigned Reg = getRegForValue(I->getOperand(0));
    1508           3 :     if (!Reg)
    1509             :       return false;
    1510           3 :     updateValueMap(I, Reg);
    1511           3 :     return true;
    1512             :   }
    1513             : 
    1514             :   // Bitcasts of other values become reg-reg copies or BITCAST operators.
    1515       18254 :   EVT SrcEVT = TLI.getValueType(DL, I->getOperand(0)->getType());
    1516       18254 :   EVT DstEVT = TLI.getValueType(DL, I->getType());
    1517             :   if (SrcEVT == MVT::Other || DstEVT == MVT::Other ||
    1518       18254 :       !TLI.isTypeLegal(SrcEVT) || !TLI.isTypeLegal(DstEVT))
    1519             :     // Unhandled type. Halt "fast" selection and bail.
    1520             :     return false;
    1521             : 
    1522             :   MVT SrcVT = SrcEVT.getSimpleVT();
    1523             :   MVT DstVT = DstEVT.getSimpleVT();
    1524       18215 :   unsigned Op0 = getRegForValue(I->getOperand(0));
    1525       18215 :   if (!Op0) // Unhandled operand. Halt "fast" selection and bail.
    1526             :     return false;
    1527       18207 :   bool Op0IsKill = hasTrivialKill(I->getOperand(0));
    1528             : 
    1529             :   // First, try to perform the bitcast by inserting a reg-reg copy.
    1530             :   unsigned ResultReg = 0;
    1531       18207 :   if (SrcVT == DstVT) {
    1532       14272 :     const TargetRegisterClass *SrcClass = TLI.getRegClassFor(SrcVT);
    1533       14272 :     const TargetRegisterClass *DstClass = TLI.getRegClassFor(DstVT);
    1534             :     // Don't attempt a cross-class copy. It will likely fail.
    1535       14272 :     if (SrcClass == DstClass) {
    1536       14272 :       ResultReg = createResultReg(DstClass);
    1537       28544 :       BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc,
    1538       42816 :               TII.get(TargetOpcode::COPY), ResultReg).addReg(Op0);
    1539             :     }
    1540             :   }
    1541             : 
    1542             :   // If the reg-reg copy failed, select a BITCAST opcode.
    1543       14272 :   if (!ResultReg)
    1544        3935 :     ResultReg = fastEmit_r(SrcVT, DstVT, ISD::BITCAST, Op0, Op0IsKill);
    1545             : 
    1546       18207 :   if (!ResultReg)
    1547             :     return false;
    1548             : 
    1549       14292 :   updateValueMap(I, ResultReg);
    1550       14292 :   return true;
    1551             : }
    1552             : 
    1553             : // Remove local value instructions starting from the instruction after
    1554             : // SavedLastLocalValue to the current function insert point.
    1555       16302 : void FastISel::removeDeadLocalValueCode(MachineInstr *SavedLastLocalValue)
    1556             : {
    1557       16302 :   MachineInstr *CurLastLocalValue = getLastLocalValue();
    1558       16302 :   if (CurLastLocalValue != SavedLastLocalValue) {
    1559             :     // Find the first local value instruction to be deleted. 
    1560             :     // This is the instruction after SavedLastLocalValue if it is non-NULL.
    1561             :     // Otherwise it's the first instruction in the block.
    1562             :     MachineBasicBlock::iterator FirstDeadInst(SavedLastLocalValue);
    1563          28 :     if (SavedLastLocalValue)
    1564             :       ++FirstDeadInst;
    1565             :     else
    1566          23 :       FirstDeadInst = FuncInfo.MBB->getFirstNonPHI();
    1567             :     setLastLocalValue(SavedLastLocalValue);
    1568          28 :     removeDeadCode(FirstDeadInst, FuncInfo.InsertPt);
    1569             :   }
    1570       16302 : }
    1571             : 
    1572      441554 : bool FastISel::selectInstruction(const Instruction *I) {
    1573      441554 :   MachineInstr *SavedLastLocalValue = getLastLocalValue();
    1574             :   // Just before the terminator instruction, insert instructions to
    1575             :   // feed PHI nodes in successor blocks.
    1576      441554 :   if (isa<TerminatorInst>(I)) {
    1577      102927 :     if (!handlePHINodesInSuccessorBlocks(I->getParent())) {
    1578             :       // PHI node handling may have generated local value instructions,
    1579             :       // even though it failed to handle all PHI nodes.
    1580             :       // We remove these instructions because SelectionDAGISel will generate 
    1581             :       // them again.
    1582          58 :       removeDeadLocalValueCode(SavedLastLocalValue);
    1583          58 :       return false;
    1584             :     }
    1585             :   }
    1586             : 
    1587             :   // FastISel does not handle any operand bundles except OB_funclet.
    1588      441496 :   if (ImmutableCallSite CS = ImmutableCallSite(I))
    1589      138672 :     for (unsigned i = 0, e = CS.getNumOperandBundles(); i != e; ++i)
    1590           0 :       if (CS.getOperandBundleAt(i).getTagID() != LLVMContext::OB_funclet)
    1591           0 :         return false;
    1592             : 
    1593             :   DbgLoc = I->getDebugLoc();
    1594             : 
    1595      441496 :   SavedInsertPt = FuncInfo.InsertPt;
    1596             : 
    1597             :   if (const auto *Call = dyn_cast<CallInst>(I)) {
    1598             :     const Function *F = Call->getCalledFunction();
    1599             :     LibFunc Func;
    1600             : 
    1601             :     // As a special case, don't handle calls to builtin library functions that
    1602             :     // may be translated directly to target instructions.
    1603      242338 :     if (F && !F->hasLocalLinkage() && F->hasName() &&
    1604      127758 :         LibInfo->getLibFunc(F->getName(), Func) &&
    1605        6589 :         LibInfo->hasOptimizedCodeGen(Func))
    1606          12 :       return false;
    1607             : 
    1608             :     // Don't handle Intrinsic::trap if a trap function is specified.
    1609      124071 :     if (F && F->getIntrinsicID() == Intrinsic::trap &&
    1610          25 :         Call->hasFnAttr("trap-func-name"))
    1611             :       return false;
    1612             :   }
    1613             : 
    1614             :   // First, try doing target-independent selection.
    1615      441484 :   if (!SkipTargetIndependentISel) {
    1616      437272 :     if (selectOperator(I, I->getOpcode())) {
    1617             :       ++NumFastIselSuccessIndependent;
    1618      382666 :       DbgLoc = DebugLoc();
    1619      191333 :       return true;
    1620             :     }
    1621             :     // Remove dead code.
    1622      245939 :     recomputeInsertPt();
    1623      491878 :     if (SavedInsertPt != FuncInfo.InsertPt)
    1624          51 :       removeDeadCode(FuncInfo.InsertPt, SavedInsertPt);
    1625      245939 :     SavedInsertPt = FuncInfo.InsertPt;
    1626             :   }
    1627             :   // Next, try calling the target to attempt to handle the instruction.
    1628      250151 :   if (fastSelectInstruction(I)) {
    1629             :     ++NumFastIselSuccessTarget;
    1630      440342 :     DbgLoc = DebugLoc();
    1631      220171 :     return true;
    1632             :   }
    1633             :   // Remove dead code.
    1634       29980 :   recomputeInsertPt();
    1635       59960 :   if (SavedInsertPt != FuncInfo.InsertPt)
    1636          15 :     removeDeadCode(FuncInfo.InsertPt, SavedInsertPt);
    1637             : 
    1638       59960 :   DbgLoc = DebugLoc();
    1639             :   // Undo phi node updates, because they will be added again by SelectionDAG.
    1640       29980 :   if (isa<TerminatorInst>(I)) {
    1641             :     // PHI node handling may have generated local value instructions. 
    1642             :     // We remove them because SelectionDAGISel will generate them again.
    1643       16244 :     removeDeadLocalValueCode(SavedLastLocalValue);
    1644       16244 :     FuncInfo.PHINodesToUpdate.resize(FuncInfo.OrigNumPHINodesToUpdate);
    1645             :   }
    1646             :   return false;
    1647             : }
    1648             : 
    1649             : /// Emit an unconditional branch to the given block, unless it is the immediate
    1650             : /// (fall-through) successor, and update the CFG.
    1651       31499 : void FastISel::fastEmitBranch(MachineBasicBlock *MSucc,
    1652             :                               const DebugLoc &DbgLoc) {
    1653       88945 :   if (FuncInfo.MBB->getBasicBlock()->size() > 1 &&
    1654       25947 :       FuncInfo.MBB->isLayoutSuccessor(MSucc)) {
    1655             :     // For more accurate line information if this is the only instruction
    1656             :     // in the block then emit it, otherwise we have the unconditional
    1657             :     // fall-through case, which needs no instructions.
    1658             :   } else {
    1659             :     // The unconditional branch case.
    1660       27072 :     TII.insertBranch(*FuncInfo.MBB, MSucc, nullptr,
    1661       27072 :                      SmallVector<MachineOperand, 0>(), DbgLoc);
    1662             :   }
    1663       31499 :   if (FuncInfo.BPI) {
    1664             :     auto BranchProbability = FuncInfo.BPI->getEdgeProbability(
    1665         244 :         FuncInfo.MBB->getBasicBlock(), MSucc->getBasicBlock());
    1666         244 :     FuncInfo.MBB->addSuccessor(MSucc, BranchProbability);
    1667             :   } else
    1668       31255 :     FuncInfo.MBB->addSuccessorWithoutProb(MSucc);
    1669       31499 : }
    1670             : 
    1671       10371 : void FastISel::finishCondBranch(const BasicBlock *BranchBB,
    1672             :                                 MachineBasicBlock *TrueMBB,
    1673             :                                 MachineBasicBlock *FalseMBB) {
    1674             :   // Add TrueMBB as successor unless it is equal to the FalseMBB: This can
    1675             :   // happen in degenerate IR and MachineIR forbids to have a block twice in the
    1676             :   // successor/predecessor lists.
    1677       10371 :   if (TrueMBB != FalseMBB) {
    1678       10368 :     if (FuncInfo.BPI) {
    1679             :       auto BranchProbability =
    1680         162 :           FuncInfo.BPI->getEdgeProbability(BranchBB, TrueMBB->getBasicBlock());
    1681         162 :       FuncInfo.MBB->addSuccessor(TrueMBB, BranchProbability);
    1682             :     } else
    1683       10206 :       FuncInfo.MBB->addSuccessorWithoutProb(TrueMBB);
    1684             :   }
    1685             : 
    1686       10371 :   fastEmitBranch(FalseMBB, DbgLoc);
    1687       10371 : }
    1688             : 
    1689             : /// Emit an FNeg operation.
    1690          74 : bool FastISel::selectFNeg(const User *I) {
    1691          74 :   unsigned OpReg = getRegForValue(BinaryOperator::getFNegArgument(I));
    1692          74 :   if (!OpReg)
    1693             :     return false;
    1694          74 :   bool OpRegIsKill = hasTrivialKill(I);
    1695             : 
    1696             :   // If the target has ISD::FNEG, use it.
    1697          74 :   EVT VT = TLI.getValueType(DL, I->getType());
    1698          74 :   unsigned ResultReg = fastEmit_r(VT.getSimpleVT(), VT.getSimpleVT(), ISD::FNEG,
    1699         148 :                                   OpReg, OpRegIsKill);
    1700          74 :   if (ResultReg) {
    1701           0 :     updateValueMap(I, ResultReg);
    1702           0 :     return true;
    1703             :   }
    1704             : 
    1705             :   // Bitcast the value to integer, twiddle the sign bit with xor,
    1706             :   // and then bitcast it back to floating-point.
    1707          74 :   if (VT.getSizeInBits() > 64)
    1708             :     return false;
    1709           2 :   EVT IntVT = EVT::getIntegerVT(I->getContext(), VT.getSizeInBits());
    1710           2 :   if (!TLI.isTypeLegal(IntVT))
    1711             :     return false;
    1712             : 
    1713             :   unsigned IntReg = fastEmit_r(VT.getSimpleVT(), IntVT.getSimpleVT(),
    1714           2 :                                ISD::BITCAST, OpReg, OpRegIsKill);
    1715           2 :   if (!IntReg)
    1716             :     return false;
    1717             : 
    1718           2 :   unsigned IntResultReg = fastEmit_ri_(
    1719             :       IntVT.getSimpleVT(), ISD::XOR, IntReg, /*IsKill=*/true,
    1720           4 :       UINT64_C(1) << (VT.getSizeInBits() - 1), IntVT.getSimpleVT());
    1721           2 :   if (!IntResultReg)
    1722             :     return false;
    1723             : 
    1724           2 :   ResultReg = fastEmit_r(IntVT.getSimpleVT(), VT.getSimpleVT(), ISD::BITCAST,
    1725           2 :                          IntResultReg, /*IsKill=*/true);
    1726           2 :   if (!ResultReg)
    1727             :     return false;
    1728             : 
    1729           2 :   updateValueMap(I, ResultReg);
    1730           2 :   return true;
    1731             : }
    1732             : 
    1733        8514 : bool FastISel::selectExtractValue(const User *U) {
    1734             :   const ExtractValueInst *EVI = dyn_cast<ExtractValueInst>(U);
    1735             :   if (!EVI)
    1736             :     return false;
    1737             : 
    1738             :   // Make sure we only try to handle extracts with a legal result.  But also
    1739             :   // allow i1 because it's easy.
    1740        8514 :   EVT RealVT = TLI.getValueType(DL, EVI->getType(), /*AllowUnknown=*/true);
    1741        8514 :   if (!RealVT.isSimple())
    1742             :     return false;
    1743             :   MVT VT = RealVT.getSimpleVT();
    1744        8721 :   if (!TLI.isTypeLegal(VT) && VT != MVT::i1)
    1745             :     return false;
    1746             : 
    1747             :   const Value *Op0 = EVI->getOperand(0);
    1748        8510 :   Type *AggTy = Op0->getType();
    1749             : 
    1750             :   // Get the base result register.
    1751             :   unsigned ResultReg;
    1752        8510 :   DenseMap<const Value *, unsigned>::iterator I = FuncInfo.ValueMap.find(Op0);
    1753       17020 :   if (I != FuncInfo.ValueMap.end())
    1754        3109 :     ResultReg = I->second;
    1755        5401 :   else if (isa<Instruction>(Op0))
    1756        5398 :     ResultReg = FuncInfo.InitializeRegForValue(Op0);
    1757             :   else
    1758             :     return false; // fast-isel can't handle aggregate constants at the moment
    1759             : 
    1760             :   // Get the actual result register, which is an offset from the base register.
    1761             :   unsigned VTIndex = ComputeLinearIndex(AggTy, EVI->getIndices());
    1762             : 
    1763             :   SmallVector<EVT, 4> AggValueVTs;
    1764        8507 :   ComputeValueVTs(TLI, DL, AggTy, AggValueVTs);
    1765             : 
    1766       14881 :   for (unsigned i = 0; i < VTIndex; i++)
    1767        6374 :     ResultReg += TLI.getNumRegisters(FuncInfo.Fn->getContext(), AggValueVTs[i]);
    1768             : 
    1769        8507 :   updateValueMap(EVI, ResultReg);
    1770             :   return true;
    1771             : }
    1772             : 
    1773      445960 : bool FastISel::selectOperator(const User *I, unsigned Opcode) {
    1774      445960 :   switch (Opcode) {
    1775        3689 :   case Instruction::Add:
    1776        3689 :     return selectBinaryOp(I, ISD::ADD);
    1777         271 :   case Instruction::FAdd:
    1778         271 :     return selectBinaryOp(I, ISD::FADD);
    1779        1593 :   case Instruction::Sub:
    1780        1593 :     return selectBinaryOp(I, ISD::SUB);
    1781         129 :   case Instruction::FSub:
    1782             :     // FNeg is currently represented in LLVM IR as a special case of FSub.
    1783         129 :     if (BinaryOperator::isFNeg(I))
    1784          74 :       return selectFNeg(I);
    1785          55 :     return selectBinaryOp(I, ISD::FSUB);
    1786         998 :   case Instruction::Mul:
    1787         998 :     return selectBinaryOp(I, ISD::MUL);
    1788          38 :   case Instruction::FMul:
    1789          38 :     return selectBinaryOp(I, ISD::FMUL);
    1790         801 :   case Instruction::SDiv:
    1791         801 :     return selectBinaryOp(I, ISD::SDIV);
    1792         179 :   case Instruction::UDiv:
    1793         179 :     return selectBinaryOp(I, ISD::UDIV);
    1794          26 :   case Instruction::FDiv:
    1795          26 :     return selectBinaryOp(I, ISD::FDIV);
    1796          94 :   case Instruction::SRem:
    1797          94 :     return selectBinaryOp(I, ISD::SREM);
    1798         115 :   case Instruction::URem:
    1799         115 :     return selectBinaryOp(I, ISD::UREM);
    1800           1 :   case Instruction::FRem:
    1801           1 :     return selectBinaryOp(I, ISD::FREM);
    1802         175 :   case Instruction::Shl:
    1803         175 :     return selectBinaryOp(I, ISD::SHL);
    1804          45 :   case Instruction::LShr:
    1805          45 :     return selectBinaryOp(I, ISD::SRL);
    1806          31 :   case Instruction::AShr:
    1807          31 :     return selectBinaryOp(I, ISD::SRA);
    1808         491 :   case Instruction::And:
    1809         491 :     return selectBinaryOp(I, ISD::AND);
    1810         289 :   case Instruction::Or:
    1811         289 :     return selectBinaryOp(I, ISD::OR);
    1812         516 :   case Instruction::Xor:
    1813         516 :     return selectBinaryOp(I, ISD::XOR);
    1814             : 
    1815       17345 :   case Instruction::GetElementPtr:
    1816       17345 :     return selectGetElementPtr(I);
    1817             : 
    1818             :   case Instruction::Br: {
    1819             :     const BranchInst *BI = cast<BranchInst>(I);
    1820             : 
    1821       31271 :     if (BI->isUnconditional()) {
    1822       20937 :       const BasicBlock *LLVMSucc = BI->getSuccessor(0);
    1823       41874 :       MachineBasicBlock *MSucc = FuncInfo.MBBMap[LLVMSucc];
    1824       20937 :       fastEmitBranch(MSucc, BI->getDebugLoc());
    1825             :       return true;
    1826             :     }
    1827             : 
    1828             :     // Conditional branches are not handed yet.
    1829             :     // Halt "fast" selection and bail.
    1830             :     return false;
    1831             :   }
    1832             : 
    1833        6322 :   case Instruction::Unreachable:
    1834        6322 :     if (TM.Options.TrapUnreachable)
    1835         108 :       return fastEmit_(MVT::Other, MVT::Other, ISD::TRAP) != 0;
    1836             :     else
    1837             :       return true;
    1838             : 
    1839          16 :   case Instruction::Alloca:
    1840             :     // FunctionLowering has the static-sized case covered.
    1841          16 :     if (FuncInfo.StaticAllocaMap.count(cast<AllocaInst>(I)))
    1842             :       return true;
    1843             : 
    1844             :     // Dynamic-sized alloca is not handled yet.
    1845             :     return false;
    1846             : 
    1847      124022 :   case Instruction::Call:
    1848      124022 :     return selectCall(I);
    1849             : 
    1850       18205 :   case Instruction::BitCast:
    1851       18205 :     return selectBitCast(I);
    1852             : 
    1853          32 :   case Instruction::FPToSI:
    1854          32 :     return selectCast(I, ISD::FP_TO_SINT);
    1855         669 :   case Instruction::ZExt:
    1856         669 :     return selectCast(I, ISD::ZERO_EXTEND);
    1857        1605 :   case Instruction::SExt:
    1858        1605 :     return selectCast(I, ISD::SIGN_EXTEND);
    1859         393 :   case Instruction::Trunc:
    1860         393 :     return selectCast(I, ISD::TRUNCATE);
    1861         115 :   case Instruction::SIToFP:
    1862         115 :     return selectCast(I, ISD::SINT_TO_FP);
    1863             : 
    1864        1857 :   case Instruction::IntToPtr: // Deliberate fall-through.
    1865             :   case Instruction::PtrToInt: {
    1866        3714 :     EVT SrcVT = TLI.getValueType(DL, I->getOperand(0)->getType());
    1867        1857 :     EVT DstVT = TLI.getValueType(DL, I->getType());
    1868        1857 :     if (DstVT.bitsGT(SrcVT))
    1869           3 :       return selectCast(I, ISD::ZERO_EXTEND);
    1870        1854 :     if (DstVT.bitsLT(SrcVT))
    1871          17 :       return selectCast(I, ISD::TRUNCATE);
    1872        1837 :     unsigned Reg = getRegForValue(I->getOperand(0));
    1873        1837 :     if (!Reg)
    1874             :       return false;
    1875        1832 :     updateValueMap(I, Reg);
    1876        1832 :     return true;
    1877             :   }
    1878             : 
    1879        8514 :   case Instruction::ExtractValue:
    1880        8514 :     return selectExtractValue(I);
    1881             : 
    1882           0 :   case Instruction::PHI:
    1883           0 :     llvm_unreachable("FastISel shouldn't visit PHI nodes!");
    1884             : 
    1885             :   default:
    1886             :     // Unhandled instruction. Halt "fast" selection and bail.
    1887             :     return false;
    1888             :   }
    1889             : }
    1890             : 
    1891       49900 : FastISel::FastISel(FunctionLoweringInfo &FuncInfo,
    1892             :                    const TargetLibraryInfo *LibInfo,
    1893       49900 :                    bool SkipTargetIndependentISel)
    1894       99800 :     : FuncInfo(FuncInfo), MF(FuncInfo.MF), MRI(FuncInfo.MF->getRegInfo()),
    1895       99800 :       MFI(FuncInfo.MF->getFrameInfo()), MCP(*FuncInfo.MF->getConstantPool()),
    1896       99800 :       TM(FuncInfo.MF->getTarget()), DL(MF->getDataLayout()),
    1897       49900 :       TII(*MF->getSubtarget().getInstrInfo()),
    1898       49900 :       TLI(*MF->getSubtarget().getTargetLowering()),
    1899       49900 :       TRI(*MF->getSubtarget().getRegisterInfo()), LibInfo(LibInfo),
    1900      598800 :       SkipTargetIndependentISel(SkipTargetIndependentISel) {}
    1901             : 
    1902             : FastISel::~FastISel() = default;
    1903             : 
    1904           0 : bool FastISel::fastLowerArguments() { return false; }
    1905             : 
    1906         594 : bool FastISel::fastLowerCall(CallLoweringInfo & /*CLI*/) { return false; }
    1907             : 
    1908         125 : bool FastISel::fastLowerIntrinsicCall(const IntrinsicInst * /*II*/) {
    1909         125 :   return false;
    1910             : }
    1911             : 
    1912          45 : unsigned FastISel::fastEmit_(MVT, MVT, unsigned) { return 0; }
    1913             : 
    1914           0 : unsigned FastISel::fastEmit_r(MVT, MVT, unsigned, unsigned /*Op0*/,
    1915             :                               bool /*Op0IsKill*/) {
    1916           0 :   return 0;
    1917             : }
    1918             : 
    1919           0 : unsigned FastISel::fastEmit_rr(MVT, MVT, unsigned, unsigned /*Op0*/,
    1920             :                                bool /*Op0IsKill*/, unsigned /*Op1*/,
    1921             :                                bool /*Op1IsKill*/) {
    1922           0 :   return 0;
    1923             : }
    1924             : 
    1925           0 : unsigned FastISel::fastEmit_i(MVT, MVT, unsigned, uint64_t /*Imm*/) {
    1926           0 :   return 0;
    1927             : }
    1928             : 
    1929           0 : unsigned FastISel::fastEmit_f(MVT, MVT, unsigned,
    1930             :                               const ConstantFP * /*FPImm*/) {
    1931           0 :   return 0;
    1932             : }
    1933             : 
    1934           9 : unsigned FastISel::fastEmit_ri(MVT, MVT, unsigned, unsigned /*Op0*/,
    1935             :                                bool /*Op0IsKill*/, uint64_t /*Imm*/) {
    1936           9 :   return 0;
    1937             : }
    1938             : 
    1939             : /// This method is a wrapper of fastEmit_ri. It first tries to emit an
    1940             : /// instruction with an immediate operand using fastEmit_ri.
    1941             : /// If that fails, it materializes the immediate into a register and try
    1942             : /// fastEmit_rr instead.
    1943        8803 : unsigned FastISel::fastEmit_ri_(MVT VT, unsigned Opcode, unsigned Op0,
    1944             :                                 bool Op0IsKill, uint64_t Imm, MVT ImmType) {
    1945             :   // If this is a multiply by a power of two, emit this as a shift left.
    1946        8803 :   if (Opcode == ISD::MUL && isPowerOf2_64(Imm)) {
    1947             :     Opcode = ISD::SHL;
    1948         954 :     Imm = Log2_64(Imm);
    1949        7849 :   } else if (Opcode == ISD::UDIV && isPowerOf2_64(Imm)) {
    1950             :     // div x, 8 -> srl x, 3
    1951             :     Opcode = ISD::SRL;
    1952          18 :     Imm = Log2_64(Imm);
    1953             :   }
    1954             : 
    1955             :   // Horrible hack (to be removed), check to make sure shift amounts are
    1956             :   // in-range.
    1957       10567 :   if ((Opcode == ISD::SHL || Opcode == ISD::SRA || Opcode == ISD::SRL) &&
    1958        1764 :       Imm >= VT.getSizeInBits())
    1959             :     return 0;
    1960             : 
    1961             :   // First check if immediate type is legal. If not, we can't use the ri form.
    1962        8799 :   unsigned ResultReg = fastEmit_ri(VT, VT, Opcode, Op0, Op0IsKill, Imm);
    1963        8799 :   if (ResultReg)
    1964             :     return ResultReg;
    1965         283 :   unsigned MaterialReg = fastEmit_i(ImmType, ImmType, ISD::Constant, Imm);
    1966             :   bool IsImmKill = true;
    1967         283 :   if (!MaterialReg) {
    1968             :     // This is a bit ugly/slow, but failing here means falling out of
    1969             :     // fast-isel, which would be very slow.
    1970             :     IntegerType *ITy =
    1971         257 :         IntegerType::get(FuncInfo.Fn->getContext(), VT.getSizeInBits());
    1972         257 :     MaterialReg = getRegForValue(ConstantInt::get(ITy, Imm));
    1973         257 :     if (!MaterialReg)
    1974             :       return 0;
    1975             :     // FIXME: If the materialized register here has no uses yet then this
    1976             :     // will be the first use and we should be able to mark it as killed.
    1977             :     // However, the local value area for materialising constant expressions
    1978             :     // grows down, not up, which means that any constant expressions we generate
    1979             :     // later which also use 'Imm' could be after this instruction and therefore
    1980             :     // after this kill.
    1981             :     IsImmKill = false;
    1982             :   }
    1983         283 :   return fastEmit_rr(VT, VT, Opcode, Op0, Op0IsKill, MaterialReg, IsImmKill);
    1984             : }
    1985             : 
    1986      207907 : unsigned FastISel::createResultReg(const TargetRegisterClass *RC) {
    1987      415814 :   return MRI.createVirtualRegister(RC);
    1988             : }
    1989             : 
    1990      291744 : unsigned FastISel::constrainOperandRegClass(const MCInstrDesc &II, unsigned Op,
    1991             :                                             unsigned OpNum) {
    1992      291744 :   if (TargetRegisterInfo::isVirtualRegister(Op)) {
    1993             :     const TargetRegisterClass *RegClass =
    1994      104323 :         TII.getRegClass(II, OpNum, &TRI, *FuncInfo.MF);
    1995      104323 :     if (!MRI.constrainRegClass(Op, RegClass)) {
    1996             :       // If it's not legal to COPY between the register classes, something
    1997             :       // has gone very wrong before we got here.
    1998           0 :       unsigned NewOp = createResultReg(RegClass);
    1999           0 :       BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc,
    2000           0 :               TII.get(TargetOpcode::COPY), NewOp).addReg(Op);
    2001           0 :       return NewOp;
    2002             :     }
    2003             :   }
    2004             :   return Op;
    2005             : }
    2006             : 
    2007        3284 : unsigned FastISel::fastEmitInst_(unsigned MachineInstOpcode,
    2008             :                                  const TargetRegisterClass *RC) {
    2009        3284 :   unsigned ResultReg = createResultReg(RC);
    2010        3284 :   const MCInstrDesc &II = TII.get(MachineInstOpcode);
    2011             : 
    2012        3284 :   BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc, II, ResultReg);
    2013        3284 :   return ResultReg;
    2014             : }
    2015             : 
    2016        3141 : unsigned FastISel::fastEmitInst_r(unsigned MachineInstOpcode,
    2017             :                                   const TargetRegisterClass *RC, unsigned Op0,
    2018             :                                   bool Op0IsKill) {
    2019        3141 :   const MCInstrDesc &II = TII.get(MachineInstOpcode);
    2020             : 
    2021        3141 :   unsigned ResultReg = createResultReg(RC);
    2022        6282 :   Op0 = constrainOperandRegClass(II, Op0, II.getNumDefs());
    2023             : 
    2024        3141 :   if (II.getNumDefs() >= 1)
    2025        6248 :     BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc, II, ResultReg)
    2026        3124 :         .addReg(Op0, getKillRegState(Op0IsKill));
    2027             :   else {
    2028          34 :     BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc, II)
    2029          17 :         .addReg(Op0, getKillRegState(Op0IsKill));
    2030          34 :     BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc,
    2031          34 :             TII.get(TargetOpcode::COPY), ResultReg).addReg(II.ImplicitDefs[0]);
    2032             :   }
    2033             : 
    2034        3141 :   return ResultReg;
    2035             : }
    2036             : 
    2037        5121 : unsigned FastISel::fastEmitInst_rr(unsigned MachineInstOpcode,
    2038             :                                    const TargetRegisterClass *RC, unsigned Op0,
    2039             :                                    bool Op0IsKill, unsigned Op1,
    2040             :                                    bool Op1IsKill) {
    2041        5121 :   const MCInstrDesc &II = TII.get(MachineInstOpcode);
    2042             : 
    2043        5121 :   unsigned ResultReg = createResultReg(RC);
    2044       10242 :   Op0 = constrainOperandRegClass(II, Op0, II.getNumDefs());
    2045       10242 :   Op1 = constrainOperandRegClass(II, Op1, II.getNumDefs() + 1);
    2046             : 
    2047        5121 :   if (II.getNumDefs() >= 1)
    2048       10242 :     BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc, II, ResultReg)
    2049        5121 :         .addReg(Op0, getKillRegState(Op0IsKill))
    2050        5121 :         .addReg(Op1, getKillRegState(Op1IsKill));
    2051             :   else {
    2052           0 :     BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc, II)
    2053           0 :         .addReg(Op0, getKillRegState(Op0IsKill))
    2054           0 :         .addReg(Op1, getKillRegState(Op1IsKill));
    2055           0 :     BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc,
    2056           0 :             TII.get(TargetOpcode::COPY), ResultReg).addReg(II.ImplicitDefs[0]);
    2057             :   }
    2058        5121 :   return ResultReg;
    2059             : }
    2060             : 
    2061          58 : unsigned FastISel::fastEmitInst_rrr(unsigned MachineInstOpcode,
    2062             :                                     const TargetRegisterClass *RC, unsigned Op0,
    2063             :                                     bool Op0IsKill, unsigned Op1,
    2064             :                                     bool Op1IsKill, unsigned Op2,
    2065             :                                     bool Op2IsKill) {
    2066          58 :   const MCInstrDesc &II = TII.get(MachineInstOpcode);
    2067             : 
    2068          58 :   unsigned ResultReg = createResultReg(RC);
    2069         116 :   Op0 = constrainOperandRegClass(II, Op0, II.getNumDefs());
    2070         116 :   Op1 = constrainOperandRegClass(II, Op1, II.getNumDefs() + 1);
    2071         116 :   Op2 = constrainOperandRegClass(II, Op2, II.getNumDefs() + 2);
    2072             : 
    2073          58 :   if (II.getNumDefs() >= 1)
    2074         116 :     BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc, II, ResultReg)
    2075          58 :         .addReg(Op0, getKillRegState(Op0IsKill))
    2076          58 :         .addReg(Op1, getKillRegState(Op1IsKill))
    2077          58 :         .addReg(Op2, getKillRegState(Op2IsKill));
    2078             :   else {
    2079           0 :     BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc, II)
    2080           0 :         .addReg(Op0, getKillRegState(Op0IsKill))
    2081           0 :         .addReg(Op1, getKillRegState(Op1IsKill))
    2082           0 :         .addReg(Op2, getKillRegState(Op2IsKill));
    2083           0 :     BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc,
    2084           0 :             TII.get(TargetOpcode::COPY), ResultReg).addReg(II.ImplicitDefs[0]);
    2085             :   }
    2086          58 :   return ResultReg;
    2087             : }
    2088             : 
    2089        9807 : unsigned FastISel::fastEmitInst_ri(unsigned MachineInstOpcode,
    2090             :                                    const TargetRegisterClass *RC, unsigned Op0,
    2091             :                                    bool Op0IsKill, uint64_t Imm) {
    2092        9807 :   const MCInstrDesc &II = TII.get(MachineInstOpcode);
    2093             : 
    2094        9807 :   unsigned ResultReg = createResultReg(RC);
    2095       19614 :   Op0 = constrainOperandRegClass(II, Op0, II.getNumDefs());
    2096             : 
    2097        9807 :   if (II.getNumDefs() >= 1)
    2098       19614 :     BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc, II, ResultReg)
    2099        9807 :         .addReg(Op0, getKillRegState(Op0IsKill))
    2100        9807 :         .addImm(Imm);
    2101             :   else {
    2102           0 :     BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc, II)
    2103           0 :         .addReg(Op0, getKillRegState(Op0IsKill))
    2104           0 :         .addImm(Imm);
    2105           0 :     BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc,
    2106           0 :             TII.get(TargetOpcode::COPY), ResultReg).addReg(II.ImplicitDefs[0]);
    2107             :   }
    2108        9807 :   return ResultReg;
    2109             : }
    2110             : 
    2111         361 : unsigned FastISel::fastEmitInst_rii(unsigned MachineInstOpcode,
    2112             :                                     const TargetRegisterClass *RC, unsigned Op0,
    2113             :                                     bool Op0IsKill, uint64_t Imm1,
    2114             :                                     uint64_t Imm2) {
    2115         361 :   const MCInstrDesc &II = TII.get(MachineInstOpcode);
    2116             : 
    2117         361 :   unsigned ResultReg = createResultReg(RC);
    2118         722 :   Op0 = constrainOperandRegClass(II, Op0, II.getNumDefs());
    2119             : 
    2120         361 :   if (II.getNumDefs() >= 1)
    2121         722 :     BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc, II, ResultReg)
    2122         361 :         .addReg(Op0, getKillRegState(Op0IsKill))
    2123         361 :         .addImm(Imm1)
    2124         361 :         .addImm(Imm2);
    2125             :   else {
    2126           0 :     BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc, II)
    2127           0 :         .addReg(Op0, getKillRegState(Op0IsKill))
    2128           0 :         .addImm(Imm1)
    2129           0 :         .addImm(Imm2);
    2130           0 :     BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc,
    2131           0 :             TII.get(TargetOpcode::COPY), ResultReg).addReg(II.ImplicitDefs[0]);
    2132             :   }
    2133         361 :   return ResultReg;
    2134             : }
    2135             : 
    2136           4 : unsigned FastISel::fastEmitInst_f(unsigned MachineInstOpcode,
    2137             :                                   const TargetRegisterClass *RC,
    2138             :                                   const ConstantFP *FPImm) {
    2139           4 :   const MCInstrDesc &II = TII.get(MachineInstOpcode);
    2140             : 
    2141           4 :   unsigned ResultReg = createResultReg(RC);
    2142             : 
    2143           4 :   if (II.getNumDefs() >= 1)
    2144           8 :     BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc, II, ResultReg)
    2145             :         .addFPImm(FPImm);
    2146             :   else {
    2147           0 :     BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc, II)
    2148             :         .addFPImm(FPImm);
    2149           0 :     BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc,
    2150           0 :             TII.get(TargetOpcode::COPY), ResultReg).addReg(II.ImplicitDefs[0]);
    2151             :   }
    2152           4 :   return ResultReg;
    2153             : }
    2154             : 
    2155         229 : unsigned FastISel::fastEmitInst_rri(unsigned MachineInstOpcode,
    2156             :                                     const TargetRegisterClass *RC, unsigned Op0,
    2157             :                                     bool Op0IsKill, unsigned Op1,
    2158             :                                     bool Op1IsKill, uint64_t Imm) {
    2159         229 :   const MCInstrDesc &II = TII.get(MachineInstOpcode);
    2160             : 
    2161         229 :   unsigned ResultReg = createResultReg(RC);
    2162         458 :   Op0 = constrainOperandRegClass(II, Op0, II.getNumDefs());
    2163         458 :   Op1 = constrainOperandRegClass(II, Op1, II.getNumDefs() + 1);
    2164             : 
    2165         229 :   if (II.getNumDefs() >= 1)
    2166         458 :     BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc, II, ResultReg)
    2167         229 :         .addReg(Op0, getKillRegState(Op0IsKill))
    2168         229 :         .addReg(Op1, getKillRegState(Op1IsKill))
    2169         229 :         .addImm(Imm);
    2170             :   else {
    2171           0 :     BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc, II)
    2172           0 :         .addReg(Op0, getKillRegState(Op0IsKill))
    2173           0 :         .addReg(Op1, getKillRegState(Op1IsKill))
    2174           0 :         .addImm(Imm);
    2175           0 :     BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc,
    2176           0 :             TII.get(TargetOpcode::COPY), ResultReg).addReg(II.ImplicitDefs[0]);
    2177             :   }
    2178         229 :   return ResultReg;
    2179             : }
    2180             : 
    2181        5728 : unsigned FastISel::fastEmitInst_i(unsigned MachineInstOpcode,
    2182             :                                   const TargetRegisterClass *RC, uint64_t Imm) {
    2183        5728 :   unsigned ResultReg = createResultReg(RC);
    2184        5728 :   const MCInstrDesc &II = TII.get(MachineInstOpcode);
    2185             : 
    2186        5728 :   if (II.getNumDefs() >= 1)
    2187       11456 :     BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc, II, ResultReg)
    2188        5728 :         .addImm(Imm);
    2189             :   else {
    2190           0 :     BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc, II).addImm(Imm);
    2191           0 :     BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc,
    2192           0 :             TII.get(TargetOpcode::COPY), ResultReg).addReg(II.ImplicitDefs[0]);
    2193             :   }
    2194        5728 :   return ResultReg;
    2195             : }
    2196             : 
    2197         541 : unsigned FastISel::fastEmitInst_extractsubreg(MVT RetVT, unsigned Op0,
    2198             :                                               bool Op0IsKill, uint32_t Idx) {
    2199         541 :   unsigned ResultReg = createResultReg(TLI.getRegClassFor(RetVT));
    2200             :   assert(TargetRegisterInfo::isVirtualRegister(Op0) &&
    2201             :          "Cannot yet extract from physregs");
    2202         541 :   const TargetRegisterClass *RC = MRI.getRegClass(Op0);
    2203         541 :   MRI.constrainRegClass(Op0, TRI.getSubClassWithSubReg(RC, Idx));
    2204        1623 :   BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc, TII.get(TargetOpcode::COPY),
    2205        1623 :           ResultReg).addReg(Op0, getKillRegState(Op0IsKill), Idx);
    2206         541 :   return ResultReg;
    2207             : }
    2208             : 
    2209             : /// Emit MachineInstrs to compute the value of Op with all but the least
    2210             : /// significant bit set to zero.
    2211        1496 : unsigned FastISel::fastEmitZExtFromI1(MVT VT, unsigned Op0, bool Op0IsKill) {
    2212        1496 :   return fastEmit_ri(VT, VT, ISD::AND, Op0, Op0IsKill, 1);
    2213             : }
    2214             : 
    2215             : /// HandlePHINodesInSuccessorBlocks - Handle PHI nodes in successor blocks.
    2216             : /// Emit code to ensure constants are copied into registers when needed.
    2217             : /// Remember the virtual registers that need to be added to the Machine PHI
    2218             : /// nodes as input.  We cannot just directly add them, because expansion
    2219             : /// might result in multiple MBB's for one BB.  As such, the start of the
    2220             : /// BB might correspond to a different MBB than the end.
    2221      102927 : bool FastISel::handlePHINodesInSuccessorBlocks(const BasicBlock *LLVMBB) {
    2222      102927 :   const TerminatorInst *TI = LLVMBB->getTerminator();
    2223             : 
    2224             :   SmallPtrSet<MachineBasicBlock *, 4> SuccsHandled;
    2225      205854 :   FuncInfo.OrigNumPHINodesToUpdate = FuncInfo.PHINodesToUpdate.size();
    2226             : 
    2227             :   // Check successor nodes' PHI nodes that expect a constant to be available
    2228             :   // from this block.
    2229      176684 :   for (unsigned succ = 0, e = TI->getNumSuccessors(); succ != e; ++succ) {
    2230       73815 :     const BasicBlock *SuccBB = TI->getSuccessor(succ);
    2231       73815 :     if (!isa<PHINode>(SuccBB->begin()))
    2232      143621 :       continue;
    2233        4010 :     MachineBasicBlock *SuccMBB = FuncInfo.MBBMap[SuccBB];
    2234             : 
    2235             :     // If this terminator has multiple identical successors (common for
    2236             :     // switches), only handle each succ once.
    2237        2005 :     if (!SuccsHandled.insert(SuccMBB).second)
    2238           1 :       continue;
    2239             : 
    2240        2004 :     MachineBasicBlock::iterator MBBI = SuccMBB->begin();
    2241             : 
    2242             :     // At this point we know that there is a 1-1 correspondence between LLVM PHI
    2243             :     // nodes and Machine PHI nodes, but the incoming operands have not been
    2244             :     // emitted yet.
    2245        5969 :     for (const PHINode &PN : SuccBB->phis()) {
    2246             :       // Ignore dead phi's.
    2247        2019 :       if (PN.use_empty())
    2248           8 :         continue;
    2249             : 
    2250             :       // Only handle legal types. Two interesting things to note here. First,
    2251             :       // by bailing out early, we may leave behind some dead instructions,
    2252             :       // since SelectionDAG's HandlePHINodesInSuccessorBlocks will insert its
    2253             :       // own moves. Second, this check is necessary because FastISel doesn't
    2254             :       // use CreateRegs to create registers, so it always creates
    2255             :       // exactly one register for each non-void instruction.
    2256        2011 :       EVT VT = TLI.getValueType(DL, PN.getType(), /*AllowUnknown=*/true);
    2257        2011 :       if (VT == MVT::Other || !TLI.isTypeLegal(VT)) {
    2258             :         // Handle integer promotions, though, because they're common and easy.
    2259             :         if (!(VT == MVT::i1 || VT == MVT::i8 || VT == MVT::i16)) {
    2260          57 :           FuncInfo.PHINodesToUpdate.resize(FuncInfo.OrigNumPHINodesToUpdate);
    2261          57 :           return false;
    2262             :         }
    2263             :       }
    2264             : 
    2265        1954 :       const Value *PHIOp = PN.getIncomingValueForBlock(LLVMBB);
    2266             : 
    2267             :       // Set the DebugLoc for the copy. Prefer the location of the operand
    2268             :       // if there is one; use the location of the PHI otherwise.
    2269             :       DbgLoc = PN.getDebugLoc();
    2270             :       if (const auto *Inst = dyn_cast<Instruction>(PHIOp))
    2271             :         DbgLoc = Inst->getDebugLoc();
    2272             : 
    2273        1954 :       unsigned Reg = getRegForValue(PHIOp);
    2274        1954 :       if (!Reg) {
    2275           1 :         FuncInfo.PHINodesToUpdate.resize(FuncInfo.OrigNumPHINodesToUpdate);
    2276           1 :         return false;
    2277             :       }
    2278        5859 :       FuncInfo.PHINodesToUpdate.push_back(std::make_pair(&*MBBI++, Reg));
    2279        3906 :       DbgLoc = DebugLoc();
    2280             :     }
    2281             :   }
    2282             : 
    2283             :   return true;
    2284             : }
    2285             : 
    2286       72204 : bool FastISel::tryToFoldLoad(const LoadInst *LI, const Instruction *FoldInst) {
    2287             :   assert(LI->hasOneUse() &&
    2288             :          "tryToFoldLoad expected a LoadInst with a single use");
    2289             :   // We know that the load has a single use, but don't know what it is.  If it
    2290             :   // isn't one of the folded instructions, then we can't succeed here.  Handle
    2291             :   // this by scanning the single-use users of the load until we get to FoldInst.
    2292             :   unsigned MaxUsers = 6; // Don't scan down huge single-use chains of instrs.
    2293             : 
    2294             :   const Instruction *TheUser = LI->user_back();
    2295       57418 :   while (TheUser != FoldInst && // Scan up until we find FoldInst.
    2296             :          // Stay in the right block.
    2297      129133 :          TheUser->getParent() == FoldInst->getParent() &&
    2298             :          --MaxUsers) { // Don't scan too far.
    2299             :     // If there are multiple or no uses of this instruction, then bail out.
    2300       22375 :     if (!TheUser->hasOneUse())
    2301             :       return false;
    2302             : 
    2303             :     TheUser = TheUser->user_back();
    2304             :   }
    2305             : 
    2306             :   // If we didn't find the fold instruction, then we failed to collapse the
    2307             :   // sequence.
    2308       60034 :   if (TheUser != FoldInst)
    2309             :     return false;
    2310             : 
    2311             :   // Don't try to fold volatile loads.  Target has to deal with alignment
    2312             :   // constraints.
    2313       59408 :   if (LI->isVolatile())
    2314             :     return false;
    2315             : 
    2316             :   // Figure out which vreg this is going into.  If there is no assigned vreg yet
    2317             :   // then there actually was no reference to it.  Perhaps the load is referenced
    2318             :   // by a dead instruction.
    2319       59211 :   unsigned LoadReg = getRegForValue(LI);
    2320       59211 :   if (!LoadReg)
    2321             :     return false;
    2322             : 
    2323             :   // We can't fold if this vreg has no uses or more than one use.  Multiple uses
    2324             :   // may mean that the instruction got lowered to multiple MIs, or the use of
    2325             :   // the loaded value ended up being multiple operands of the result.
    2326       59211 :   if (!MRI.hasOneUse(LoadReg))
    2327             :     return false;
    2328             : 
    2329       55843 :   MachineRegisterInfo::reg_iterator RI = MRI.reg_begin(LoadReg);
    2330       55843 :   MachineInstr *User = RI->getParent();
    2331             : 
    2332             :   // Set the insertion point properly.  Folding the load can cause generation of
    2333             :   // other random instructions (like sign extends) for addressing modes; make
    2334             :   // sure they get inserted in a logical place before the new instruction.
    2335       55843 :   FuncInfo.InsertPt = User;
    2336       55843 :   FuncInfo.MBB = User->getParent();
    2337             : 
    2338             :   // Ask the target to try folding the load.
    2339      111686 :   return tryToFoldLoadIntoMI(User, RI.getOperandNo(), LI);
    2340             : }
    2341             : 
    2342         643 : bool FastISel::canFoldAddIntoGEP(const User *GEP, const Value *Add) {
    2343             :   // Must be an add.
    2344             :   if (!isa<AddOperator>(Add))
    2345             :     return false;
    2346             :   // Type size needs to match.
    2347          32 :   if (DL.getTypeSizeInBits(GEP->getType()) !=
    2348          16 :       DL.getTypeSizeInBits(Add->getType()))
    2349             :     return false;
    2350             :   // Must be in the same basic block.
    2351          22 :   if (isa<Instruction>(Add) &&
    2352          33 :       FuncInfo.MBBMap[cast<Instruction>(Add)->getParent()] != FuncInfo.MBB)
    2353             :     return false;
    2354             :   // Must have a constant operand.
    2355          10 :   return isa<ConstantInt>(cast<AddOperator>(Add)->getOperand(1));
    2356             : }
    2357             : 
    2358             : MachineMemOperand *
    2359      159630 : FastISel::createMachineMemOperandFor(const Instruction *I) const {
    2360             :   const Value *Ptr;
    2361             :   Type *ValTy;
    2362             :   unsigned Alignment;
    2363             :   MachineMemOperand::Flags Flags;
    2364             :   bool IsVolatile;
    2365             : 
    2366             :   if (const auto *LI = dyn_cast<LoadInst>(I)) {
    2367             :     Alignment = LI->getAlignment();
    2368             :     IsVolatile = LI->isVolatile();
    2369             :     Flags = MachineMemOperand::MOLoad;
    2370             :     Ptr = LI->getPointerOperand();
    2371       78613 :     ValTy = LI->getType();
    2372             :   } else if (const auto *SI = dyn_cast<StoreInst>(I)) {
    2373             :     Alignment = SI->getAlignment();
    2374             :     IsVolatile = SI->isVolatile();
    2375             :     Flags = MachineMemOperand::MOStore;
    2376             :     Ptr = SI->getPointerOperand();
    2377       81017 :     ValTy = SI->getValueOperand()->getType();
    2378             :   } else
    2379             :     return nullptr;
    2380             : 
    2381             :   bool IsNonTemporal = I->getMetadata(LLVMContext::MD_nontemporal) != nullptr;
    2382             :   bool IsInvariant = I->getMetadata(LLVMContext::MD_invariant_load) != nullptr;
    2383             :   bool IsDereferenceable =
    2384             :       I->getMetadata(LLVMContext::MD_dereferenceable) != nullptr;
    2385             :   const MDNode *Ranges = I->getMetadata(LLVMContext::MD_range);
    2386             : 
    2387             :   AAMDNodes AAInfo;
    2388      159630 :   I->getAAMetadata(AAInfo);
    2389             : 
    2390      159630 :   if (Alignment == 0) // Ensure that codegen never sees alignment 0.
    2391        1152 :     Alignment = DL.getABITypeAlignment(ValTy);
    2392             : 
    2393      159630 :   unsigned Size = DL.getTypeStoreSize(ValTy);
    2394             : 
    2395      159630 :   if (IsVolatile)
    2396             :     Flags |= MachineMemOperand::MOVolatile;
    2397      159630 :   if (IsNonTemporal)
    2398             :     Flags |= MachineMemOperand::MONonTemporal;
    2399      159630 :   if (IsDereferenceable)
    2400             :     Flags |= MachineMemOperand::MODereferenceable;
    2401      159630 :   if (IsInvariant)
    2402             :     Flags |= MachineMemOperand::MOInvariant;
    2403             : 
    2404      319260 :   return FuncInfo.MF->getMachineMemOperand(MachinePointerInfo(Ptr), Flags, Size,
    2405      159630 :                                            Alignment, AAInfo, Ranges);
    2406             : }
    2407             : 
    2408       10054 : CmpInst::Predicate FastISel::optimizeCmpPredicate(const CmpInst *CI) const {
    2409             :   // If both operands are the same, then try to optimize or fold the cmp.
    2410             :   CmpInst::Predicate Predicate = CI->getPredicate();
    2411       10054 :   if (CI->getOperand(0) != CI->getOperand(1))
    2412             :     return Predicate;
    2413             : 
    2414         101 :   switch (Predicate) {
    2415           0 :   default: llvm_unreachable("Invalid predicate!");
    2416             :   case CmpInst::FCMP_FALSE: Predicate = CmpInst::FCMP_FALSE; break;
    2417           4 :   case CmpInst::FCMP_OEQ:   Predicate = CmpInst::FCMP_ORD;   break;
    2418             :   case CmpInst::FCMP_OGT:   Predicate = CmpInst::FCMP_FALSE; break;
    2419           4 :   case CmpInst::FCMP_OGE:   Predicate = CmpInst::FCMP_ORD;   break;
    2420             :   case CmpInst::FCMP_OLT:   Predicate = CmpInst::FCMP_FALSE; break;
    2421           4 :   case CmpInst::FCMP_OLE:   Predicate = CmpInst::FCMP_ORD;   break;
    2422             :   case CmpInst::FCMP_ONE:   Predicate = CmpInst::FCMP_FALSE; break;
    2423           4 :   case CmpInst::FCMP_ORD:   Predicate = CmpInst::FCMP_ORD;   break;
    2424           4 :   case CmpInst::FCMP_UNO:   Predicate = CmpInst::FCMP_UNO;   break;
    2425           6 :   case CmpInst::FCMP_UEQ:   Predicate = CmpInst::FCMP_TRUE;  break;
    2426           4 :   case CmpInst::FCMP_UGT:   Predicate = CmpInst::FCMP_UNO;   break;
    2427           4 :   case CmpInst::FCMP_UGE:   Predicate = CmpInst::FCMP_TRUE;  break;
    2428           4 :   case CmpInst::FCMP_ULT:   Predicate = CmpInst::FCMP_UNO;   break;
    2429           4 :   case CmpInst::FCMP_ULE:   Predicate = CmpInst::FCMP_TRUE;  break;
    2430           4 :   case CmpInst::FCMP_UNE:   Predicate = CmpInst::FCMP_UNO;   break;
    2431           0 :   case CmpInst::FCMP_TRUE:  Predicate = CmpInst::FCMP_TRUE;  break;
    2432             : 
    2433           4 :   case CmpInst::ICMP_EQ:    Predicate = CmpInst::FCMP_TRUE;  break;
    2434             :   case CmpInst::ICMP_NE:    Predicate = CmpInst::FCMP_FALSE; break;
    2435             :   case CmpInst::ICMP_UGT:   Predicate = CmpInst::FCMP_FALSE; break;
    2436           4 :   case CmpInst::ICMP_UGE:   Predicate = CmpInst::FCMP_TRUE;  break;
    2437             :   case CmpInst::ICMP_ULT:   Predicate = CmpInst::FCMP_FALSE; break;
    2438           4 :   case CmpInst::ICMP_ULE:   Predicate = CmpInst::FCMP_TRUE;  break;
    2439             :   case CmpInst::ICMP_SGT:   Predicate = CmpInst::FCMP_FALSE; break;
    2440           4 :   case CmpInst::ICMP_SGE:   Predicate = CmpInst::FCMP_TRUE;  break;
    2441             :   case CmpInst::ICMP_SLT:   Predicate = CmpInst::FCMP_FALSE; break;
    2442           4 :   case CmpInst::ICMP_SLE:   Predicate = CmpInst::FCMP_TRUE;  break;
    2443             :   }
    2444             : 
    2445             :   return Predicate;
    2446      303507 : }

Generated by: LCOV version 1.13