LCOV - code coverage report
Current view: top level - lib/CodeGen/SelectionDAG - FastISel.cpp (source / functions) Hit Total Coverage
Test: llvm-toolchain.info Lines: 1007 1116 90.2 %
Date: 2017-09-14 15:23:50 Functions: 61 67 91.0 %
Legend: Lines: hit not hit

          Line data    Source code
       1             : //===- FastISel.cpp - Implementation of the FastISel class ----------------===//
       2             : //
       3             : //                     The LLVM Compiler Infrastructure
       4             : //
       5             : // This file is distributed under the University of Illinois Open Source
       6             : // License. See LICENSE.TXT for details.
       7             : //
       8             : //===----------------------------------------------------------------------===//
       9             : //
      10             : // This file contains the implementation of the FastISel class.
      11             : //
      12             : // "Fast" instruction selection is designed to emit very poor code quickly.
      13             : // Also, it is not designed to be able to do much lowering, so most illegal
      14             : // types (e.g. i64 on 32-bit targets) and operations are not supported.  It is
      15             : // also not intended to be able to do much optimization, except in a few cases
      16             : // where doing optimizations reduces overall compile time.  For example, folding
      17             : // constants into immediate fields is often done, because it's cheap and it
      18             : // reduces the number of instructions later phases have to examine.
      19             : //
      20             : // "Fast" instruction selection is able to fail gracefully and transfer
      21             : // control to the SelectionDAG selector for operations that it doesn't
      22             : // support.  In many cases, this allows us to avoid duplicating a lot of
      23             : // the complicated lowering logic that SelectionDAG currently has.
      24             : //
      25             : // The intended use for "fast" instruction selection is "-O0" mode
      26             : // compilation, where the quality of the generated code is irrelevant when
      27             : // weighed against the speed at which the code can be generated.  Also,
      28             : // at -O0, the LLVM optimizers are not running, and this makes the
      29             : // compile time of codegen a much higher portion of the overall compile
      30             : // time.  Despite its limitations, "fast" instruction selection is able to
      31             : // handle enough code on its own to provide noticeable overall speedups
      32             : // in -O0 compiles.
      33             : //
      34             : // Basic operations are supported in a target-independent way, by reading
      35             : // the same instruction descriptions that the SelectionDAG selector reads,
      36             : // and identifying simple arithmetic operations that can be directly selected
      37             : // from simple operators.  More complicated operations currently require
      38             : // target-specific code.
      39             : //
      40             : //===----------------------------------------------------------------------===//
      41             : 
      42             : #include "llvm/CodeGen/FastISel.h"
      43             : #include "llvm/ADT/APFloat.h"
      44             : #include "llvm/ADT/APSInt.h"
      45             : #include "llvm/ADT/DenseMap.h"
      46             : #include "llvm/ADT/Optional.h"
      47             : #include "llvm/ADT/SmallPtrSet.h"
      48             : #include "llvm/ADT/SmallString.h"
      49             : #include "llvm/ADT/SmallVector.h"
      50             : #include "llvm/ADT/Statistic.h"
      51             : #include "llvm/Analysis/BranchProbabilityInfo.h"
      52             : #include "llvm/Analysis/TargetLibraryInfo.h"
      53             : #include "llvm/CodeGen/Analysis.h"
      54             : #include "llvm/CodeGen/FunctionLoweringInfo.h"
      55             : #include "llvm/CodeGen/ISDOpcodes.h"
      56             : #include "llvm/CodeGen/MachineBasicBlock.h"
      57             : #include "llvm/CodeGen/MachineFrameInfo.h"
      58             : #include "llvm/CodeGen/MachineInstr.h"
      59             : #include "llvm/CodeGen/MachineInstrBuilder.h"
      60             : #include "llvm/CodeGen/MachineMemOperand.h"
      61             : #include "llvm/CodeGen/MachineModuleInfo.h"
      62             : #include "llvm/CodeGen/MachineOperand.h"
      63             : #include "llvm/CodeGen/MachineRegisterInfo.h"
      64             : #include "llvm/CodeGen/MachineValueType.h"
      65             : #include "llvm/CodeGen/StackMaps.h"
      66             : #include "llvm/CodeGen/ValueTypes.h"
      67             : #include "llvm/IR/Argument.h"
      68             : #include "llvm/IR/Attributes.h"
      69             : #include "llvm/IR/BasicBlock.h"
      70             : #include "llvm/IR/CallSite.h"
      71             : #include "llvm/IR/CallingConv.h"
      72             : #include "llvm/IR/Constant.h"
      73             : #include "llvm/IR/Constants.h"
      74             : #include "llvm/IR/DataLayout.h"
      75             : #include "llvm/IR/DebugInfo.h"
      76             : #include "llvm/IR/DebugLoc.h"
      77             : #include "llvm/IR/DerivedTypes.h"
      78             : #include "llvm/IR/Function.h"
      79             : #include "llvm/IR/GetElementPtrTypeIterator.h"
      80             : #include "llvm/IR/GlobalValue.h"
      81             : #include "llvm/IR/InlineAsm.h"
      82             : #include "llvm/IR/InstrTypes.h"
      83             : #include "llvm/IR/Instruction.h"
      84             : #include "llvm/IR/Instructions.h"
      85             : #include "llvm/IR/IntrinsicInst.h"
      86             : #include "llvm/IR/LLVMContext.h"
      87             : #include "llvm/IR/Mangler.h"
      88             : #include "llvm/IR/Metadata.h"
      89             : #include "llvm/IR/Operator.h"
      90             : #include "llvm/IR/Type.h"
      91             : #include "llvm/IR/User.h"
      92             : #include "llvm/IR/Value.h"
      93             : #include "llvm/MC/MCContext.h"
      94             : #include "llvm/MC/MCInstrDesc.h"
      95             : #include "llvm/MC/MCRegisterInfo.h"
      96             : #include "llvm/Support/Casting.h"
      97             : #include "llvm/Support/Debug.h"
      98             : #include "llvm/Support/ErrorHandling.h"
      99             : #include "llvm/Support/MathExtras.h"
     100             : #include "llvm/Support/raw_ostream.h"
     101             : #include "llvm/Target/TargetInstrInfo.h"
     102             : #include "llvm/Target/TargetLowering.h"
     103             : #include "llvm/Target/TargetMachine.h"
     104             : #include "llvm/Target/TargetOptions.h"
     105             : #include "llvm/Target/TargetSubtargetInfo.h"
     106             : #include <algorithm>
     107             : #include <cassert>
     108             : #include <cstdint>
     109             : #include <iterator>
     110             : #include <utility>
     111             : 
     112             : using namespace llvm;
     113             : 
     114             : #define DEBUG_TYPE "isel"
     115             : 
     116             : STATISTIC(NumFastIselSuccessIndependent, "Number of insts selected by "
     117             :                                          "target-independent selector");
     118             : STATISTIC(NumFastIselSuccessTarget, "Number of insts selected by "
     119             :                                     "target-specific selector");
     120             : STATISTIC(NumFastIselDead, "Number of dead insts removed on failure");
     121             : 
     122             : /// Set the current block to which generated machine instructions will be
     123             : /// appended, and clear the local CSE map.
     124        9708 : void FastISel::startNewBlock() {
     125        9708 :   LocalValueMap.clear();
     126             : 
     127             :   // Instructions are appended to FuncInfo.MBB. If the basic block already
     128             :   // contains labels or copies, use the last instruction as the last local
     129             :   // value.
     130        9708 :   EmitStartPt = nullptr;
     131       19416 :   if (!FuncInfo.MBB->empty())
     132         458 :     EmitStartPt = &FuncInfo.MBB->back();
     133        9708 :   LastLocalValue = EmitStartPt;
     134        9708 : }
     135             : 
     136        7750 : bool FastISel::lowerArguments() {
     137        7750 :   if (!FuncInfo.CanLowerReturn)
     138             :     // Fallback to SDISel argument lowering code to deal with sret pointer
     139             :     // parameter.
     140             :     return false;
     141             : 
     142        7746 :   if (!fastLowerArguments())
     143             :     return false;
     144             : 
     145             :   // Enter arguments into ValueMap for uses in non-entry BBs.
     146       11801 :   for (Function::const_arg_iterator I = FuncInfo.Fn->arg_begin(),
     147        7390 :                                     E = FuncInfo.Fn->arg_end();
     148        8106 :        I != E; ++I) {
     149        4411 :     DenseMap<const Value *, unsigned>::iterator VI = LocalValueMap.find(&*I);
     150             :     assert(VI != LocalValueMap.end() && "Missed an argument?");
     151        8822 :     FuncInfo.ValueMap[&*I] = VI->second;
     152             :   }
     153             :   return true;
     154             : }
     155             : 
     156        1899 : void FastISel::flushLocalValueMap() {
     157        1899 :   LocalValueMap.clear();
     158        1899 :   LastLocalValue = EmitStartPt;
     159        1899 :   recomputeInsertPt();
     160        1899 :   SavedInsertPt = FuncInfo.InsertPt;
     161        1899 : }
     162             : 
     163        8218 : bool FastISel::hasTrivialKill(const Value *V) {
     164             :   // Don't consider constants or arguments to have trivial kills.
     165        3674 :   const Instruction *I = dyn_cast<Instruction>(V);
     166             :   if (!I)
     167             :     return false;
     168             : 
     169             :   // No-op casts are trivially coalesced by fast-isel.
     170         649 :   if (const auto *Cast = dyn_cast<CastInst>(I))
     171         934 :     if (Cast->isNoopCast(DL.getIntPtrType(Cast->getContext())) &&
     172         570 :         !hasTrivialKill(Cast->getOperand(0)))
     173             :       return false;
     174             : 
     175             :   // Even the value might have only one use in the LLVM IR, it is possible that
     176             :   // FastISel might fold the use into another instruction and now there is more
     177             :   // than one use at the Machine Instruction level.
     178        3514 :   unsigned Reg = lookUpRegForValue(V);
     179        6888 :   if (Reg && !MRI.use_empty(Reg))
     180             :     return false;
     181             : 
     182             :   // GEPs with all zero indices are trivially coalesced by fast-isel.
     183          46 :   if (const auto *GEP = dyn_cast<GetElementPtrInst>(I))
     184          68 :     if (GEP->hasAllZeroIndices() && !hasTrivialKill(GEP->getOperand(0)))
     185             :       return false;
     186             : 
     187             :   // Only instructions with a single use in the same basic block are considered
     188             :   // to have trivial kills.
     189        9527 :   return I->hasOneUse() &&
     190        8617 :          !(I->getOpcode() == Instruction::BitCast ||
     191        2866 :            I->getOpcode() == Instruction::PtrToInt ||
     192        9037 :            I->getOpcode() == Instruction::IntToPtr) &&
     193       11436 :          cast<Instruction>(*I->user_begin())->getParent() == I->getParent();
     194             : }
     195             : 
     196       27437 : unsigned FastISel::getRegForValue(const Value *V) {
     197       27437 :   EVT RealVT = TLI.getValueType(DL, V->getType(), /*AllowUnknown=*/true);
     198             :   // Don't handle non-simple values in FastISel.
     199       27437 :   if (!RealVT.isSimple())
     200             :     return 0;
     201             : 
     202             :   // Ignore illegal types. We must do this before looking up the value
     203             :   // in ValueMap because Arguments are given virtual registers regardless
     204             :   // of whether FastISel can handle them.
     205       27428 :   MVT VT = RealVT.getSimpleVT();
     206       82284 :   if (!TLI.isTypeLegal(VT)) {
     207             :     // Handle integer promotions, though, because they're common and easy.
     208        5002 :     if (VT == MVT::i1 || VT == MVT::i8 || VT == MVT::i16)
     209        5342 :       VT = TLI.getTypeToTransformTo(V->getContext(), VT).getSimpleVT();
     210             :     else
     211             :       return 0;
     212             :   }
     213             : 
     214             :   // Look up the value to see if we already have a register for it.
     215       27117 :   unsigned Reg = lookUpRegForValue(V);
     216       27117 :   if (Reg)
     217             :     return Reg;
     218             : 
     219             :   // In bottom-up mode, just create the virtual register which will be used
     220             :   // to hold the value. It will be materialized later.
     221       27614 :   if (isa<Instruction>(V) &&
     222       10116 :       (!isa<AllocaInst>(V) ||
     223         812 :        !FuncInfo.StaticAllocaMap.count(cast<AllocaInst>(V))))
     224        9725 :     return FuncInfo.InitializeRegForValue(V);
     225             : 
     226        4082 :   SavePoint SaveInsertPt = enterLocalValueArea();
     227             : 
     228             :   // Materialize the value in a register. Emit any instructions in the
     229             :   // local value area.
     230        4082 :   Reg = materializeRegForValue(V, VT);
     231             : 
     232        8164 :   leaveLocalValueArea(SaveInsertPt);
     233             : 
     234        4082 :   return Reg;
     235             : }
     236             : 
     237        1011 : unsigned FastISel::materializeConstant(const Value *V, MVT VT) {
     238        1011 :   unsigned Reg = 0;
     239        1101 :   if (const auto *CI = dyn_cast<ConstantInt>(V)) {
     240         180 :     if (CI->getValue().getActiveBits() <= 64)
     241         180 :       Reg = fastEmit_i(VT, VT, ISD::Constant, CI->getZExtValue());
     242         921 :   } else if (isa<AllocaInst>(V))
     243         782 :     Reg = fastMaterializeAlloca(cast<AllocaInst>(V));
     244        1060 :   else if (isa<ConstantPointerNull>(V))
     245             :     // Translate this as an integer zero so that it can be
     246             :     // local-CSE'd with actual integer zeros.
     247          30 :     Reg = getRegForValue(
     248          30 :         Constant::getNullValue(DL.getIntPtrType(V->getContext())));
     249         503 :   else if (const auto *CF = dyn_cast<ConstantFP>(V)) {
     250           3 :     if (CF->isNullValue())
     251           1 :       Reg = fastMaterializeFloatZero(CF);
     252             :     else
     253             :       // Try to emit the constant directly.
     254           2 :       Reg = fastEmit_f(VT, VT, ISD::ConstantFP, CF);
     255             : 
     256           3 :     if (!Reg) {
     257             :       // Try to emit the constant by using an integer constant with a cast.
     258           3 :       const APFloat &Flt = CF->getValueAPF();
     259           9 :       EVT IntVT = TLI.getPointerTy(DL);
     260           3 :       uint32_t IntBitWidth = IntVT.getSizeInBits();
     261           6 :       APSInt SIntVal(IntBitWidth, /*isUnsigned=*/false);
     262             :       bool isExact;
     263           3 :       (void)Flt.convertToInteger(SIntVal, APFloat::rmTowardZero, &isExact);
     264           3 :       if (isExact) {
     265             :         unsigned IntegerReg =
     266           3 :             getRegForValue(ConstantInt::get(V->getContext(), SIntVal));
     267           3 :         if (IntegerReg != 0)
     268           3 :           Reg = fastEmit_r(IntVT.getSimpleVT(), VT, ISD::SINT_TO_FP, IntegerReg,
     269           3 :                            /*Kill=*/false);
     270             :       }
     271             :     }
     272         683 :   } else if (const auto *Op = dyn_cast<Operator>(V)) {
     273         372 :     if (!selectOperator(Op, Op->getOpcode()))
     274          60 :       if (!isa<Instruction>(Op) ||
     275           0 :           !fastSelectInstruction(cast<Instruction>(Op)))
     276             :         return 0;
     277         156 :     Reg = lookUpRegForValue(Op);
     278         622 :   } else if (isa<UndefValue>(V)) {
     279         167 :     Reg = createResultReg(TLI.getRegClassFor(VT));
     280         167 :     BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc,
     281         334 :             TII.get(TargetOpcode::IMPLICIT_DEF), Reg);
     282             :   }
     283             :   return Reg;
     284             : }
     285             : 
     286             : /// Helper for getRegForValue. This function is called when the value isn't
     287             : /// already available in a register and must be materialized with new
     288             : /// instructions.
     289        4082 : unsigned FastISel::materializeRegForValue(const Value *V, MVT VT) {
     290        4082 :   unsigned Reg = 0;
     291             :   // Give the target-specific code a try first.
     292        4082 :   if (isa<Constant>(V))
     293        7382 :     Reg = fastMaterializeConstant(cast<Constant>(V));
     294             : 
     295             :   // If target-specific code couldn't or didn't want to handle the value, then
     296             :   // give target-independent code a try.
     297        3691 :   if (!Reg)
     298        1011 :     Reg = materializeConstant(V, VT);
     299             : 
     300             :   // Don't cache constant materializations in the general ValueMap.
     301             :   // To do so would require tracking what uses they dominate.
     302        4082 :   if (Reg) {
     303        7810 :     LocalValueMap[V] = Reg;
     304        3905 :     LastLocalValue = MRI.getVRegDef(Reg);
     305             :   }
     306        4082 :   return Reg;
     307             : }
     308             : 
     309       31322 : unsigned FastISel::lookUpRegForValue(const Value *V) {
     310             :   // Look up the value to see if we already have a register for it. We
     311             :   // cache values defined by Instructions across blocks, and other values
     312             :   // only locally. This is because Instructions already have the SSA
     313             :   // def-dominates-use requirement enforced.
     314       31322 :   DenseMap<const Value *, unsigned>::iterator I = FuncInfo.ValueMap.find(V);
     315       93966 :   if (I != FuncInfo.ValueMap.end())
     316       15091 :     return I->second;
     317       32462 :   return LocalValueMap[V];
     318             : }
     319             : 
     320       12794 : void FastISel::updateValueMap(const Value *I, unsigned Reg, unsigned NumRegs) {
     321       25588 :   if (!isa<Instruction>(I)) {
     322        9134 :     LocalValueMap[I] = Reg;
     323        4567 :     return;
     324             :   }
     325             : 
     326       16454 :   unsigned &AssignedReg = FuncInfo.ValueMap[I];
     327        8227 :   if (AssignedReg == 0)
     328             :     // Use the new register.
     329         249 :     AssignedReg = Reg;
     330        7978 :   else if (Reg != AssignedReg) {
     331             :     // Arrange for uses of AssignedReg to be replaced by uses of Reg.
     332       24182 :     for (unsigned i = 0; i < NumRegs; i++)
     333       16204 :       FuncInfo.RegFixups[AssignedReg + i] = Reg + i;
     334             : 
     335        7978 :     AssignedReg = Reg;
     336             :   }
     337             : }
     338             : 
     339          95 : std::pair<unsigned, bool> FastISel::getRegForGEPIndex(const Value *Idx) {
     340          95 :   unsigned IdxN = getRegForValue(Idx);
     341          95 :   if (IdxN == 0)
     342             :     // Unhandled operand. Halt "fast" selection and bail.
     343          10 :     return std::pair<unsigned, bool>(0, false);
     344             : 
     345          90 :   bool IdxNIsKill = hasTrivialKill(Idx);
     346             : 
     347             :   // If the index is smaller or larger than intptr_t, truncate or extend it.
     348         180 :   MVT PtrVT = TLI.getPointerTy(DL);
     349          90 :   EVT IdxVT = EVT::getEVT(Idx->getType(), /*HandleUnknown=*/false);
     350          90 :   if (IdxVT.bitsLT(PtrVT)) {
     351          12 :     IdxN = fastEmit_r(IdxVT.getSimpleVT(), PtrVT, ISD::SIGN_EXTEND, IdxN,
     352          12 :                       IdxNIsKill);
     353          12 :     IdxNIsKill = true;
     354          78 :   } else if (IdxVT.bitsGT(PtrVT)) {
     355           7 :     IdxN =
     356           7 :         fastEmit_r(IdxVT.getSimpleVT(), PtrVT, ISD::TRUNCATE, IdxN, IdxNIsKill);
     357           7 :     IdxNIsKill = true;
     358             :   }
     359          90 :   return std::pair<unsigned, bool>(IdxN, IdxNIsKill);
     360             : }
     361             : 
     362       63288 : void FastISel::recomputeInsertPt() {
     363       63288 :   if (getLastLocalValue()) {
     364       93748 :     FuncInfo.InsertPt = getLastLocalValue();
     365       93748 :     FuncInfo.MBB = FuncInfo.InsertPt->getParent();
     366       46874 :     ++FuncInfo.InsertPt;
     367             :   } else
     368       16414 :     FuncInfo.InsertPt = FuncInfo.MBB->getFirstNonPHI();
     369             : 
     370             :   // Now skip past any EH_LABELs, which must remain at the beginning.
     371      232448 :   while (FuncInfo.InsertPt != FuncInfo.MBB->end() &&
     372       85168 :          FuncInfo.InsertPt->getOpcode() == TargetOpcode::EH_LABEL)
     373           0 :     ++FuncInfo.InsertPt;
     374       63288 : }
     375             : 
     376         144 : void FastISel::removeDeadCode(MachineBasicBlock::iterator I,
     377             :                               MachineBasicBlock::iterator E) {
     378             :   assert(I.isValid() && E.isValid() && std::distance(I, E) > 0 &&
     379             :          "Invalid iterator!");
     380         395 :   while (I != E) {
     381         251 :     MachineInstr *Dead = &*I;
     382         251 :     ++I;
     383         251 :     Dead->eraseFromParent();
     384             :     ++NumFastIselDead;
     385             :   }
     386         144 :   recomputeInsertPt();
     387         144 : }
     388             : 
     389        4153 : FastISel::SavePoint FastISel::enterLocalValueArea() {
     390        4153 :   MachineBasicBlock::iterator OldInsertPt = FuncInfo.InsertPt;
     391       12459 :   DebugLoc OldDL = DbgLoc;
     392        4153 :   recomputeInsertPt();
     393       16612 :   DbgLoc = DebugLoc();
     394        8306 :   SavePoint SP = {OldInsertPt, OldDL};
     395        4153 :   return SP;
     396             : }
     397             : 
     398        4153 : void FastISel::leaveLocalValueArea(SavePoint OldInsertPt) {
     399       12459 :   if (FuncInfo.InsertPt != FuncInfo.MBB->begin())
     400        8156 :     LastLocalValue = &*std::prev(FuncInfo.InsertPt);
     401             : 
     402             :   // Restore the previous insert position.
     403        4153 :   FuncInfo.InsertPt = OldInsertPt.InsertPt;
     404        8306 :   DbgLoc = OldInsertPt.DL;
     405        4153 : }
     406             : 
     407        1424 : bool FastISel::selectBinaryOp(const User *I, unsigned ISDOpcode) {
     408        1424 :   EVT VT = EVT::getEVT(I->getType(), /*HandleUnknown=*/true);
     409        2848 :   if (VT == MVT::Other || !VT.isSimple())
     410             :     // Unhandled type. Halt "fast" selection and bail.
     411             :     return false;
     412             : 
     413             :   // We only handle legal types. For example, on x86-32 the instruction
     414             :   // selector contains all of the 64-bit instructions from x86-64,
     415             :   // under the assumption that i64 won't be used if the target doesn't
     416             :   // support it.
     417        1555 :   if (!TLI.isTypeLegal(VT)) {
     418             :     // MVT::i1 is special. Allow AND, OR, or XOR because they
     419             :     // don't require additional zeroing, which makes them easy.
     420         156 :     if (VT == MVT::i1 && (ISDOpcode == ISD::AND || ISDOpcode == ISD::OR ||
     421             :                           ISDOpcode == ISD::XOR))
     422          36 :       VT = TLI.getTypeToTransformTo(I->getContext(), VT);
     423             :     else
     424             :       return false;
     425             :   }
     426             : 
     427             :   // Check if the first operand is a constant, and handle it as "ri".  At -O0,
     428             :   // we don't have anything that canonicalizes operand order.
     429        2669 :   if (const auto *CI = dyn_cast<ConstantInt>(I->getOperand(0)))
     430         163 :     if (isa<Instruction>(I) && cast<Instruction>(I)->isCommutative()) {
     431          44 :       unsigned Op1 = getRegForValue(I->getOperand(1));
     432          22 :       if (!Op1)
     433             :         return false;
     434          44 :       bool Op1IsKill = hasTrivialKill(I->getOperand(1));
     435             : 
     436             :       unsigned ResultReg =
     437          44 :           fastEmit_ri_(VT.getSimpleVT(), ISDOpcode, Op1, Op1IsKill,
     438          22 :                        CI->getZExtValue(), VT.getSimpleVT());
     439          22 :       if (!ResultReg)
     440             :         return false;
     441             : 
     442             :       // We successfully emitted code for the given LLVM Instruction.
     443          22 :       updateValueMap(I, ResultReg);
     444          22 :       return true;
     445             :     }
     446             : 
     447        2578 :   unsigned Op0 = getRegForValue(I->getOperand(0));
     448        1289 :   if (!Op0) // Unhandled operand. Halt "fast" selection and bail.
     449             :     return false;
     450        2558 :   bool Op0IsKill = hasTrivialKill(I->getOperand(0));
     451             : 
     452             :   // Check if the second operand is a constant and handle it appropriately.
     453        2911 :   if (const auto *CI = dyn_cast<ConstantInt>(I->getOperand(1))) {
     454         353 :     uint64_t Imm = CI->getSExtValue();
     455             : 
     456             :     // Transform "sdiv exact X, 8" -> "sra X, 3".
     457           8 :     if (ISDOpcode == ISD::SDIV && isa<BinaryOperator>(I) &&
     458         365 :         cast<BinaryOperator>(I)->isExact() && isPowerOf2_64(Imm)) {
     459           4 :       Imm = Log2_64(Imm);
     460           4 :       ISDOpcode = ISD::SRA;
     461             :     }
     462             : 
     463             :     // Transform "urem x, pow2" -> "and x, pow2-1".
     464         352 :     if (ISDOpcode == ISD::UREM && isa<BinaryOperator>(I) &&
     465             :         isPowerOf2_64(Imm)) {
     466             :       --Imm;
     467             :       ISDOpcode = ISD::AND;
     468             :     }
     469             : 
     470         353 :     unsigned ResultReg = fastEmit_ri_(VT.getSimpleVT(), ISDOpcode, Op0,
     471         353 :                                       Op0IsKill, Imm, VT.getSimpleVT());
     472         353 :     if (!ResultReg)
     473             :       return false;
     474             : 
     475             :     // We successfully emitted code for the given LLVM Instruction.
     476         342 :     updateValueMap(I, ResultReg);
     477         342 :     return true;
     478             :   }
     479             : 
     480        1852 :   unsigned Op1 = getRegForValue(I->getOperand(1));
     481         926 :   if (!Op1) // Unhandled operand. Halt "fast" selection and bail.
     482             :     return false;
     483        1804 :   bool Op1IsKill = hasTrivialKill(I->getOperand(1));
     484             : 
     485             :   // Now we have both operands in registers. Emit the instruction.
     486         902 :   unsigned ResultReg = fastEmit_rr(VT.getSimpleVT(), VT.getSimpleVT(),
     487        1804 :                                    ISDOpcode, Op0, Op0IsKill, Op1, Op1IsKill);
     488         902 :   if (!ResultReg)
     489             :     // Target-specific code wasn't able to find a machine opcode for
     490             :     // the given ISD opcode and type. Halt "fast" selection and bail.
     491             :     return false;
     492             : 
     493             :   // We successfully emitted code for the given LLVM Instruction.
     494         742 :   updateValueMap(I, ResultReg);
     495         742 :   return true;
     496             : }
     497             : 
     498         251 : bool FastISel::selectGetElementPtr(const User *I) {
     499         251 :   unsigned N = getRegForValue(I->getOperand(0));
     500         251 :   if (!N) // Unhandled operand. Halt "fast" selection and bail.
     501             :     return false;
     502         218 :   bool NIsKill = hasTrivialKill(I->getOperand(0));
     503             : 
     504             :   // Keep a running tab of the total offset to coalesce multiple N = N + Offset
     505             :   // into a single N = N + TotalOffset.
     506         218 :   uint64_t TotalOffs = 0;
     507             :   // FIXME: What's a good SWAG number for MaxOffs?
     508         218 :   uint64_t MaxOffs = 2048;
     509         436 :   MVT VT = TLI.getPointerTy(DL);
     510         684 :   for (gep_type_iterator GTI = gep_type_begin(I), E = gep_type_end(I);
     511         684 :        GTI != E; ++GTI) {
     512         958 :     const Value *Idx = GTI.getOperand();
     513          37 :     if (StructType *StTy = GTI.getStructTypeOrNull()) {
     514          74 :       uint64_t Field = cast<ConstantInt>(Idx)->getZExtValue();
     515          37 :       if (Field) {
     516             :         // N = N + Offset
     517          29 :         TotalOffs += DL.getStructLayout(StTy)->getElementOffset(Field);
     518          29 :         if (TotalOffs >= MaxOffs) {
     519           2 :           N = fastEmit_ri_(VT, ISD::ADD, N, NIsKill, TotalOffs, VT);
     520           2 :           if (!N) // Unhandled operand. Halt "fast" selection and bail.
     521          13 :             return false;
     522             :           NIsKill = true;
     523             :           TotalOffs = 0;
     524             :         }
     525             :       }
     526             :     } else {
     527         442 :       Type *Ty = GTI.getIndexedType();
     528             : 
     529             :       // If this is a constant subscript, handle it quickly.
     530         404 :       if (const auto *CI = dyn_cast<ConstantInt>(Idx)) {
     531         404 :         if (CI->isZero())
     532         673 :           continue;
     533             :         // N = N + Offset
     534         405 :         uint64_t IdxN = CI->getValue().sextOrTrunc(64).getSExtValue();
     535         135 :         TotalOffs += DL.getTypeAllocSize(Ty) * IdxN;
     536         135 :         if (TotalOffs >= MaxOffs) {
     537          11 :           N = fastEmit_ri_(VT, ISD::ADD, N, NIsKill, TotalOffs, VT);
     538          11 :           if (!N) // Unhandled operand. Halt "fast" selection and bail.
     539          13 :             return false;
     540             :           NIsKill = true;
     541             :           TotalOffs = 0;
     542             :         }
     543         135 :         continue;
     544             :       }
     545          38 :       if (TotalOffs) {
     546           0 :         N = fastEmit_ri_(VT, ISD::ADD, N, NIsKill, TotalOffs, VT);
     547           0 :         if (!N) // Unhandled operand. Halt "fast" selection and bail.
     548             :           return false;
     549             :         NIsKill = true;
     550             :         TotalOffs = 0;
     551             :       }
     552             : 
     553             :       // N = N + Idx * ElementSize;
     554          38 :       uint64_t ElementSize = DL.getTypeAllocSize(Ty);
     555          38 :       std::pair<unsigned, bool> Pair = getRegForGEPIndex(Idx);
     556          38 :       unsigned IdxN = Pair.first;
     557          38 :       bool IdxNIsKill = Pair.second;
     558          38 :       if (!IdxN) // Unhandled operand. Halt "fast" selection and bail.
     559             :         return false;
     560             : 
     561          27 :       if (ElementSize != 1) {
     562          11 :         IdxN = fastEmit_ri_(VT, ISD::MUL, IdxN, IdxNIsKill, ElementSize, VT);
     563          11 :         if (!IdxN) // Unhandled operand. Halt "fast" selection and bail.
     564             :           return false;
     565             :         IdxNIsKill = true;
     566             :       }
     567          25 :       N = fastEmit_rr(VT, VT, ISD::ADD, N, NIsKill, IdxN, IdxNIsKill);
     568          25 :       if (!N) // Unhandled operand. Halt "fast" selection and bail.
     569             :         return false;
     570             :     }
     571             :   }
     572         205 :   if (TotalOffs) {
     573          83 :     N = fastEmit_ri_(VT, ISD::ADD, N, NIsKill, TotalOffs, VT);
     574          83 :     if (!N) // Unhandled operand. Halt "fast" selection and bail.
     575             :       return false;
     576             :   }
     577             : 
     578             :   // We successfully emitted code for the given LLVM Instruction.
     579         205 :   updateValueMap(I, N);
     580         205 :   return true;
     581             : }
     582             : 
     583          65 : bool FastISel::addStackMapLiveVars(SmallVectorImpl<MachineOperand> &Ops,
     584             :                                    const CallInst *CI, unsigned StartIdx) {
     585         251 :   for (unsigned i = StartIdx, e = CI->getNumArgOperands(); i != e; ++i) {
     586         121 :     Value *Val = CI->getArgOperand(i);
     587             :     // Check for constants and encode them with a StackMaps::ConstantOp prefix.
     588         139 :     if (const auto *C = dyn_cast<ConstantInt>(Val)) {
     589          36 :       Ops.push_back(MachineOperand::CreateImm(StackMaps::ConstantOp));
     590          54 :       Ops.push_back(MachineOperand::CreateImm(C->getSExtValue()));
     591         206 :     } else if (isa<ConstantPointerNull>(Val)) {
     592           0 :       Ops.push_back(MachineOperand::CreateImm(StackMaps::ConstantOp));
     593           0 :       Ops.push_back(MachineOperand::CreateImm(0));
     594         111 :     } else if (auto *AI = dyn_cast<AllocaInst>(Val)) {
     595             :       // Values coming from a stack location also require a special encoding,
     596             :       // but that is added later on by the target specific frame index
     597             :       // elimination implementation.
     598           8 :       auto SI = FuncInfo.StaticAllocaMap.find(AI);
     599          24 :       if (SI != FuncInfo.StaticAllocaMap.end())
     600          16 :         Ops.push_back(MachineOperand::CreateFI(SI->second));
     601             :       else
     602           0 :         return false;
     603             :     } else {
     604          95 :       unsigned Reg = getRegForValue(Val);
     605          95 :       if (!Reg)
     606             :         return false;
     607         190 :       Ops.push_back(MachineOperand::CreateReg(Reg, /*IsDef=*/false));
     608             :     }
     609             :   }
     610             :   return true;
     611             : }
     612             : 
     613          26 : bool FastISel::selectStackmap(const CallInst *I) {
     614             :   // void @llvm.experimental.stackmap(i64 <id>, i32 <numShadowBytes>,
     615             :   //                                  [live variables...])
     616             :   assert(I->getCalledFunction()->getReturnType()->isVoidTy() &&
     617             :          "Stackmap cannot return a value.");
     618             : 
     619             :   // The stackmap intrinsic only records the live variables (the arguments
     620             :   // passed to it) and emits NOPS (if requested). Unlike the patchpoint
     621             :   // intrinsic, this won't be lowered to a function call. This means we don't
     622             :   // have to worry about calling conventions and target-specific lowering code.
     623             :   // Instead we perform the call lowering right here.
     624             :   //
     625             :   // CALLSEQ_START(0, 0...)
     626             :   // STACKMAP(id, nbytes, ...)
     627             :   // CALLSEQ_END(0, 0)
     628             :   //
     629          52 :   SmallVector<MachineOperand, 32> Ops;
     630             : 
     631             :   // Add the <id> and <numBytes> constants.
     632             :   assert(isa<ConstantInt>(I->getOperand(PatchPointOpers::IDPos)) &&
     633             :          "Expected a constant integer.");
     634          52 :   const auto *ID = cast<ConstantInt>(I->getOperand(PatchPointOpers::IDPos));
     635          52 :   Ops.push_back(MachineOperand::CreateImm(ID->getZExtValue()));
     636             : 
     637             :   assert(isa<ConstantInt>(I->getOperand(PatchPointOpers::NBytesPos)) &&
     638             :          "Expected a constant integer.");
     639             :   const auto *NumBytes =
     640          52 :       cast<ConstantInt>(I->getOperand(PatchPointOpers::NBytesPos));
     641          52 :   Ops.push_back(MachineOperand::CreateImm(NumBytes->getZExtValue()));
     642             : 
     643             :   // Push live variables for the stack map (skipping the first two arguments
     644             :   // <id> and <numBytes>).
     645          26 :   if (!addStackMapLiveVars(Ops, I, 2))
     646             :     return false;
     647             : 
     648             :   // We are not adding any register mask info here, because the stackmap doesn't
     649             :   // clobber anything.
     650             : 
     651             :   // Add scratch registers as implicit def and early clobber.
     652          26 :   CallingConv::ID CC = I->getCallingConv();
     653          26 :   const MCPhysReg *ScratchRegs = TLI.getScratchRegisters(CC);
     654          82 :   for (unsigned i = 0; ScratchRegs[i]; ++i)
     655          56 :     Ops.push_back(MachineOperand::CreateReg(
     656             :         ScratchRegs[i], /*IsDef=*/true, /*IsImp=*/true, /*IsKill=*/false,
     657          56 :         /*IsDead=*/false, /*IsUndef=*/false, /*IsEarlyClobber=*/true));
     658             : 
     659             :   // Issue CALLSEQ_START
     660          26 :   unsigned AdjStackDown = TII.getCallFrameSetupOpcode();
     661             :   auto Builder =
     662          52 :       BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc, TII.get(AdjStackDown));
     663          26 :   const MCInstrDesc &MCID = Builder.getInstr()->getDesc();
     664         115 :   for (unsigned I = 0, E = MCID.getNumOperands(); I < E; ++I)
     665          63 :     Builder.addImm(0);
     666             : 
     667             :   // Issue STACKMAP.
     668          26 :   MachineInstrBuilder MIB = BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc,
     669          78 :                                     TII.get(TargetOpcode::STACKMAP));
     670         283 :   for (auto const &MO : Ops)
     671         205 :     MIB.add(MO);
     672             : 
     673             :   // Issue CALLSEQ_END
     674          26 :   unsigned AdjStackUp = TII.getCallFrameDestroyOpcode();
     675          52 :   BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc, TII.get(AdjStackUp))
     676          26 :       .addImm(0)
     677          26 :       .addImm(0);
     678             : 
     679             :   // Inform the Frame Information that we have a stackmap in this function.
     680          52 :   FuncInfo.MF->getFrameInfo().setHasStackMap();
     681             : 
     682          26 :   return true;
     683             : }
     684             : 
     685             : /// \brief Lower an argument list according to the target calling convention.
     686             : ///
     687             : /// This is a helper for lowering intrinsics that follow a target calling
     688             : /// convention or require stack pointer adjustment. Only a subset of the
     689             : /// intrinsic's operands need to participate in the calling convention.
     690          39 : bool FastISel::lowerCallOperands(const CallInst *CI, unsigned ArgIdx,
     691             :                                  unsigned NumArgs, const Value *Callee,
     692             :                                  bool ForceRetVoidTy, CallLoweringInfo &CLI) {
     693          78 :   ArgListTy Args;
     694          39 :   Args.reserve(NumArgs);
     695             : 
     696             :   // Populate the argument list.
     697          39 :   ImmutableCallSite CS(CI);
     698         151 :   for (unsigned ArgI = ArgIdx, ArgE = ArgIdx + NumArgs; ArgI != ArgE; ++ArgI) {
     699         112 :     Value *V = CI->getOperand(ArgI);
     700             : 
     701             :     assert(!V->getType()->isEmptyTy() && "Empty type passed to intrinsic.");
     702             : 
     703         112 :     ArgListEntry Entry;
     704         112 :     Entry.Val = V;
     705         112 :     Entry.Ty = V->getType();
     706         112 :     Entry.setAttributes(&CS, ArgIdx);
     707         112 :     Args.push_back(Entry);
     708             :   }
     709             : 
     710          77 :   Type *RetTy = ForceRetVoidTy ? Type::getVoidTy(CI->getType()->getContext())
     711          77 :                                : CI->getType();
     712         117 :   CLI.setCallee(CI->getCallingConv(), RetTy, Callee, std::move(Args), NumArgs);
     713             : 
     714          78 :   return lowerCallTo(CLI);
     715             : }
     716             : 
     717          16 : FastISel::CallLoweringInfo &FastISel::CallLoweringInfo::setCallee(
     718             :     const DataLayout &DL, MCContext &Ctx, CallingConv::ID CC, Type *ResultTy,
     719             :     StringRef Target, ArgListTy &&ArgsList, unsigned FixedArgs) {
     720          32 :   SmallString<32> MangledName;
     721          16 :   Mangler::getNameWithPrefix(MangledName, Target, DL);
     722          16 :   MCSymbol *Sym = Ctx.getOrCreateSymbol(MangledName);
     723          48 :   return setCallee(CC, ResultTy, Sym, std::move(ArgsList), FixedArgs);
     724             : }
     725             : 
     726          39 : bool FastISel::selectPatchpoint(const CallInst *I) {
     727             :   // void|i64 @llvm.experimental.patchpoint.void|i64(i64 <id>,
     728             :   //                                                 i32 <numBytes>,
     729             :   //                                                 i8* <target>,
     730             :   //                                                 i32 <numArgs>,
     731             :   //                                                 [Args...],
     732             :   //                                                 [live variables...])
     733          39 :   CallingConv::ID CC = I->getCallingConv();
     734          39 :   bool IsAnyRegCC = CC == CallingConv::AnyReg;
     735          78 :   bool HasDef = !I->getType()->isVoidTy();
     736          78 :   Value *Callee = I->getOperand(PatchPointOpers::TargetPos)->stripPointerCasts();
     737             : 
     738             :   // Get the real number of arguments participating in the call <numArgs>
     739             :   assert(isa<ConstantInt>(I->getOperand(PatchPointOpers::NArgPos)) &&
     740             :          "Expected a constant integer.");
     741             :   const auto *NumArgsVal =
     742          78 :       cast<ConstantInt>(I->getOperand(PatchPointOpers::NArgPos));
     743          39 :   unsigned NumArgs = NumArgsVal->getZExtValue();
     744             : 
     745             :   // Skip the four meta args: <id>, <numNopBytes>, <target>, <numArgs>
     746             :   // This includes all meta-operands up to but not including CC.
     747          39 :   unsigned NumMetaOpers = PatchPointOpers::CCPos;
     748             :   assert(I->getNumArgOperands() >= NumMetaOpers + NumArgs &&
     749             :          "Not enough arguments provided to the patchpoint intrinsic");
     750             : 
     751             :   // For AnyRegCC the arguments are lowered later on manually.
     752          39 :   unsigned NumCallArgs = IsAnyRegCC ? 0 : NumArgs;
     753          78 :   CallLoweringInfo CLI;
     754          39 :   CLI.setIsPatchPoint();
     755          39 :   if (!lowerCallOperands(I, NumMetaOpers, NumCallArgs, Callee, IsAnyRegCC, CLI))
     756             :     return false;
     757             : 
     758             :   assert(CLI.Call && "No call instruction specified.");
     759             : 
     760          39 :   SmallVector<MachineOperand, 32> Ops;
     761             : 
     762             :   // Add an explicit result reg if we use the anyreg calling convention.
     763          39 :   if (IsAnyRegCC && HasDef) {
     764             :     assert(CLI.NumResultRegs == 0 && "Unexpected result register.");
     765           0 :     CLI.ResultReg = createResultReg(TLI.getRegClassFor(MVT::i64));
     766           0 :     CLI.NumResultRegs = 1;
     767           0 :     Ops.push_back(MachineOperand::CreateReg(CLI.ResultReg, /*IsDef=*/true));
     768             :   }
     769             : 
     770             :   // Add the <id> and <numBytes> constants.
     771             :   assert(isa<ConstantInt>(I->getOperand(PatchPointOpers::IDPos)) &&
     772             :          "Expected a constant integer.");
     773          78 :   const auto *ID = cast<ConstantInt>(I->getOperand(PatchPointOpers::IDPos));
     774          78 :   Ops.push_back(MachineOperand::CreateImm(ID->getZExtValue()));
     775             : 
     776             :   assert(isa<ConstantInt>(I->getOperand(PatchPointOpers::NBytesPos)) &&
     777             :          "Expected a constant integer.");
     778             :   const auto *NumBytes =
     779          78 :       cast<ConstantInt>(I->getOperand(PatchPointOpers::NBytesPos));
     780          78 :   Ops.push_back(MachineOperand::CreateImm(NumBytes->getZExtValue()));
     781             : 
     782             :   // Add the call target.
     783          58 :   if (const auto *C = dyn_cast<IntToPtrInst>(Callee)) {
     784             :     uint64_t CalleeConstAddr =
     785          76 :       cast<ConstantInt>(C->getOperand(0))->getZExtValue();
     786          38 :     Ops.push_back(MachineOperand::CreateImm(CalleeConstAddr));
     787          24 :   } else if (const auto *C = dyn_cast<ConstantExpr>(Callee)) {
     788           4 :     if (C->getOpcode() == Instruction::IntToPtr) {
     789             :       uint64_t CalleeConstAddr =
     790          12 :         cast<ConstantInt>(C->getOperand(0))->getZExtValue();
     791           8 :       Ops.push_back(MachineOperand::CreateImm(CalleeConstAddr));
     792             :     } else
     793           0 :       llvm_unreachable("Unsupported ConstantExpr.");
     794          19 :   } else if (const auto *GV = dyn_cast<GlobalValue>(Callee)) {
     795           3 :     Ops.push_back(MachineOperand::CreateGA(GV, 0));
     796          26 :   } else if (isa<ConstantPointerNull>(Callee))
     797          13 :     Ops.push_back(MachineOperand::CreateImm(0));
     798             :   else
     799           0 :     llvm_unreachable("Unsupported callee address.");
     800             : 
     801             :   // Adjust <numArgs> to account for any arguments that have been passed on
     802             :   // the stack instead.
     803          77 :   unsigned NumCallRegArgs = IsAnyRegCC ? NumArgs : CLI.OutRegs.size();
     804          78 :   Ops.push_back(MachineOperand::CreateImm(NumCallRegArgs));
     805             : 
     806             :   // Add the calling convention
     807          78 :   Ops.push_back(MachineOperand::CreateImm((unsigned)CC));
     808             : 
     809             :   // Add the arguments we omitted previously. The register allocator should
     810             :   // place these in any free register.
     811          39 :   if (IsAnyRegCC) {
     812           3 :     for (unsigned i = NumMetaOpers, e = NumMetaOpers + NumArgs; i != e; ++i) {
     813           2 :       unsigned Reg = getRegForValue(I->getArgOperand(i));
     814           2 :       if (!Reg)
     815             :         return false;
     816           2 :       Ops.push_back(MachineOperand::CreateReg(Reg, /*IsDef=*/false));
     817             :     }
     818             :   }
     819             : 
     820             :   // Push the arguments from the call instruction.
     821         165 :   for (auto Reg : CLI.OutRegs)
     822          48 :     Ops.push_back(MachineOperand::CreateReg(Reg, /*IsDef=*/false));
     823             : 
     824             :   // Push live variables for the stack map.
     825          39 :   if (!addStackMapLiveVars(Ops, I, NumMetaOpers + NumArgs))
     826             :     return false;
     827             : 
     828             :   // Push the register mask info.
     829          39 :   Ops.push_back(MachineOperand::CreateRegMask(
     830          39 :       TRI.getCallPreservedMask(*FuncInfo.MF, CC)));
     831             : 
     832             :   // Add scratch registers as implicit def and early clobber.
     833          39 :   const MCPhysReg *ScratchRegs = TLI.getScratchRegisters(CC);
     834         134 :   for (unsigned i = 0; ScratchRegs[i]; ++i)
     835          95 :     Ops.push_back(MachineOperand::CreateReg(
     836             :         ScratchRegs[i], /*IsDef=*/true, /*IsImp=*/true, /*IsKill=*/false,
     837          95 :         /*IsDead=*/false, /*IsUndef=*/false, /*IsEarlyClobber=*/true));
     838             : 
     839             :   // Add implicit defs (return values).
     840         137 :   for (auto Reg : CLI.InRegs)
     841          20 :     Ops.push_back(MachineOperand::CreateReg(Reg, /*IsDef=*/true,
     842             :                                             /*IsImpl=*/true));
     843             : 
     844             :   // Insert the patchpoint instruction before the call generated by the target.
     845          39 :   MachineInstrBuilder MIB = BuildMI(*FuncInfo.MBB, CLI.Call, DbgLoc,
     846         117 :                                     TII.get(TargetOpcode::PATCHPOINT));
     847             : 
     848         558 :   for (auto &MO : Ops)
     849         441 :     MIB.add(MO);
     850             : 
     851          78 :   MIB->setPhysRegsDeadExcept(CLI.InRegs, TRI);
     852             : 
     853             :   // Delete the original call instruction.
     854          39 :   CLI.Call->eraseFromParent();
     855             : 
     856             :   // Inform the Frame Information that we have a patchpoint in this function.
     857          78 :   FuncInfo.MF->getFrameInfo().setHasPatchPoint();
     858             : 
     859          39 :   if (CLI.NumResultRegs)
     860          20 :     updateValueMap(I, CLI.ResultReg, CLI.NumResultRegs);
     861             :   return true;
     862             : }
     863             : 
     864           0 : bool FastISel::selectXRayCustomEvent(const CallInst *I) {
     865           0 :   const auto &Triple = TM.getTargetTriple();
     866           0 :   if (Triple.getArch() != Triple::x86_64 || !Triple.isOSLinux())
     867             :     return true; // don't do anything to this instruction.
     868           0 :   SmallVector<MachineOperand, 8> Ops;
     869           0 :   Ops.push_back(MachineOperand::CreateReg(getRegForValue(I->getArgOperand(0)),
     870           0 :                                           /*IsDef=*/false));
     871           0 :   Ops.push_back(MachineOperand::CreateReg(getRegForValue(I->getArgOperand(1)),
     872           0 :                                           /*IsDef=*/false));
     873             :   MachineInstrBuilder MIB =
     874           0 :       BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc,
     875           0 :               TII.get(TargetOpcode::PATCHABLE_EVENT_CALL));
     876           0 :   for (auto &MO : Ops)
     877           0 :     MIB.add(MO);
     878             :   // Insert the Patchable Event Call instruction, that gets lowered properly.
     879           0 :   return true;
     880             : }
     881             : 
     882             : 
     883             : /// Returns an AttributeList representing the attributes applied to the return
     884             : /// value of the given call.
     885        1858 : static AttributeList getReturnAttrs(FastISel::CallLoweringInfo &CLI) {
     886        3716 :   SmallVector<Attribute::AttrKind, 2> Attrs;
     887        1858 :   if (CLI.RetSExt)
     888          41 :     Attrs.push_back(Attribute::SExt);
     889        1858 :   if (CLI.RetZExt)
     890          36 :     Attrs.push_back(Attribute::ZExt);
     891        1858 :   if (CLI.IsInReg)
     892           0 :     Attrs.push_back(Attribute::InReg);
     893             : 
     894        1858 :   return AttributeList::get(CLI.RetTy->getContext(), AttributeList::ReturnIndex,
     895        5574 :                             Attrs);
     896             : }
     897             : 
     898          25 : bool FastISel::lowerCallTo(const CallInst *CI, const char *SymName,
     899             :                            unsigned NumArgs) {
     900          25 :   MCContext &Ctx = MF->getContext();
     901          50 :   SmallString<32> MangledName;
     902          50 :   Mangler::getNameWithPrefix(MangledName, SymName, DL);
     903          25 :   MCSymbol *Sym = Ctx.getOrCreateSymbol(MangledName);
     904          50 :   return lowerCallTo(CI, Sym, NumArgs);
     905             : }
     906             : 
     907          25 : bool FastISel::lowerCallTo(const CallInst *CI, MCSymbol *Symbol,
     908             :                            unsigned NumArgs) {
     909          25 :   ImmutableCallSite CS(CI);
     910             : 
     911          25 :   FunctionType *FTy = CS.getFunctionType();
     912          25 :   Type *RetTy = CS.getType();
     913             : 
     914          50 :   ArgListTy Args;
     915          25 :   Args.reserve(NumArgs);
     916             : 
     917             :   // Populate the argument list.
     918             :   // Attributes for args start at offset 1, after the return attribute.
     919         100 :   for (unsigned ArgI = 0; ArgI != NumArgs; ++ArgI) {
     920          75 :     Value *V = CI->getOperand(ArgI);
     921             : 
     922             :     assert(!V->getType()->isEmptyTy() && "Empty type passed to intrinsic.");
     923             : 
     924          75 :     ArgListEntry Entry;
     925          75 :     Entry.Val = V;
     926          75 :     Entry.Ty = V->getType();
     927          75 :     Entry.setAttributes(&CS, ArgI);
     928          75 :     Args.push_back(Entry);
     929             :   }
     930          50 :   TLI.markLibCallAttributes(MF, CS.getCallingConv(), Args);
     931             : 
     932          50 :   CallLoweringInfo CLI;
     933          25 :   CLI.setCallee(RetTy, FTy, Symbol, std::move(Args), CS, NumArgs);
     934             : 
     935          50 :   return lowerCallTo(CLI);
     936             : }
     937             : 
     938        1858 : bool FastISel::lowerCallTo(CallLoweringInfo &CLI) {
     939             :   // Handle the incoming return values from the call.
     940        1858 :   CLI.clearIns();
     941        3716 :   SmallVector<EVT, 4> RetTys;
     942        1858 :   ComputeValueVTs(TLI, DL, CLI.RetTy, RetTys);
     943             : 
     944        3716 :   SmallVector<ISD::OutputArg, 4> Outs;
     945        1858 :   GetReturnInfo(CLI.RetTy, getReturnAttrs(CLI), Outs, TLI, DL);
     946             : 
     947        7432 :   bool CanLowerReturn = TLI.CanLowerReturn(
     948        7432 :       CLI.CallConv, *FuncInfo.MF, CLI.IsVarArg, Outs, CLI.RetTy->getContext());
     949             : 
     950             :   // FIXME: sret demotion isn't supported yet - bail out.
     951        1858 :   if (!CanLowerReturn)
     952             :     return false;
     953             : 
     954        4505 :   for (unsigned I = 0, E = RetTys.size(); I != E; ++I) {
     955        1590 :     EVT VT = RetTys[I];
     956         795 :     MVT RegisterVT = TLI.getRegisterType(CLI.RetTy->getContext(), VT);
     957         795 :     unsigned NumRegs = TLI.getNumRegisters(CLI.RetTy->getContext(), VT);
     958        1590 :     for (unsigned i = 0; i != NumRegs; ++i) {
     959         795 :       ISD::InputArg MyFlags;
     960         795 :       MyFlags.VT = RegisterVT;
     961         795 :       MyFlags.ArgVT = VT;
     962         795 :       MyFlags.Used = CLI.IsReturnValueUsed;
     963         795 :       if (CLI.RetSExt)
     964             :         MyFlags.Flags.setSExt();
     965         795 :       if (CLI.RetZExt)
     966             :         MyFlags.Flags.setZExt();
     967         795 :       if (CLI.IsInReg)
     968             :         MyFlags.Flags.setInReg();
     969         795 :       CLI.Ins.push_back(MyFlags);
     970             :     }
     971             :   }
     972             : 
     973             :   // Handle all of the outgoing arguments.
     974        1855 :   CLI.clearOuts();
     975       11578 :   for (auto &Arg : CLI.getArgs()) {
     976        4158 :     Type *FinalType = Arg.Ty;
     977        4158 :     if (Arg.IsByVal)
     978          60 :       FinalType = cast<PointerType>(Arg.Ty)->getElementType();
     979        8316 :     bool NeedsRegBlock = TLI.functionArgumentNeedsConsecutiveRegisters(
     980        8316 :         FinalType, CLI.CallConv, CLI.IsVarArg);
     981             : 
     982        4158 :     ISD::ArgFlagsTy Flags;
     983        4158 :     if (Arg.IsZExt)
     984             :       Flags.setZExt();
     985        4158 :     if (Arg.IsSExt)
     986             :       Flags.setSExt();
     987        4158 :     if (Arg.IsInReg)
     988             :       Flags.setInReg();
     989        4158 :     if (Arg.IsSRet)
     990             :       Flags.setSRet();
     991        4158 :     if (Arg.IsSwiftSelf)
     992             :       Flags.setSwiftSelf();
     993        4158 :     if (Arg.IsSwiftError)
     994             :       Flags.setSwiftError();
     995        4158 :     if (Arg.IsByVal)
     996             :       Flags.setByVal();
     997        4158 :     if (Arg.IsInAlloca) {
     998           6 :       Flags.setInAlloca();
     999             :       // Set the byval flag for CCAssignFn callbacks that don't know about
    1000             :       // inalloca. This way we can know how many bytes we should've allocated
    1001             :       // and how many bytes a callee cleanup function will pop.  If we port
    1002             :       // inalloca to more targets, we'll have to add custom inalloca handling in
    1003             :       // the various CC lowering callbacks.
    1004             :       Flags.setByVal();
    1005             :     }
    1006        4158 :     if (Arg.IsByVal || Arg.IsInAlloca) {
    1007          72 :       PointerType *Ty = cast<PointerType>(Arg.Ty);
    1008          36 :       Type *ElementTy = Ty->getElementType();
    1009          36 :       unsigned FrameSize = DL.getTypeAllocSize(ElementTy);
    1010             :       // For ByVal, alignment should come from FE. BE will guess if this info is
    1011             :       // not there, but there are cases it cannot get right.
    1012          36 :       unsigned FrameAlign = Arg.Alignment;
    1013          36 :       if (!FrameAlign)
    1014           9 :         FrameAlign = TLI.getByValTypeAlignment(ElementTy, DL);
    1015          36 :       Flags.setByValSize(FrameSize);
    1016             :       Flags.setByValAlign(FrameAlign);
    1017             :     }
    1018        4158 :     if (Arg.IsNest)
    1019             :       Flags.setNest();
    1020        4158 :     if (NeedsRegBlock)
    1021             :       Flags.setInConsecutiveRegs();
    1022        4158 :     unsigned OriginalAlignment = DL.getABITypeAlignment(Arg.Ty);
    1023        4158 :     Flags.setOrigAlign(OriginalAlignment);
    1024             : 
    1025        4158 :     CLI.OutVals.push_back(Arg.Val);
    1026        4158 :     CLI.OutFlags.push_back(Flags);
    1027             :   }
    1028             : 
    1029        1855 :   if (!fastLowerCall(CLI))
    1030             :     return false;
    1031             : 
    1032             :   // Set all unused physreg defs as dead.
    1033             :   assert(CLI.Call && "No call instruction specified.");
    1034        1964 :   CLI.Call->setPhysRegsDeadExcept(CLI.InRegs, TRI);
    1035             : 
    1036         982 :   if (CLI.NumResultRegs && CLI.CS)
    1037         482 :     updateValueMap(CLI.CS->getInstruction(), CLI.ResultReg, CLI.NumResultRegs);
    1038             : 
    1039             :   return true;
    1040             : }
    1041             : 
    1042        1778 : bool FastISel::lowerCall(const CallInst *CI) {
    1043        1778 :   ImmutableCallSite CS(CI);
    1044             : 
    1045        1778 :   FunctionType *FuncTy = CS.getFunctionType();
    1046        1778 :   Type *RetTy = CS.getType();
    1047             : 
    1048        3556 :   ArgListTy Args;
    1049        1778 :   ArgListEntry Entry;
    1050        1778 :   Args.reserve(CS.arg_size());
    1051             : 
    1052        5728 :   for (ImmutableCallSite::arg_iterator i = CS.arg_begin(), e = CS.arg_end();
    1053        5728 :        i != e; ++i) {
    1054        3950 :     Value *V = *i;
    1055             : 
    1056             :     // Skip empty types
    1057        3950 :     if (V->getType()->isEmptyTy())
    1058           0 :       continue;
    1059             : 
    1060        3950 :     Entry.Val = V;
    1061        3950 :     Entry.Ty = V->getType();
    1062             : 
    1063             :     // Skip the first return-type Attribute to get to params.
    1064        7900 :     Entry.setAttributes(&CS, i - CS.arg_begin());
    1065        3950 :     Args.push_back(Entry);
    1066             :   }
    1067             : 
    1068             :   // Check if target-independent constraints permit a tail call here.
    1069             :   // Target-dependent constraints are checked within fastLowerCall.
    1070        1778 :   bool IsTailCall = CI->isTailCall();
    1071        1778 :   if (IsTailCall && !isInTailCallPosition(CS, TM))
    1072             :     IsTailCall = false;
    1073             : 
    1074        3556 :   CallLoweringInfo CLI;
    1075        3556 :   CLI.setCallee(RetTy, FuncTy, CI->getCalledValue(), std::move(Args), CS)
    1076        3556 :       .setTailCall(IsTailCall);
    1077             : 
    1078        3556 :   return lowerCallTo(CLI);
    1079             : }
    1080             : 
    1081        3681 : bool FastISel::selectCall(const User *I) {
    1082        3681 :   const CallInst *Call = cast<CallInst>(I);
    1083             : 
    1084             :   // Handle simple inline asms.
    1085        3817 :   if (const InlineAsm *IA = dyn_cast<InlineAsm>(Call->getCalledValue())) {
    1086             :     // If the inline asm has side effects, then make sure that no local value
    1087             :     // lives across by flushing the local value map.
    1088         136 :     if (IA->hasSideEffects())
    1089         121 :       flushLocalValueMap();
    1090             : 
    1091             :     // Don't attempt to handle constraints.
    1092         136 :     if (!IA->getConstraintString().empty())
    1093             :       return false;
    1094             : 
    1095          16 :     unsigned ExtraInfo = 0;
    1096          16 :     if (IA->hasSideEffects())
    1097          16 :       ExtraInfo |= InlineAsm::Extra_HasSideEffects;
    1098          16 :     if (IA->isAlignStack())
    1099           0 :       ExtraInfo |= InlineAsm::Extra_IsAlignStack;
    1100             : 
    1101          16 :     BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc,
    1102          32 :             TII.get(TargetOpcode::INLINEASM))
    1103          32 :         .addExternalSymbol(IA->getAsmString().c_str())
    1104          32 :         .addImm(ExtraInfo);
    1105          16 :     return true;
    1106             :   }
    1107             : 
    1108        3545 :   MachineModuleInfo &MMI = FuncInfo.MF->getMMI();
    1109        3545 :   computeUsesVAFloatArgument(*Call, MMI);
    1110             : 
    1111             :   // Handle intrinsic function calls.
    1112        1767 :   if (const auto *II = dyn_cast<IntrinsicInst>(Call))
    1113        1767 :     return selectIntrinsicCall(II);
    1114             : 
    1115             :   // Usually, it does not make sense to initialize a value,
    1116             :   // make an unrelated function call and use the value, because
    1117             :   // it tends to be spilled on the stack. So, we move the pointer
    1118             :   // to the last local value to the beginning of the block, so that
    1119             :   // all the values which have already been materialized,
    1120             :   // appear after the call. It also makes sense to skip intrinsics
    1121             :   // since they tend to be inlined.
    1122        1778 :   flushLocalValueMap();
    1123             : 
    1124        1778 :   return lowerCall(Call);
    1125             : }
    1126             : 
    1127        1767 : bool FastISel::selectIntrinsicCall(const IntrinsicInst *II) {
    1128        1767 :   switch (II->getIntrinsicID()) {
    1129             :   default:
    1130             :     break;
    1131             :   // At -O0 we don't care about the lifetime intrinsics.
    1132             :   case Intrinsic::lifetime_start:
    1133             :   case Intrinsic::lifetime_end:
    1134             :   // The donothing intrinsic does, well, nothing.
    1135             :   case Intrinsic::donothing:
    1136             :   // Neither does the assume intrinsic; it's also OK not to codegen its operand.
    1137             :   case Intrinsic::assume:
    1138             :     return true;
    1139         275 :   case Intrinsic::dbg_declare: {
    1140         275 :     const DbgDeclareInst *DI = cast<DbgDeclareInst>(II);
    1141             :     assert(DI->getVariable() && "Missing variable");
    1142         275 :     if (!FuncInfo.MF->getMMI().hasDebugInfo()) {
    1143             :       DEBUG(dbgs() << "Dropping debug info for " << *DI << "\n");
    1144             :       return true;
    1145             :     }
    1146             : 
    1147         275 :     const Value *Address = DI->getAddress();
    1148         550 :     if (!Address || isa<UndefValue>(Address)) {
    1149             :       DEBUG(dbgs() << "Dropping debug info for " << *DI << "\n");
    1150             :       return true;
    1151             :     }
    1152             : 
    1153             :     // Byval arguments with frame indices were already handled after argument
    1154             :     // lowering and before isel.
    1155             :     const auto *Arg =
    1156         302 :         dyn_cast<Argument>(Address->stripInBoundsConstantOffsets());
    1157          29 :     if (Arg && FuncInfo.getArgumentFrameIndex(Arg) != INT_MAX)
    1158             :       return true;
    1159             : 
    1160         263 :     Optional<MachineOperand> Op;
    1161         263 :     if (unsigned Reg = lookUpRegForValue(Address))
    1162          36 :       Op = MachineOperand::CreateReg(Reg, false);
    1163             : 
    1164             :     // If we have a VLA that has a "use" in a metadata node that's then used
    1165             :     // here but it has no other uses, then we have a problem. E.g.,
    1166             :     //
    1167             :     //   int foo (const int *x) {
    1168             :     //     char a[*x];
    1169             :     //     return 0;
    1170             :     //   }
    1171             :     //
    1172             :     // If we assign 'a' a vreg and fast isel later on has to use the selection
    1173             :     // DAG isel, it will want to copy the value to the vreg. However, there are
    1174             :     // no uses, which goes counter to what selection DAG isel expects.
    1175         694 :     if (!Op && !Address->use_empty() && isa<Instruction>(Address) &&
    1176         204 :         (!isa<AllocaInst>(Address) ||
    1177         406 :          !FuncInfo.StaticAllocaMap.count(cast<AllocaInst>(Address))))
    1178           2 :       Op = MachineOperand::CreateReg(FuncInfo.InitializeRegForValue(Address),
    1179           1 :                                      false);
    1180             : 
    1181         263 :     if (Op) {
    1182             :       assert(DI->getVariable()->isValidLocationForIntrinsic(DbgLoc) &&
    1183             :              "Expected inlined-at fields to agree");
    1184          74 :       if (Op->isReg()) {
    1185          74 :         Op->setIsDebug(true);
    1186             :         // A dbg.declare describes the address of a source variable, so lower it
    1187             :         // into an indirect DBG_VALUE.
    1188          37 :         BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc,
    1189          37 :                 TII.get(TargetOpcode::DBG_VALUE), /*IsIndirect*/ true,
    1190         222 :                 Op->getReg(), DI->getVariable(), DI->getExpression());
    1191             :       } else
    1192           0 :         BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc,
    1193           0 :                 TII.get(TargetOpcode::DBG_VALUE))
    1194           0 :             .add(*Op)
    1195           0 :             .addImm(0)
    1196           0 :             .addMetadata(DI->getVariable())
    1197           0 :             .addMetadata(DI->getExpression());
    1198             :     } else {
    1199             :       // We can't yet handle anything else here because it would require
    1200             :       // generating code, thus altering codegen because of debug info.
    1201             :       DEBUG(dbgs() << "Dropping debug info for " << *DI << "\n");
    1202             :     }
    1203         263 :     return true;
    1204             :   }
    1205          58 :   case Intrinsic::dbg_value: {
    1206             :     // This form of DBG_VALUE is target-independent.
    1207          58 :     const DbgValueInst *DI = cast<DbgValueInst>(II);
    1208         116 :     const MCInstrDesc &II = TII.get(TargetOpcode::DBG_VALUE);
    1209          58 :     const Value *V = DI->getValue();
    1210             :     assert(DI->getVariable()->isValidLocationForIntrinsic(DbgLoc) &&
    1211             :            "Expected inlined-at fields to agree");
    1212          58 :     if (!V) {
    1213             :       // Currently the optimizer can produce this; insert an undef to
    1214             :       // help debugging.  Probably the optimizer should not do this.
    1215           0 :       BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc, II, false, 0U,
    1216           0 :               DI->getVariable(), DI->getExpression());
    1217          58 :     } else if (const auto *CI = dyn_cast<ConstantInt>(V)) {
    1218          16 :       if (CI->getBitWidth() > 64)
    1219           0 :         BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc, II)
    1220           0 :             .addCImm(CI)
    1221           0 :             .addImm(0U)
    1222           0 :             .addMetadata(DI->getVariable())
    1223           0 :             .addMetadata(DI->getExpression());
    1224             :       else
    1225          16 :         BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc, II)
    1226          32 :             .addImm(CI->getZExtValue())
    1227          16 :             .addImm(0U)
    1228          48 :             .addMetadata(DI->getVariable())
    1229          48 :             .addMetadata(DI->getExpression());
    1230          42 :     } else if (const auto *CF = dyn_cast<ConstantFP>(V)) {
    1231           0 :       BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc, II)
    1232           0 :           .addFPImm(CF)
    1233           0 :           .addImm(0U)
    1234           0 :           .addMetadata(DI->getVariable())
    1235           0 :           .addMetadata(DI->getExpression());
    1236          42 :     } else if (unsigned Reg = lookUpRegForValue(V)) {
    1237             :       // FIXME: This does not handle register-indirect values at offset 0.
    1238          35 :       bool IsIndirect = false;
    1239          35 :       BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc, II, IsIndirect, Reg,
    1240         105 :               DI->getVariable(), DI->getExpression());
    1241             :     } else {
    1242             :       // We can't yet handle anything else here because it would require
    1243             :       // generating code, thus altering codegen because of debug info.
    1244             :       DEBUG(dbgs() << "Dropping debug info for " << *DI << "\n");
    1245             :     }
    1246             :     return true;
    1247             :   }
    1248           3 :   case Intrinsic::objectsize: {
    1249           9 :     ConstantInt *CI = cast<ConstantInt>(II->getArgOperand(1));
    1250           3 :     unsigned long long Res = CI->isZero() ? -1ULL : 0;
    1251           3 :     Constant *ResCI = ConstantInt::get(II->getType(), Res);
    1252           3 :     unsigned ResultReg = getRegForValue(ResCI);
    1253           3 :     if (!ResultReg)
    1254             :       return false;
    1255           3 :     updateValueMap(II, ResultReg);
    1256           3 :     return true;
    1257             :   }
    1258           7 :   case Intrinsic::invariant_group_barrier:
    1259             :   case Intrinsic::expect: {
    1260          14 :     unsigned ResultReg = getRegForValue(II->getArgOperand(0));
    1261           7 :     if (!ResultReg)
    1262             :       return false;
    1263           7 :     updateValueMap(II, ResultReg);
    1264           7 :     return true;
    1265             :   }
    1266          26 :   case Intrinsic::experimental_stackmap:
    1267          26 :     return selectStackmap(II);
    1268          39 :   case Intrinsic::experimental_patchpoint_void:
    1269             :   case Intrinsic::experimental_patchpoint_i64:
    1270          39 :     return selectPatchpoint(II);
    1271             : 
    1272           0 :   case Intrinsic::xray_customevent:
    1273           0 :     return selectXRayCustomEvent(II);
    1274             :   }
    1275             : 
    1276        1346 :   return fastLowerIntrinsicCall(II);
    1277             : }
    1278             : 
    1279         856 : bool FastISel::selectCast(const User *I, unsigned Opcode) {
    1280        1712 :   EVT SrcVT = TLI.getValueType(DL, I->getOperand(0)->getType());
    1281         856 :   EVT DstVT = TLI.getValueType(DL, I->getType());
    1282             : 
    1283        3420 :   if (SrcVT == MVT::Other || !SrcVT.isSimple() || DstVT == MVT::Other ||
    1284         854 :       !DstVT.isSimple())
    1285             :     // Unhandled type. Halt "fast" selection and bail.
    1286             :     return false;
    1287             : 
    1288             :   // Check if the destination type is legal.
    1289        1544 :   if (!TLI.isTypeLegal(DstVT))
    1290             :     return false;
    1291             : 
    1292             :   // Check if the source operand is legal.
    1293         945 :   if (!TLI.isTypeLegal(SrcVT))
    1294             :     return false;
    1295             : 
    1296         255 :   unsigned InputReg = getRegForValue(I->getOperand(0));
    1297         255 :   if (!InputReg)
    1298             :     // Unhandled operand.  Halt "fast" selection and bail.
    1299             :     return false;
    1300             : 
    1301         252 :   bool InputRegIsKill = hasTrivialKill(I->getOperand(0));
    1302             : 
    1303         252 :   unsigned ResultReg = fastEmit_r(SrcVT.getSimpleVT(), DstVT.getSimpleVT(),
    1304         504 :                                   Opcode, InputReg, InputRegIsKill);
    1305         252 :   if (!ResultReg)
    1306             :     return false;
    1307             : 
    1308         161 :   updateValueMap(I, ResultReg);
    1309         161 :   return true;
    1310             : }
    1311             : 
    1312        2007 : bool FastISel::selectBitCast(const User *I) {
    1313             :   // If the bitcast doesn't change the type, just use the operand value.
    1314        4014 :   if (I->getType() == I->getOperand(0)->getType()) {
    1315           3 :     unsigned Reg = getRegForValue(I->getOperand(0));
    1316           3 :     if (!Reg)
    1317             :       return false;
    1318           3 :     updateValueMap(I, Reg);
    1319           3 :     return true;
    1320             :   }
    1321             : 
    1322             :   // Bitcasts of other values become reg-reg copies or BITCAST operators.
    1323        4008 :   EVT SrcEVT = TLI.getValueType(DL, I->getOperand(0)->getType());
    1324        2004 :   EVT DstEVT = TLI.getValueType(DL, I->getType());
    1325        4008 :   if (SrcEVT == MVT::Other || DstEVT == MVT::Other ||
    1326        5946 :       !TLI.isTypeLegal(SrcEVT) || !TLI.isTypeLegal(DstEVT))
    1327             :     // Unhandled type. Halt "fast" selection and bail.
    1328             :     return false;
    1329             : 
    1330        1971 :   MVT SrcVT = SrcEVT.getSimpleVT();
    1331        1971 :   MVT DstVT = DstEVT.getSimpleVT();
    1332        1971 :   unsigned Op0 = getRegForValue(I->getOperand(0));
    1333        1971 :   if (!Op0) // Unhandled operand. Halt "fast" selection and bail.
    1334             :     return false;
    1335        1963 :   bool Op0IsKill = hasTrivialKill(I->getOperand(0));
    1336             : 
    1337             :   // First, try to perform the bitcast by inserting a reg-reg copy.
    1338        1963 :   unsigned ResultReg = 0;
    1339        1963 :   if (SrcVT == DstVT) {
    1340         266 :     const TargetRegisterClass *SrcClass = TLI.getRegClassFor(SrcVT);
    1341         266 :     const TargetRegisterClass *DstClass = TLI.getRegClassFor(DstVT);
    1342             :     // Don't attempt a cross-class copy. It will likely fail.
    1343         266 :     if (SrcClass == DstClass) {
    1344         266 :       ResultReg = createResultReg(DstClass);
    1345         532 :       BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc,
    1346         798 :               TII.get(TargetOpcode::COPY), ResultReg).addReg(Op0);
    1347             :     }
    1348             :   }
    1349             : 
    1350             :   // If the reg-reg copy failed, select a BITCAST opcode.
    1351         266 :   if (!ResultReg)
    1352        1697 :     ResultReg = fastEmit_r(SrcVT, DstVT, ISD::BITCAST, Op0, Op0IsKill);
    1353             : 
    1354        1963 :   if (!ResultReg)
    1355             :     return false;
    1356             : 
    1357         279 :   updateValueMap(I, ResultReg);
    1358         279 :   return true;
    1359             : }
    1360             : 
    1361             : // Remove local value instructions starting from the instruction after
    1362             : // SavedLastLocalValue to the current function insert point.
    1363         712 : void FastISel::removeDeadLocalValueCode(MachineInstr *SavedLastLocalValue)
    1364             : {
    1365         712 :   MachineInstr *CurLastLocalValue = getLastLocalValue();
    1366         712 :   if (CurLastLocalValue != SavedLastLocalValue) {
    1367             :     // Find the first local value instruction to be deleted. 
    1368             :     // This is the instruction after SavedLastLocalValue if it is non-NULL.
    1369             :     // Otherwise it's the first instruction in the block.
    1370          27 :     MachineBasicBlock::iterator FirstDeadInst(SavedLastLocalValue);
    1371          27 :     if (SavedLastLocalValue)
    1372             :       ++FirstDeadInst;
    1373             :     else
    1374          24 :       FirstDeadInst = FuncInfo.MBB->getFirstNonPHI();
    1375          54 :     setLastLocalValue(SavedLastLocalValue);
    1376          27 :     removeDeadCode(FirstDeadInst, FuncInfo.InsertPt);
    1377             :   }
    1378         712 : }
    1379             : 
    1380       25457 : bool FastISel::selectInstruction(const Instruction *I) {
    1381       25457 :   MachineInstr *SavedLastLocalValue = getLastLocalValue();
    1382             :   // Just before the terminator instruction, insert instructions to
    1383             :   // feed PHI nodes in successor blocks.
    1384       50914 :   if (isa<TerminatorInst>(I)) {
    1385        9706 :     if (!handlePHINodesInSuccessorBlocks(I->getParent())) {
    1386             :       // PHI node handling may have generated local value instructions,
    1387             :       // even though it failed to handle all PHI nodes.
    1388             :       // We remove these instructions because SelectionDAGISel will generate 
    1389             :       // them again.
    1390          58 :       removeDeadLocalValueCode(SavedLastLocalValue);
    1391          58 :       return false;
    1392             :     }
    1393             :   }
    1394             : 
    1395             :   // FastISel does not handle any operand bundles except OB_funclet.
    1396       76197 :   if (ImmutableCallSite CS = ImmutableCallSite(I))
    1397        3748 :     for (unsigned i = 0, e = CS.getNumOperandBundles(); i != e; ++i)
    1398           0 :       if (CS.getOperandBundleAt(i).getTagID() != LLVMContext::OB_funclet)
    1399           0 :         return false;
    1400             : 
    1401       50798 :   DbgLoc = I->getDebugLoc();
    1402             : 
    1403       25399 :   SavedInsertPt = FuncInfo.InsertPt;
    1404             : 
    1405       29034 :   if (const auto *Call = dyn_cast<CallInst>(I)) {
    1406        3416 :     const Function *F = Call->getCalledFunction();
    1407             :     LibFunc Func;
    1408             : 
    1409             :     // As a special case, don't handle calls to builtin library functions that
    1410             :     // may be translated directly to target instructions.
    1411       13508 :     if (F && !F->hasLocalLinkage() && F->hasName() &&
    1412        6812 :         LibInfo->getLibFunc(F->getName(), Func) &&
    1413          84 :         LibInfo->hasOptimizedCodeGen(Func))
    1414           7 :       return false;
    1415             : 
    1416             :     // Don't handle Intrinsic::trap if a trap function is specified.
    1417        3633 :     if (F && F->getIntrinsicID() == Intrinsic::trap &&
    1418          10 :         Call->hasFnAttr("trap-func-name"))
    1419             :       return false;
    1420             :   }
    1421             : 
    1422             :   // First, try doing target-independent selection.
    1423       25392 :   if (!SkipTargetIndependentISel) {
    1424       42770 :     if (selectOperator(I, I->getOpcode())) {
    1425        3796 :       ++NumFastIselSuccessIndependent;
    1426       15184 :       DbgLoc = DebugLoc();
    1427        3796 :       return true;
    1428             :     }
    1429             :     // Remove dead code.
    1430       17589 :     recomputeInsertPt();
    1431       35178 :     if (SavedInsertPt != FuncInfo.InsertPt)
    1432          50 :       removeDeadCode(FuncInfo.InsertPt, SavedInsertPt);
    1433       17589 :     SavedInsertPt = FuncInfo.InsertPt;
    1434             :   }
    1435             :   // Next, try calling the target to attempt to handle the instruction.
    1436       21596 :   if (fastSelectInstruction(I)) {
    1437       17255 :     ++NumFastIselSuccessTarget;
    1438       69020 :     DbgLoc = DebugLoc();
    1439       17255 :     return true;
    1440             :   }
    1441             :   // Remove dead code.
    1442        4341 :   recomputeInsertPt();
    1443        8682 :   if (SavedInsertPt != FuncInfo.InsertPt)
    1444          15 :     removeDeadCode(FuncInfo.InsertPt, SavedInsertPt);
    1445             : 
    1446       17364 :   DbgLoc = DebugLoc();
    1447             :   // Undo phi node updates, because they will be added again by SelectionDAG.
    1448        8682 :   if (isa<TerminatorInst>(I)) {
    1449             :     // PHI node handling may have generated local value instructions. 
    1450             :     // We remove them because SelectionDAGISel will generate them again.
    1451         654 :     removeDeadLocalValueCode(SavedLastLocalValue);
    1452         654 :     FuncInfo.PHINodesToUpdate.resize(FuncInfo.OrigNumPHINodesToUpdate);
    1453             :   }
    1454             :   return false;
    1455             : }
    1456             : 
    1457             : /// Emit an unconditional branch to the given block, unless it is the immediate
    1458             : /// (fall-through) successor, and update the CFG.
    1459        1428 : void FastISel::fastEmitBranch(MachineBasicBlock *MSucc,
    1460             :                               const DebugLoc &DbgLoc) {
    1461        4041 :   if (FuncInfo.MBB->getBasicBlock()->size() > 1 &&
    1462        1185 :       FuncInfo.MBB->isLayoutSuccessor(MSucc)) {
    1463             :     // For more accurate line information if this is the only instruction
    1464             :     // in the block then emit it, otherwise we have the unconditional
    1465             :     // fall-through case, which needs no instructions.
    1466             :   } else {
    1467             :     // The unconditional branch case.
    1468        1046 :     TII.insertBranch(*FuncInfo.MBB, MSucc, nullptr,
    1469        1569 :                      SmallVector<MachineOperand, 0>(), DbgLoc);
    1470             :   }
    1471        1428 :   if (FuncInfo.BPI) {
    1472             :     auto BranchProbability = FuncInfo.BPI->getEdgeProbability(
    1473         240 :         FuncInfo.MBB->getBasicBlock(), MSucc->getBasicBlock());
    1474         240 :     FuncInfo.MBB->addSuccessor(MSucc, BranchProbability);
    1475             :   } else
    1476        1188 :     FuncInfo.MBB->addSuccessorWithoutProb(MSucc);
    1477        1428 : }
    1478             : 
    1479         571 : void FastISel::finishCondBranch(const BasicBlock *BranchBB,
    1480             :                                 MachineBasicBlock *TrueMBB,
    1481             :                                 MachineBasicBlock *FalseMBB) {
    1482             :   // Add TrueMBB as successor unless it is equal to the FalseMBB: This can
    1483             :   // happen in degenerate IR and MachineIR forbids to have a block twice in the
    1484             :   // successor/predecessor lists.
    1485         571 :   if (TrueMBB != FalseMBB) {
    1486         568 :     if (FuncInfo.BPI) {
    1487             :       auto BranchProbability =
    1488         160 :           FuncInfo.BPI->getEdgeProbability(BranchBB, TrueMBB->getBasicBlock());
    1489         160 :       FuncInfo.MBB->addSuccessor(TrueMBB, BranchProbability);
    1490             :     } else
    1491         408 :       FuncInfo.MBB->addSuccessorWithoutProb(TrueMBB);
    1492             :   }
    1493             : 
    1494         571 :   fastEmitBranch(FalseMBB, DbgLoc);
    1495         571 : }
    1496             : 
    1497             : /// Emit an FNeg operation.
    1498           3 : bool FastISel::selectFNeg(const User *I) {
    1499           3 :   unsigned OpReg = getRegForValue(BinaryOperator::getFNegArgument(I));
    1500           3 :   if (!OpReg)
    1501             :     return false;
    1502           3 :   bool OpRegIsKill = hasTrivialKill(I);
    1503             : 
    1504             :   // If the target has ISD::FNEG, use it.
    1505           3 :   EVT VT = TLI.getValueType(DL, I->getType());
    1506           3 :   unsigned ResultReg = fastEmit_r(VT.getSimpleVT(), VT.getSimpleVT(), ISD::FNEG,
    1507           6 :                                   OpReg, OpRegIsKill);
    1508           3 :   if (ResultReg) {
    1509           0 :     updateValueMap(I, ResultReg);
    1510           0 :     return true;
    1511             :   }
    1512             : 
    1513             :   // Bitcast the value to integer, twiddle the sign bit with xor,
    1514             :   // and then bitcast it back to floating-point.
    1515           3 :   if (VT.getSizeInBits() > 64)
    1516             :     return false;
    1517           3 :   EVT IntVT = EVT::getIntegerVT(I->getContext(), VT.getSizeInBits());
    1518           6 :   if (!TLI.isTypeLegal(IntVT))
    1519             :     return false;
    1520             : 
    1521           3 :   unsigned IntReg = fastEmit_r(VT.getSimpleVT(), IntVT.getSimpleVT(),
    1522           6 :                                ISD::BITCAST, OpReg, OpRegIsKill);
    1523           3 :   if (!IntReg)
    1524             :     return false;
    1525             : 
    1526           6 :   unsigned IntResultReg = fastEmit_ri_(
    1527             :       IntVT.getSimpleVT(), ISD::XOR, IntReg, /*IsKill=*/true,
    1528           6 :       UINT64_C(1) << (VT.getSizeInBits() - 1), IntVT.getSimpleVT());
    1529           3 :   if (!IntResultReg)
    1530             :     return false;
    1531             : 
    1532           3 :   ResultReg = fastEmit_r(IntVT.getSimpleVT(), VT.getSimpleVT(), ISD::BITCAST,
    1533           3 :                          IntResultReg, /*IsKill=*/true);
    1534           3 :   if (!ResultReg)
    1535             :     return false;
    1536             : 
    1537           3 :   updateValueMap(I, ResultReg);
    1538           3 :   return true;
    1539             : }
    1540             : 
    1541         432 : bool FastISel::selectExtractValue(const User *U) {
    1542         432 :   const ExtractValueInst *EVI = dyn_cast<ExtractValueInst>(U);
    1543             :   if (!EVI)
    1544             :     return false;
    1545             : 
    1546             :   // Make sure we only try to handle extracts with a legal result.  But also
    1547             :   // allow i1 because it's easy.
    1548         432 :   EVT RealVT = TLI.getValueType(DL, EVI->getType(), /*AllowUnknown=*/true);
    1549         432 :   if (!RealVT.isSimple())
    1550             :     return false;
    1551         432 :   MVT VT = RealVT.getSimpleVT();
    1552        1050 :   if (!TLI.isTypeLegal(VT) && VT != MVT::i1)
    1553             :     return false;
    1554             : 
    1555         856 :   const Value *Op0 = EVI->getOperand(0);
    1556         428 :   Type *AggTy = Op0->getType();
    1557             : 
    1558             :   // Get the base result register.
    1559             :   unsigned ResultReg;
    1560         428 :   DenseMap<const Value *, unsigned>::iterator I = FuncInfo.ValueMap.find(Op0);
    1561        1284 :   if (I != FuncInfo.ValueMap.end())
    1562         206 :     ResultReg = I->second;
    1563         444 :   else if (isa<Instruction>(Op0))
    1564         218 :     ResultReg = FuncInfo.InitializeRegForValue(Op0);
    1565             :   else
    1566             :     return false; // fast-isel can't handle aggregate constants at the moment
    1567             : 
    1568             :   // Get the actual result register, which is an offset from the base register.
    1569         848 :   unsigned VTIndex = ComputeLinearIndex(AggTy, EVI->getIndices());
    1570             : 
    1571         424 :   SmallVector<EVT, 4> AggValueVTs;
    1572         424 :   ComputeValueVTs(TLI, DL, AggTy, AggValueVTs);
    1573             : 
    1574         751 :   for (unsigned i = 0; i < VTIndex; i++)
    1575         654 :     ResultReg += TLI.getNumRegisters(FuncInfo.Fn->getContext(), AggValueVTs[i]);
    1576             : 
    1577         424 :   updateValueMap(EVI, ResultReg);
    1578         424 :   return true;
    1579             : }
    1580             : 
    1581       22276 : bool FastISel::selectOperator(const User *I, unsigned Opcode) {
    1582       22276 :   switch (Opcode) {
    1583         431 :   case Instruction::Add:
    1584         431 :     return selectBinaryOp(I, ISD::ADD);
    1585         236 :   case Instruction::FAdd:
    1586         236 :     return selectBinaryOp(I, ISD::FADD);
    1587          84 :   case Instruction::Sub:
    1588          84 :     return selectBinaryOp(I, ISD::SUB);
    1589          13 :   case Instruction::FSub:
    1590             :     // FNeg is currently represented in LLVM IR as a special case of FSub.
    1591          13 :     if (BinaryOperator::isFNeg(I))
    1592           3 :       return selectFNeg(I);
    1593          10 :     return selectBinaryOp(I, ISD::FSUB);
    1594          61 :   case Instruction::Mul:
    1595          61 :     return selectBinaryOp(I, ISD::MUL);
    1596          31 :   case Instruction::FMul:
    1597          31 :     return selectBinaryOp(I, ISD::FMUL);
    1598          18 :   case Instruction::SDiv:
    1599          18 :     return selectBinaryOp(I, ISD::SDIV);
    1600          23 :   case Instruction::UDiv:
    1601          23 :     return selectBinaryOp(I, ISD::UDIV);
    1602          26 :   case Instruction::FDiv:
    1603          26 :     return selectBinaryOp(I, ISD::FDIV);
    1604          41 :   case Instruction::SRem:
    1605          41 :     return selectBinaryOp(I, ISD::SREM);
    1606          18 :   case Instruction::URem:
    1607          18 :     return selectBinaryOp(I, ISD::UREM);
    1608           1 :   case Instruction::FRem:
    1609           1 :     return selectBinaryOp(I, ISD::FREM);
    1610          23 :   case Instruction::Shl:
    1611          23 :     return selectBinaryOp(I, ISD::SHL);
    1612          29 :   case Instruction::LShr:
    1613          29 :     return selectBinaryOp(I, ISD::SRL);
    1614          16 :   case Instruction::AShr:
    1615          16 :     return selectBinaryOp(I, ISD::SRA);
    1616         150 :   case Instruction::And:
    1617         150 :     return selectBinaryOp(I, ISD::AND);
    1618          95 :   case Instruction::Or:
    1619          95 :     return selectBinaryOp(I, ISD::OR);
    1620         115 :   case Instruction::Xor:
    1621         115 :     return selectBinaryOp(I, ISD::XOR);
    1622             : 
    1623         251 :   case Instruction::GetElementPtr:
    1624         251 :     return selectGetElementPtr(I);
    1625             : 
    1626        1173 :   case Instruction::Br: {
    1627        1173 :     const BranchInst *BI = cast<BranchInst>(I);
    1628             : 
    1629        1173 :     if (BI->isUnconditional()) {
    1630         657 :       const BasicBlock *LLVMSucc = BI->getSuccessor(0);
    1631        1314 :       MachineBasicBlock *MSucc = FuncInfo.MBBMap[LLVMSucc];
    1632        1314 :       fastEmitBranch(MSucc, BI->getDebugLoc());
    1633             :       return true;
    1634             :     }
    1635             : 
    1636             :     // Conditional branches are not handed yet.
    1637             :     // Halt "fast" selection and bail.
    1638             :     return false;
    1639             :   }
    1640             : 
    1641          91 :   case Instruction::Unreachable:
    1642          91 :     if (TM.Options.TrapUnreachable)
    1643          18 :       return fastEmit_(MVT::Other, MVT::Other, ISD::TRAP) != 0;
    1644             :     else
    1645             :       return true;
    1646             : 
    1647          12 :   case Instruction::Alloca:
    1648             :     // FunctionLowering has the static-sized case covered.
    1649          24 :     if (FuncInfo.StaticAllocaMap.count(cast<AllocaInst>(I)))
    1650             :       return true;
    1651             : 
    1652             :     // Dynamic-sized alloca is not handled yet.
    1653             :     return false;
    1654             : 
    1655        3628 :   case Instruction::Call:
    1656        3628 :     return selectCall(I);
    1657             : 
    1658        1957 :   case Instruction::BitCast:
    1659        1957 :     return selectBitCast(I);
    1660             : 
    1661          32 :   case Instruction::FPToSI:
    1662          32 :     return selectCast(I, ISD::FP_TO_SINT);
    1663         311 :   case Instruction::ZExt:
    1664         311 :     return selectCast(I, ISD::ZERO_EXTEND);
    1665         245 :   case Instruction::SExt:
    1666         245 :     return selectCast(I, ISD::SIGN_EXTEND);
    1667         124 :   case Instruction::Trunc:
    1668         124 :     return selectCast(I, ISD::TRUNCATE);
    1669          70 :   case Instruction::SIToFP:
    1670          70 :     return selectCast(I, ISD::SINT_TO_FP);
    1671             : 
    1672          92 :   case Instruction::IntToPtr: // Deliberate fall-through.
    1673             :   case Instruction::PtrToInt: {
    1674         184 :     EVT SrcVT = TLI.getValueType(DL, I->getOperand(0)->getType());
    1675          92 :     EVT DstVT = TLI.getValueType(DL, I->getType());
    1676          92 :     if (DstVT.bitsGT(SrcVT))
    1677           3 :       return selectCast(I, ISD::ZERO_EXTEND);
    1678          89 :     if (DstVT.bitsLT(SrcVT))
    1679           8 :       return selectCast(I, ISD::TRUNCATE);
    1680          81 :     unsigned Reg = getRegForValue(I->getOperand(0));
    1681          81 :     if (!Reg)
    1682             :       return false;
    1683          76 :     updateValueMap(I, Reg);
    1684          76 :     return true;
    1685             :   }
    1686             : 
    1687         432 :   case Instruction::ExtractValue:
    1688         432 :     return selectExtractValue(I);
    1689             : 
    1690           0 :   case Instruction::PHI:
    1691           0 :     llvm_unreachable("FastISel shouldn't visit PHI nodes!");
    1692             : 
    1693             :   default:
    1694             :     // Unhandled instruction. Halt "fast" selection and bail.
    1695             :     return false;
    1696             :   }
    1697             : }
    1698             : 
    1699        7750 : FastISel::FastISel(FunctionLoweringInfo &FuncInfo,
    1700             :                    const TargetLibraryInfo *LibInfo,
    1701        7750 :                    bool SkipTargetIndependentISel)
    1702       15500 :     : FuncInfo(FuncInfo), MF(FuncInfo.MF), MRI(FuncInfo.MF->getRegInfo()),
    1703       15500 :       MFI(FuncInfo.MF->getFrameInfo()), MCP(*FuncInfo.MF->getConstantPool()),
    1704       15500 :       TM(FuncInfo.MF->getTarget()), DL(MF->getDataLayout()),
    1705        7750 :       TII(*MF->getSubtarget().getInstrInfo()),
    1706        7750 :       TLI(*MF->getSubtarget().getTargetLowering()),
    1707        7750 :       TRI(*MF->getSubtarget().getRegisterInfo()), LibInfo(LibInfo),
    1708      100750 :       SkipTargetIndependentISel(SkipTargetIndependentISel) {}
    1709             : 
    1710             : FastISel::~FastISel() = default;
    1711             : 
    1712           0 : bool FastISel::fastLowerArguments() { return false; }
    1713             : 
    1714         594 : bool FastISel::fastLowerCall(CallLoweringInfo & /*CLI*/) { return false; }
    1715             : 
    1716         127 : bool FastISel::fastLowerIntrinsicCall(const IntrinsicInst * /*II*/) {
    1717         127 :   return false;
    1718             : }
    1719             : 
    1720           6 : unsigned FastISel::fastEmit_(MVT, MVT, unsigned) { return 0; }
    1721             : 
    1722           0 : unsigned FastISel::fastEmit_r(MVT, MVT, unsigned, unsigned /*Op0*/,
    1723             :                               bool /*Op0IsKill*/) {
    1724           0 :   return 0;
    1725             : }
    1726             : 
    1727           0 : unsigned FastISel::fastEmit_rr(MVT, MVT, unsigned, unsigned /*Op0*/,
    1728             :                                bool /*Op0IsKill*/, unsigned /*Op1*/,
    1729             :                                bool /*Op1IsKill*/) {
    1730           0 :   return 0;
    1731             : }
    1732             : 
    1733           0 : unsigned FastISel::fastEmit_i(MVT, MVT, unsigned, uint64_t /*Imm*/) {
    1734           0 :   return 0;
    1735             : }
    1736             : 
    1737           2 : unsigned FastISel::fastEmit_f(MVT, MVT, unsigned,
    1738             :                               const ConstantFP * /*FPImm*/) {
    1739           2 :   return 0;
    1740             : }
    1741             : 
    1742           5 : unsigned FastISel::fastEmit_ri(MVT, MVT, unsigned, unsigned /*Op0*/,
    1743             :                                bool /*Op0IsKill*/, uint64_t /*Imm*/) {
    1744           5 :   return 0;
    1745             : }
    1746             : 
    1747             : /// This method is a wrapper of fastEmit_ri. It first tries to emit an
    1748             : /// instruction with an immediate operand using fastEmit_ri.
    1749             : /// If that fails, it materializes the immediate into a register and try
    1750             : /// fastEmit_rr instead.
    1751         889 : unsigned FastISel::fastEmit_ri_(MVT VT, unsigned Opcode, unsigned Op0,
    1752             :                                 bool Op0IsKill, uint64_t Imm, MVT ImmType) {
    1753             :   // If this is a multiply by a power of two, emit this as a shift left.
    1754         919 :   if (Opcode == ISD::MUL && isPowerOf2_64(Imm)) {
    1755          30 :     Opcode = ISD::SHL;
    1756          30 :     Imm = Log2_64(Imm);
    1757         862 :   } else if (Opcode == ISD::UDIV && isPowerOf2_64(Imm)) {
    1758             :     // div x, 8 -> srl x, 3
    1759           3 :     Opcode = ISD::SRL;
    1760           3 :     Imm = Log2_64(Imm);
    1761             :   }
    1762             : 
    1763             :   // Horrible hack (to be removed), check to make sure shift amounts are
    1764             :   // in-range.
    1765         962 :   if ((Opcode == ISD::SHL || Opcode == ISD::SRA || Opcode == ISD::SRL) &&
    1766          73 :       Imm >= VT.getSizeInBits())
    1767             :     return 0;
    1768             : 
    1769             :   // First check if immediate type is legal. If not, we can't use the ri form.
    1770         886 :   unsigned ResultReg = fastEmit_ri(VT, VT, Opcode, Op0, Op0IsKill, Imm);
    1771         886 :   if (ResultReg)
    1772             :     return ResultReg;
    1773          63 :   unsigned MaterialReg = fastEmit_i(ImmType, ImmType, ISD::Constant, Imm);
    1774          63 :   bool IsImmKill = true;
    1775          63 :   if (!MaterialReg) {
    1776             :     // This is a bit ugly/slow, but failing here means falling out of
    1777             :     // fast-isel, which would be very slow.
    1778             :     IntegerType *ITy =
    1779          41 :         IntegerType::get(FuncInfo.Fn->getContext(), VT.getSizeInBits());
    1780          41 :     MaterialReg = getRegForValue(ConstantInt::get(ITy, Imm));
    1781          41 :     if (!MaterialReg)
    1782             :       return 0;
    1783             :     // FIXME: If the materialized register here has no uses yet then this
    1784             :     // will be the first use and we should be able to mark it as killed.
    1785             :     // However, the local value area for materialising constant expressions
    1786             :     // grows down, not up, which means that any constant expressions we generate
    1787             :     // later which also use 'Imm' could be after this instruction and therefore
    1788             :     // after this kill.
    1789             :     IsImmKill = false;
    1790             :   }
    1791          63 :   return fastEmit_rr(VT, VT, Opcode, Op0, Op0IsKill, MaterialReg, IsImmKill);
    1792             : }
    1793             : 
    1794       17557 : unsigned FastISel::createResultReg(const TargetRegisterClass *RC) {
    1795       17557 :   return MRI.createVirtualRegister(RC);
    1796             : }
    1797             : 
    1798       13217 : unsigned FastISel::constrainOperandRegClass(const MCInstrDesc &II, unsigned Op,
    1799             :                                             unsigned OpNum) {
    1800       13217 :   if (TargetRegisterInfo::isVirtualRegister(Op)) {
    1801             :     const TargetRegisterClass *RegClass =
    1802        9330 :         TII.getRegClass(II, OpNum, &TRI, *FuncInfo.MF);
    1803        9330 :     if (!MRI.constrainRegClass(Op, RegClass)) {
    1804             :       // If it's not legal to COPY between the register classes, something
    1805             :       // has gone very wrong before we got here.
    1806           0 :       unsigned NewOp = createResultReg(RegClass);
    1807           0 :       BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc,
    1808           0 :               TII.get(TargetOpcode::COPY), NewOp).addReg(Op);
    1809           0 :       return NewOp;
    1810             :     }
    1811             :   }
    1812             :   return Op;
    1813             : }
    1814             : 
    1815         279 : unsigned FastISel::fastEmitInst_(unsigned MachineInstOpcode,
    1816             :                                  const TargetRegisterClass *RC) {
    1817         279 :   unsigned ResultReg = createResultReg(RC);
    1818         558 :   const MCInstrDesc &II = TII.get(MachineInstOpcode);
    1819             : 
    1820         279 :   BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc, II, ResultReg);
    1821         279 :   return ResultReg;
    1822             : }
    1823             : 
    1824         374 : unsigned FastISel::fastEmitInst_r(unsigned MachineInstOpcode,
    1825             :                                   const TargetRegisterClass *RC, unsigned Op0,
    1826             :                                   bool Op0IsKill) {
    1827         748 :   const MCInstrDesc &II = TII.get(MachineInstOpcode);
    1828             : 
    1829         374 :   unsigned ResultReg = createResultReg(RC);
    1830         374 :   Op0 = constrainOperandRegClass(II, Op0, II.getNumDefs());
    1831             : 
    1832         374 :   if (II.getNumDefs() >= 1)
    1833         726 :     BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc, II, ResultReg)
    1834         363 :         .addReg(Op0, getKillRegState(Op0IsKill));
    1835             :   else {
    1836          22 :     BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc, II)
    1837          11 :         .addReg(Op0, getKillRegState(Op0IsKill));
    1838          22 :     BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc,
    1839          33 :             TII.get(TargetOpcode::COPY), ResultReg).addReg(II.ImplicitDefs[0]);
    1840             :   }
    1841             : 
    1842         374 :   return ResultReg;
    1843             : }
    1844             : 
    1845         942 : unsigned FastISel::fastEmitInst_rr(unsigned MachineInstOpcode,
    1846             :                                    const TargetRegisterClass *RC, unsigned Op0,
    1847             :                                    bool Op0IsKill, unsigned Op1,
    1848             :                                    bool Op1IsKill) {
    1849        1884 :   const MCInstrDesc &II = TII.get(MachineInstOpcode);
    1850             : 
    1851         942 :   unsigned ResultReg = createResultReg(RC);
    1852         942 :   Op0 = constrainOperandRegClass(II, Op0, II.getNumDefs());
    1853         942 :   Op1 = constrainOperandRegClass(II, Op1, II.getNumDefs() + 1);
    1854             : 
    1855         942 :   if (II.getNumDefs() >= 1)
    1856        1884 :     BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc, II, ResultReg)
    1857         942 :         .addReg(Op0, getKillRegState(Op0IsKill))
    1858         942 :         .addReg(Op1, getKillRegState(Op1IsKill));
    1859             :   else {
    1860           0 :     BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc, II)
    1861           0 :         .addReg(Op0, getKillRegState(Op0IsKill))
    1862           0 :         .addReg(Op1, getKillRegState(Op1IsKill));
    1863           0 :     BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc,
    1864           0 :             TII.get(TargetOpcode::COPY), ResultReg).addReg(II.ImplicitDefs[0]);
    1865             :   }
    1866         942 :   return ResultReg;
    1867             : }
    1868             : 
    1869          57 : unsigned FastISel::fastEmitInst_rrr(unsigned MachineInstOpcode,
    1870             :                                     const TargetRegisterClass *RC, unsigned Op0,
    1871             :                                     bool Op0IsKill, unsigned Op1,
    1872             :                                     bool Op1IsKill, unsigned Op2,
    1873             :                                     bool Op2IsKill) {
    1874         114 :   const MCInstrDesc &II = TII.get(MachineInstOpcode);
    1875             : 
    1876          57 :   unsigned ResultReg = createResultReg(RC);
    1877          57 :   Op0 = constrainOperandRegClass(II, Op0, II.getNumDefs());
    1878          57 :   Op1 = constrainOperandRegClass(II, Op1, II.getNumDefs() + 1);
    1879          57 :   Op2 = constrainOperandRegClass(II, Op2, II.getNumDefs() + 2);
    1880             : 
    1881          57 :   if (II.getNumDefs() >= 1)
    1882         114 :     BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc, II, ResultReg)
    1883          57 :         .addReg(Op0, getKillRegState(Op0IsKill))
    1884          57 :         .addReg(Op1, getKillRegState(Op1IsKill))
    1885          57 :         .addReg(Op2, getKillRegState(Op2IsKill));
    1886             :   else {
    1887           0 :     BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc, II)
    1888           0 :         .addReg(Op0, getKillRegState(Op0IsKill))
    1889           0 :         .addReg(Op1, getKillRegState(Op1IsKill))
    1890           0 :         .addReg(Op2, getKillRegState(Op2IsKill));
    1891           0 :     BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc,
    1892           0 :             TII.get(TargetOpcode::COPY), ResultReg).addReg(II.ImplicitDefs[0]);
    1893             :   }
    1894          57 :   return ResultReg;
    1895             : }
    1896             : 
    1897         751 : unsigned FastISel::fastEmitInst_ri(unsigned MachineInstOpcode,
    1898             :                                    const TargetRegisterClass *RC, unsigned Op0,
    1899             :                                    bool Op0IsKill, uint64_t Imm) {
    1900        1502 :   const MCInstrDesc &II = TII.get(MachineInstOpcode);
    1901             : 
    1902         751 :   unsigned ResultReg = createResultReg(RC);
    1903         751 :   Op0 = constrainOperandRegClass(II, Op0, II.getNumDefs());
    1904             : 
    1905         751 :   if (II.getNumDefs() >= 1)
    1906        1502 :     BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc, II, ResultReg)
    1907         751 :         .addReg(Op0, getKillRegState(Op0IsKill))
    1908        1502 :         .addImm(Imm);
    1909             :   else {
    1910           0 :     BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc, II)
    1911           0 :         .addReg(Op0, getKillRegState(Op0IsKill))
    1912           0 :         .addImm(Imm);
    1913           0 :     BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc,
    1914           0 :             TII.get(TargetOpcode::COPY), ResultReg).addReg(II.ImplicitDefs[0]);
    1915             :   }
    1916         751 :   return ResultReg;
    1917             : }
    1918             : 
    1919         371 : unsigned FastISel::fastEmitInst_rii(unsigned MachineInstOpcode,
    1920             :                                     const TargetRegisterClass *RC, unsigned Op0,
    1921             :                                     bool Op0IsKill, uint64_t Imm1,
    1922             :                                     uint64_t Imm2) {
    1923         742 :   const MCInstrDesc &II = TII.get(MachineInstOpcode);
    1924             : 
    1925         371 :   unsigned ResultReg = createResultReg(RC);
    1926         371 :   Op0 = constrainOperandRegClass(II, Op0, II.getNumDefs());
    1927             : 
    1928         371 :   if (II.getNumDefs() >= 1)
    1929         742 :     BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc, II, ResultReg)
    1930         371 :         .addReg(Op0, getKillRegState(Op0IsKill))
    1931         742 :         .addImm(Imm1)
    1932         742 :         .addImm(Imm2);
    1933             :   else {
    1934           0 :     BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc, II)
    1935           0 :         .addReg(Op0, getKillRegState(Op0IsKill))
    1936           0 :         .addImm(Imm1)
    1937           0 :         .addImm(Imm2);
    1938           0 :     BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc,
    1939           0 :             TII.get(TargetOpcode::COPY), ResultReg).addReg(II.ImplicitDefs[0]);
    1940             :   }
    1941         371 :   return ResultReg;
    1942             : }
    1943             : 
    1944           0 : unsigned FastISel::fastEmitInst_f(unsigned MachineInstOpcode,
    1945             :                                   const TargetRegisterClass *RC,
    1946             :                                   const ConstantFP *FPImm) {
    1947           0 :   const MCInstrDesc &II = TII.get(MachineInstOpcode);
    1948             : 
    1949           0 :   unsigned ResultReg = createResultReg(RC);
    1950             : 
    1951           0 :   if (II.getNumDefs() >= 1)
    1952           0 :     BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc, II, ResultReg)
    1953           0 :         .addFPImm(FPImm);
    1954             :   else {
    1955           0 :     BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc, II)
    1956           0 :         .addFPImm(FPImm);
    1957           0 :     BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc,
    1958           0 :             TII.get(TargetOpcode::COPY), ResultReg).addReg(II.ImplicitDefs[0]);
    1959             :   }
    1960           0 :   return ResultReg;
    1961             : }
    1962             : 
    1963         228 : unsigned FastISel::fastEmitInst_rri(unsigned MachineInstOpcode,
    1964             :                                     const TargetRegisterClass *RC, unsigned Op0,
    1965             :                                     bool Op0IsKill, unsigned Op1,
    1966             :                                     bool Op1IsKill, uint64_t Imm) {
    1967         456 :   const MCInstrDesc &II = TII.get(MachineInstOpcode);
    1968             : 
    1969         228 :   unsigned ResultReg = createResultReg(RC);
    1970         228 :   Op0 = constrainOperandRegClass(II, Op0, II.getNumDefs());
    1971         228 :   Op1 = constrainOperandRegClass(II, Op1, II.getNumDefs() + 1);
    1972             : 
    1973         228 :   if (II.getNumDefs() >= 1)
    1974         456 :     BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc, II, ResultReg)
    1975         228 :         .addReg(Op0, getKillRegState(Op0IsKill))
    1976         228 :         .addReg(Op1, getKillRegState(Op1IsKill))
    1977         456 :         .addImm(Imm);
    1978             :   else {
    1979           0 :     BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc, II)
    1980           0 :         .addReg(Op0, getKillRegState(Op0IsKill))
    1981           0 :         .addReg(Op1, getKillRegState(Op1IsKill))
    1982           0 :         .addImm(Imm);
    1983           0 :     BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc,
    1984           0 :             TII.get(TargetOpcode::COPY), ResultReg).addReg(II.ImplicitDefs[0]);
    1985             :   }
    1986         228 :   return ResultReg;
    1987             : }
    1988             : 
    1989         613 : unsigned FastISel::fastEmitInst_i(unsigned MachineInstOpcode,
    1990             :                                   const TargetRegisterClass *RC, uint64_t Imm) {
    1991         613 :   unsigned ResultReg = createResultReg(RC);
    1992        1226 :   const MCInstrDesc &II = TII.get(MachineInstOpcode);
    1993             : 
    1994         613 :   if (II.getNumDefs() >= 1)
    1995        1226 :     BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc, II, ResultReg)
    1996        1226 :         .addImm(Imm);
    1997             :   else {
    1998           0 :     BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc, II).addImm(Imm);
    1999           0 :     BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc,
    2000           0 :             TII.get(TargetOpcode::COPY), ResultReg).addReg(II.ImplicitDefs[0]);
    2001             :   }
    2002         613 :   return ResultReg;
    2003             : }
    2004             : 
    2005         145 : unsigned FastISel::fastEmitInst_extractsubreg(MVT RetVT, unsigned Op0,
    2006             :                                               bool Op0IsKill, uint32_t Idx) {
    2007         145 :   unsigned ResultReg = createResultReg(TLI.getRegClassFor(RetVT));
    2008             :   assert(TargetRegisterInfo::isVirtualRegister(Op0) &&
    2009             :          "Cannot yet extract from physregs");
    2010         290 :   const TargetRegisterClass *RC = MRI.getRegClass(Op0);
    2011         145 :   MRI.constrainRegClass(Op0, TRI.getSubClassWithSubReg(RC, Idx));
    2012         435 :   BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc, TII.get(TargetOpcode::COPY),
    2013         435 :           ResultReg).addReg(Op0, getKillRegState(Op0IsKill), Idx);
    2014         145 :   return ResultReg;
    2015             : }
    2016             : 
    2017             : /// Emit MachineInstrs to compute the value of Op with all but the least
    2018             : /// significant bit set to zero.
    2019         174 : unsigned FastISel::fastEmitZExtFromI1(MVT VT, unsigned Op0, bool Op0IsKill) {
    2020         174 :   return fastEmit_ri(VT, VT, ISD::AND, Op0, Op0IsKill, 1);
    2021             : }
    2022             : 
    2023             : /// HandlePHINodesInSuccessorBlocks - Handle PHI nodes in successor blocks.
    2024             : /// Emit code to ensure constants are copied into registers when needed.
    2025             : /// Remember the virtual registers that need to be added to the Machine PHI
    2026             : /// nodes as input.  We cannot just directly add them, because expansion
    2027             : /// might result in multiple MBB's for one BB.  As such, the start of the
    2028             : /// BB might correspond to a different MBB than the end.
    2029        9706 : bool FastISel::handlePHINodesInSuccessorBlocks(const BasicBlock *LLVMBB) {
    2030        9706 :   const TerminatorInst *TI = LLVMBB->getTerminator();
    2031             : 
    2032       19412 :   SmallPtrSet<MachineBasicBlock *, 4> SuccsHandled;
    2033       19412 :   FuncInfo.OrigNumPHINodesToUpdate = FuncInfo.PHINodesToUpdate.size();
    2034             : 
    2035             :   // Check successor nodes' PHI nodes that expect a constant to be available
    2036             :   // from this block.
    2037       12491 :   for (unsigned succ = 0, e = TI->getNumSuccessors(); succ != e; ++succ) {
    2038        2843 :     const BasicBlock *SuccBB = TI->getSuccessor(succ);
    2039        8529 :     if (!isa<PHINode>(SuccBB->begin()))
    2040        5045 :       continue;
    2041         642 :     MachineBasicBlock *SuccMBB = FuncInfo.MBBMap[SuccBB];
    2042             : 
    2043             :     // If this terminator has multiple identical successors (common for
    2044             :     // switches), only handle each succ once.
    2045         321 :     if (!SuccsHandled.insert(SuccMBB).second)
    2046           1 :       continue;
    2047             : 
    2048         320 :     MachineBasicBlock::iterator MBBI = SuccMBB->begin();
    2049             : 
    2050             :     // At this point we know that there is a 1-1 correspondence between LLVM PHI
    2051             :     // nodes and Machine PHI nodes, but the incoming operands have not been
    2052             :     // emitted yet.
    2053         640 :     for (BasicBlock::const_iterator I = SuccBB->begin();
    2054         335 :          const auto *PN = dyn_cast<PHINode>(I); ++I) {
    2055             : 
    2056             :       // Ignore dead phi's.
    2057         670 :       if (PN->use_empty())
    2058           8 :         continue;
    2059             : 
    2060             :       // Only handle legal types. Two interesting things to note here. First,
    2061             :       // by bailing out early, we may leave behind some dead instructions,
    2062             :       // since SelectionDAG's HandlePHINodesInSuccessorBlocks will insert its
    2063             :       // own moves. Second, this check is necessary because FastISel doesn't
    2064             :       // use CreateRegs to create registers, so it always creates
    2065             :       // exactly one register for each non-void instruction.
    2066         327 :       EVT VT = TLI.getValueType(DL, PN->getType(), /*AllowUnknown=*/true);
    2067         654 :       if (VT == MVT::Other || !TLI.isTypeLegal(VT)) {
    2068             :         // Handle integer promotions, though, because they're common and easy.
    2069         201 :         if (!(VT == MVT::i1 || VT == MVT::i8 || VT == MVT::i16)) {
    2070          57 :           FuncInfo.PHINodesToUpdate.resize(FuncInfo.OrigNumPHINodesToUpdate);
    2071          57 :           return false;
    2072             :         }
    2073             :       }
    2074             : 
    2075         270 :       const Value *PHIOp = PN->getIncomingValueForBlock(LLVMBB);
    2076             : 
    2077             :       // Set the DebugLoc for the copy. Prefer the location of the operand
    2078             :       // if there is one; use the location of the PHI otherwise.
    2079         540 :       DbgLoc = PN->getDebugLoc();
    2080         201 :       if (const auto *Inst = dyn_cast<Instruction>(PHIOp))
    2081         201 :         DbgLoc = Inst->getDebugLoc();
    2082             : 
    2083         270 :       unsigned Reg = getRegForValue(PHIOp);
    2084         270 :       if (!Reg) {
    2085           1 :         FuncInfo.PHINodesToUpdate.resize(FuncInfo.OrigNumPHINodesToUpdate);
    2086           1 :         return false;
    2087             :       }
    2088        1345 :       FuncInfo.PHINodesToUpdate.push_back(std::make_pair(&*MBBI++, Reg));
    2089        1076 :       DbgLoc = DebugLoc();
    2090             :     }
    2091             :   }
    2092             : 
    2093             :   return true;
    2094             : }
    2095             : 
    2096        2008 : bool FastISel::tryToFoldLoad(const LoadInst *LI, const Instruction *FoldInst) {
    2097             :   assert(LI->hasOneUse() &&
    2098             :          "tryToFoldLoad expected a LoadInst with a single use");
    2099             :   // We know that the load has a single use, but don't know what it is.  If it
    2100             :   // isn't one of the folded instructions, then we can't succeed here.  Handle
    2101             :   // this by scanning the single-use users of the load until we get to FoldInst.
    2102        2008 :   unsigned MaxUsers = 6; // Don't scan down huge single-use chains of instrs.
    2103             : 
    2104        4016 :   const Instruction *TheUser = LI->user_back();
    2105        1661 :   while (TheUser != FoldInst && // Scan up until we find FoldInst.
    2106             :          // Stay in the right block.
    2107        3576 :          TheUser->getParent() == FoldInst->getParent() &&
    2108             :          --MaxUsers) { // Don't scan too far.
    2109             :     // If there are multiple or no uses of this instruction, then bail out.
    2110        1574 :     if (!TheUser->hasOneUse())
    2111             :       return false;
    2112             : 
    2113         574 :     TheUser = TheUser->user_back();
    2114             :   }
    2115             : 
    2116             :   // If we didn't find the fold instruction, then we failed to collapse the
    2117             :   // sequence.
    2118        1593 :   if (TheUser != FoldInst)
    2119             :     return false;
    2120             : 
    2121             :   // Don't try to fold volatile loads.  Target has to deal with alignment
    2122             :   // constraints.
    2123        1495 :   if (LI->isVolatile())
    2124             :     return false;
    2125             : 
    2126             :   // Figure out which vreg this is going into.  If there is no assigned vreg yet
    2127             :   // then there actually was no reference to it.  Perhaps the load is referenced
    2128             :   // by a dead instruction.
    2129        1440 :   unsigned LoadReg = getRegForValue(LI);
    2130        1440 :   if (!LoadReg)
    2131             :     return false;
    2132             : 
    2133             :   // We can't fold if this vreg has no uses or more than one use.  Multiple uses
    2134             :   // may mean that the instruction got lowered to multiple MIs, or the use of
    2135             :   // the loaded value ended up being multiple operands of the result.
    2136        1440 :   if (!MRI.hasOneUse(LoadReg))
    2137             :     return false;
    2138             : 
    2139        2796 :   MachineRegisterInfo::reg_iterator RI = MRI.reg_begin(LoadReg);
    2140        1398 :   MachineInstr *User = RI->getParent();
    2141             : 
    2142             :   // Set the insertion point properly.  Folding the load can cause generation of
    2143             :   // other random instructions (like sign extends) for addressing modes; make
    2144             :   // sure they get inserted in a logical place before the new instruction.
    2145        1398 :   FuncInfo.InsertPt = User;
    2146        1398 :   FuncInfo.MBB = User->getParent();
    2147             : 
    2148             :   // Ask the target to try folding the load.
    2149        2796 :   return tryToFoldLoadIntoMI(User, RI.getOperandNo(), LI);
    2150             : }
    2151             : 
    2152          76 : bool FastISel::canFoldAddIntoGEP(const User *GEP, const Value *Add) {
    2153             :   // Must be an add.
    2154          76 :   if (!isa<AddOperator>(Add))
    2155             :     return false;
    2156             :   // Type size needs to match.
    2157          22 :   if (DL.getTypeSizeInBits(GEP->getType()) !=
    2158          11 :       DL.getTypeSizeInBits(Add->getType()))
    2159             :     return false;
    2160             :   // Must be in the same basic block.
    2161          15 :   if (isa<Instruction>(Add) &&
    2162          20 :       FuncInfo.MBBMap[cast<Instruction>(Add)->getParent()] != FuncInfo.MBB)
    2163             :     return false;
    2164             :   // Must have a constant operand.
    2165          12 :   return isa<ConstantInt>(cast<AddOperator>(Add)->getOperand(1));
    2166             : }
    2167             : 
    2168             : MachineMemOperand *
    2169        3040 : FastISel::createMachineMemOperandFor(const Instruction *I) const {
    2170             :   const Value *Ptr;
    2171             :   Type *ValTy;
    2172             :   unsigned Alignment;
    2173             :   MachineMemOperand::Flags Flags;
    2174             :   bool IsVolatile;
    2175             : 
    2176        1431 :   if (const auto *LI = dyn_cast<LoadInst>(I)) {
    2177        1431 :     Alignment = LI->getAlignment();
    2178        1431 :     IsVolatile = LI->isVolatile();
    2179        1431 :     Flags = MachineMemOperand::MOLoad;
    2180        1431 :     Ptr = LI->getPointerOperand();
    2181        1431 :     ValTy = LI->getType();
    2182        1609 :   } else if (const auto *SI = dyn_cast<StoreInst>(I)) {
    2183        1609 :     Alignment = SI->getAlignment();
    2184        1609 :     IsVolatile = SI->isVolatile();
    2185        1609 :     Flags = MachineMemOperand::MOStore;
    2186        1609 :     Ptr = SI->getPointerOperand();
    2187        1609 :     ValTy = SI->getValueOperand()->getType();
    2188             :   } else
    2189             :     return nullptr;
    2190             : 
    2191        3040 :   bool IsNonTemporal = I->getMetadata(LLVMContext::MD_nontemporal) != nullptr;
    2192        3040 :   bool IsInvariant = I->getMetadata(LLVMContext::MD_invariant_load) != nullptr;
    2193             :   bool IsDereferenceable =
    2194        3040 :       I->getMetadata(LLVMContext::MD_dereferenceable) != nullptr;
    2195        3040 :   const MDNode *Ranges = I->getMetadata(LLVMContext::MD_range);
    2196             : 
    2197        3040 :   AAMDNodes AAInfo;
    2198        3040 :   I->getAAMetadata(AAInfo);
    2199             : 
    2200        3040 :   if (Alignment == 0) // Ensure that codegen never sees alignment 0.
    2201        1004 :     Alignment = DL.getABITypeAlignment(ValTy);
    2202             : 
    2203        6080 :   unsigned Size = DL.getTypeStoreSize(ValTy);
    2204             : 
    2205        3040 :   if (IsVolatile)
    2206             :     Flags |= MachineMemOperand::MOVolatile;
    2207        3040 :   if (IsNonTemporal)
    2208             :     Flags |= MachineMemOperand::MONonTemporal;
    2209        3040 :   if (IsDereferenceable)
    2210             :     Flags |= MachineMemOperand::MODereferenceable;
    2211        3040 :   if (IsInvariant)
    2212             :     Flags |= MachineMemOperand::MOInvariant;
    2213             : 
    2214        9120 :   return FuncInfo.MF->getMachineMemOperand(MachinePointerInfo(Ptr), Flags, Size,
    2215        3040 :                                            Alignment, AAInfo, Ranges);
    2216             : }
    2217             : 
    2218         893 : CmpInst::Predicate FastISel::optimizeCmpPredicate(const CmpInst *CI) const {
    2219             :   // If both operands are the same, then try to optimize or fold the cmp.
    2220         893 :   CmpInst::Predicate Predicate = CI->getPredicate();
    2221        1786 :   if (CI->getOperand(0) != CI->getOperand(1))
    2222             :     return Predicate;
    2223             : 
    2224          53 :   switch (Predicate) {
    2225           0 :   default: llvm_unreachable("Invalid predicate!");
    2226             :   case CmpInst::FCMP_FALSE: Predicate = CmpInst::FCMP_FALSE; break;
    2227           2 :   case CmpInst::FCMP_OEQ:   Predicate = CmpInst::FCMP_ORD;   break;
    2228             :   case CmpInst::FCMP_OGT:   Predicate = CmpInst::FCMP_FALSE; break;
    2229           2 :   case CmpInst::FCMP_OGE:   Predicate = CmpInst::FCMP_ORD;   break;
    2230             :   case CmpInst::FCMP_OLT:   Predicate = CmpInst::FCMP_FALSE; break;
    2231           2 :   case CmpInst::FCMP_OLE:   Predicate = CmpInst::FCMP_ORD;   break;
    2232             :   case CmpInst::FCMP_ONE:   Predicate = CmpInst::FCMP_FALSE; break;
    2233           2 :   case CmpInst::FCMP_ORD:   Predicate = CmpInst::FCMP_ORD;   break;
    2234           2 :   case CmpInst::FCMP_UNO:   Predicate = CmpInst::FCMP_UNO;   break;
    2235           4 :   case CmpInst::FCMP_UEQ:   Predicate = CmpInst::FCMP_TRUE;  break;
    2236           2 :   case CmpInst::FCMP_UGT:   Predicate = CmpInst::FCMP_UNO;   break;
    2237           2 :   case CmpInst::FCMP_UGE:   Predicate = CmpInst::FCMP_TRUE;  break;
    2238           2 :   case CmpInst::FCMP_ULT:   Predicate = CmpInst::FCMP_UNO;   break;
    2239           2 :   case CmpInst::FCMP_ULE:   Predicate = CmpInst::FCMP_TRUE;  break;
    2240           2 :   case CmpInst::FCMP_UNE:   Predicate = CmpInst::FCMP_UNO;   break;
    2241           0 :   case CmpInst::FCMP_TRUE:  Predicate = CmpInst::FCMP_TRUE;  break;
    2242             : 
    2243           2 :   case CmpInst::ICMP_EQ:    Predicate = CmpInst::FCMP_TRUE;  break;
    2244             :   case CmpInst::ICMP_NE:    Predicate = CmpInst::FCMP_FALSE; break;
    2245             :   case CmpInst::ICMP_UGT:   Predicate = CmpInst::FCMP_FALSE; break;
    2246           2 :   case CmpInst::ICMP_UGE:   Predicate = CmpInst::FCMP_TRUE;  break;
    2247             :   case CmpInst::ICMP_ULT:   Predicate = CmpInst::FCMP_FALSE; break;
    2248           2 :   case CmpInst::ICMP_ULE:   Predicate = CmpInst::FCMP_TRUE;  break;
    2249             :   case CmpInst::ICMP_SGT:   Predicate = CmpInst::FCMP_FALSE; break;
    2250           2 :   case CmpInst::ICMP_SGE:   Predicate = CmpInst::FCMP_TRUE;  break;
    2251             :   case CmpInst::ICMP_SLT:   Predicate = CmpInst::FCMP_FALSE; break;
    2252           2 :   case CmpInst::ICMP_SLE:   Predicate = CmpInst::FCMP_TRUE;  break;
    2253             :   }
    2254             : 
    2255             :   return Predicate;
    2256             : }

Generated by: LCOV version 1.13