LLVM  mainline
SelectionDAGBuilder.cpp
Go to the documentation of this file.
00001 //===-- SelectionDAGBuilder.cpp - Selection-DAG building ------------------===//
00002 //
00003 //                     The LLVM Compiler Infrastructure
00004 //
00005 // This file is distributed under the University of Illinois Open Source
00006 // License. See LICENSE.TXT for details.
00007 //
00008 //===----------------------------------------------------------------------===//
00009 //
00010 // This implements routines for translating from LLVM IR into SelectionDAG IR.
00011 //
00012 //===----------------------------------------------------------------------===//
00013 
00014 #include "SelectionDAGBuilder.h"
00015 #include "SDNodeDbgValue.h"
00016 #include "llvm/ADT/BitVector.h"
00017 #include "llvm/ADT/Optional.h"
00018 #include "llvm/ADT/SmallSet.h"
00019 #include "llvm/ADT/Statistic.h"
00020 #include "llvm/Analysis/AliasAnalysis.h"
00021 #include "llvm/Analysis/BranchProbabilityInfo.h"
00022 #include "llvm/Analysis/ConstantFolding.h"
00023 #include "llvm/Analysis/TargetLibraryInfo.h"
00024 #include "llvm/Analysis/ValueTracking.h"
00025 #include "llvm/CodeGen/FastISel.h"
00026 #include "llvm/CodeGen/FunctionLoweringInfo.h"
00027 #include "llvm/CodeGen/GCMetadata.h"
00028 #include "llvm/CodeGen/GCStrategy.h"
00029 #include "llvm/CodeGen/MachineFrameInfo.h"
00030 #include "llvm/CodeGen/MachineFunction.h"
00031 #include "llvm/CodeGen/MachineInstrBuilder.h"
00032 #include "llvm/CodeGen/MachineJumpTableInfo.h"
00033 #include "llvm/CodeGen/MachineModuleInfo.h"
00034 #include "llvm/CodeGen/MachineRegisterInfo.h"
00035 #include "llvm/CodeGen/SelectionDAG.h"
00036 #include "llvm/CodeGen/StackMaps.h"
00037 #include "llvm/CodeGen/WinEHFuncInfo.h"
00038 #include "llvm/IR/CallingConv.h"
00039 #include "llvm/IR/Constants.h"
00040 #include "llvm/IR/DataLayout.h"
00041 #include "llvm/IR/DebugInfo.h"
00042 #include "llvm/IR/DerivedTypes.h"
00043 #include "llvm/IR/Function.h"
00044 #include "llvm/IR/GlobalVariable.h"
00045 #include "llvm/IR/InlineAsm.h"
00046 #include "llvm/IR/Instructions.h"
00047 #include "llvm/IR/IntrinsicInst.h"
00048 #include "llvm/IR/Intrinsics.h"
00049 #include "llvm/IR/LLVMContext.h"
00050 #include "llvm/IR/Module.h"
00051 #include "llvm/IR/Statepoint.h"
00052 #include "llvm/MC/MCSymbol.h"
00053 #include "llvm/Support/CommandLine.h"
00054 #include "llvm/Support/Debug.h"
00055 #include "llvm/Support/ErrorHandling.h"
00056 #include "llvm/Support/MathExtras.h"
00057 #include "llvm/Support/raw_ostream.h"
00058 #include "llvm/Target/TargetFrameLowering.h"
00059 #include "llvm/Target/TargetInstrInfo.h"
00060 #include "llvm/Target/TargetIntrinsicInfo.h"
00061 #include "llvm/Target/TargetLowering.h"
00062 #include "llvm/Target/TargetOptions.h"
00063 #include "llvm/Target/TargetSelectionDAGInfo.h"
00064 #include "llvm/Target/TargetSubtargetInfo.h"
00065 #include <algorithm>
00066 using namespace llvm;
00067 
00068 #define DEBUG_TYPE "isel"
00069 
00070 /// LimitFloatPrecision - Generate low-precision inline sequences for
00071 /// some float libcalls (6, 8 or 12 bits).
00072 static unsigned LimitFloatPrecision;
00073 
00074 static cl::opt<unsigned, true>
00075 LimitFPPrecision("limit-float-precision",
00076                  cl::desc("Generate low-precision inline sequences "
00077                           "for some float libcalls"),
00078                  cl::location(LimitFloatPrecision),
00079                  cl::init(0));
00080 
00081 static cl::opt<bool>
00082 EnableFMFInDAG("enable-fmf-dag", cl::init(false), cl::Hidden,
00083                 cl::desc("Enable fast-math-flags for DAG nodes"));
00084 
00085 // Limit the width of DAG chains. This is important in general to prevent
00086 // DAG-based analysis from blowing up. For example, alias analysis and
00087 // load clustering may not complete in reasonable time. It is difficult to
00088 // recognize and avoid this situation within each individual analysis, and
00089 // future analyses are likely to have the same behavior. Limiting DAG width is
00090 // the safe approach and will be especially important with global DAGs.
00091 //
00092 // MaxParallelChains default is arbitrarily high to avoid affecting
00093 // optimization, but could be lowered to improve compile time. Any ld-ld-st-st
00094 // sequence over this should have been converted to llvm.memcpy by the
00095 // frontend. It easy to induce this behavior with .ll code such as:
00096 // %buffer = alloca [4096 x i8]
00097 // %data = load [4096 x i8]* %argPtr
00098 // store [4096 x i8] %data, [4096 x i8]* %buffer
00099 static const unsigned MaxParallelChains = 64;
00100 
00101 static SDValue getCopyFromPartsVector(SelectionDAG &DAG, SDLoc DL,
00102                                       const SDValue *Parts, unsigned NumParts,
00103                                       MVT PartVT, EVT ValueVT, const Value *V);
00104 
00105 /// getCopyFromParts - Create a value that contains the specified legal parts
00106 /// combined into the value they represent.  If the parts combine to a type
00107 /// larger then ValueVT then AssertOp can be used to specify whether the extra
00108 /// bits are known to be zero (ISD::AssertZext) or sign extended from ValueVT
00109 /// (ISD::AssertSext).
00110 static SDValue getCopyFromParts(SelectionDAG &DAG, SDLoc DL,
00111                                 const SDValue *Parts,
00112                                 unsigned NumParts, MVT PartVT, EVT ValueVT,
00113                                 const Value *V,
00114                                 ISD::NodeType AssertOp = ISD::DELETED_NODE) {
00115   if (ValueVT.isVector())
00116     return getCopyFromPartsVector(DAG, DL, Parts, NumParts,
00117                                   PartVT, ValueVT, V);
00118 
00119   assert(NumParts > 0 && "No parts to assemble!");
00120   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
00121   SDValue Val = Parts[0];
00122 
00123   if (NumParts > 1) {
00124     // Assemble the value from multiple parts.
00125     if (ValueVT.isInteger()) {
00126       unsigned PartBits = PartVT.getSizeInBits();
00127       unsigned ValueBits = ValueVT.getSizeInBits();
00128 
00129       // Assemble the power of 2 part.
00130       unsigned RoundParts = NumParts & (NumParts - 1) ?
00131         1 << Log2_32(NumParts) : NumParts;
00132       unsigned RoundBits = PartBits * RoundParts;
00133       EVT RoundVT = RoundBits == ValueBits ?
00134         ValueVT : EVT::getIntegerVT(*DAG.getContext(), RoundBits);
00135       SDValue Lo, Hi;
00136 
00137       EVT HalfVT = EVT::getIntegerVT(*DAG.getContext(), RoundBits/2);
00138 
00139       if (RoundParts > 2) {
00140         Lo = getCopyFromParts(DAG, DL, Parts, RoundParts / 2,
00141                               PartVT, HalfVT, V);
00142         Hi = getCopyFromParts(DAG, DL, Parts + RoundParts / 2,
00143                               RoundParts / 2, PartVT, HalfVT, V);
00144       } else {
00145         Lo = DAG.getNode(ISD::BITCAST, DL, HalfVT, Parts[0]);
00146         Hi = DAG.getNode(ISD::BITCAST, DL, HalfVT, Parts[1]);
00147       }
00148 
00149       if (TLI.isBigEndian())
00150         std::swap(Lo, Hi);
00151 
00152       Val = DAG.getNode(ISD::BUILD_PAIR, DL, RoundVT, Lo, Hi);
00153 
00154       if (RoundParts < NumParts) {
00155         // Assemble the trailing non-power-of-2 part.
00156         unsigned OddParts = NumParts - RoundParts;
00157         EVT OddVT = EVT::getIntegerVT(*DAG.getContext(), OddParts * PartBits);
00158         Hi = getCopyFromParts(DAG, DL,
00159                               Parts + RoundParts, OddParts, PartVT, OddVT, V);
00160 
00161         // Combine the round and odd parts.
00162         Lo = Val;
00163         if (TLI.isBigEndian())
00164           std::swap(Lo, Hi);
00165         EVT TotalVT = EVT::getIntegerVT(*DAG.getContext(), NumParts * PartBits);
00166         Hi = DAG.getNode(ISD::ANY_EXTEND, DL, TotalVT, Hi);
00167         Hi = DAG.getNode(ISD::SHL, DL, TotalVT, Hi,
00168                          DAG.getConstant(Lo.getValueType().getSizeInBits(), DL,
00169                                          TLI.getPointerTy()));
00170         Lo = DAG.getNode(ISD::ZERO_EXTEND, DL, TotalVT, Lo);
00171         Val = DAG.getNode(ISD::OR, DL, TotalVT, Lo, Hi);
00172       }
00173     } else if (PartVT.isFloatingPoint()) {
00174       // FP split into multiple FP parts (for ppcf128)
00175       assert(ValueVT == EVT(MVT::ppcf128) && PartVT == MVT::f64 &&
00176              "Unexpected split");
00177       SDValue Lo, Hi;
00178       Lo = DAG.getNode(ISD::BITCAST, DL, EVT(MVT::f64), Parts[0]);
00179       Hi = DAG.getNode(ISD::BITCAST, DL, EVT(MVT::f64), Parts[1]);
00180       if (TLI.hasBigEndianPartOrdering(ValueVT))
00181         std::swap(Lo, Hi);
00182       Val = DAG.getNode(ISD::BUILD_PAIR, DL, ValueVT, Lo, Hi);
00183     } else {
00184       // FP split into integer parts (soft fp)
00185       assert(ValueVT.isFloatingPoint() && PartVT.isInteger() &&
00186              !PartVT.isVector() && "Unexpected split");
00187       EVT IntVT = EVT::getIntegerVT(*DAG.getContext(), ValueVT.getSizeInBits());
00188       Val = getCopyFromParts(DAG, DL, Parts, NumParts, PartVT, IntVT, V);
00189     }
00190   }
00191 
00192   // There is now one part, held in Val.  Correct it to match ValueVT.
00193   EVT PartEVT = Val.getValueType();
00194 
00195   if (PartEVT == ValueVT)
00196     return Val;
00197 
00198   if (PartEVT.isInteger() && ValueVT.isInteger()) {
00199     if (ValueVT.bitsLT(PartEVT)) {
00200       // For a truncate, see if we have any information to
00201       // indicate whether the truncated bits will always be
00202       // zero or sign-extension.
00203       if (AssertOp != ISD::DELETED_NODE)
00204         Val = DAG.getNode(AssertOp, DL, PartEVT, Val,
00205                           DAG.getValueType(ValueVT));
00206       return DAG.getNode(ISD::TRUNCATE, DL, ValueVT, Val);
00207     }
00208     return DAG.getNode(ISD::ANY_EXTEND, DL, ValueVT, Val);
00209   }
00210 
00211   if (PartEVT.isFloatingPoint() && ValueVT.isFloatingPoint()) {
00212     // FP_ROUND's are always exact here.
00213     if (ValueVT.bitsLT(Val.getValueType()))
00214       return DAG.getNode(ISD::FP_ROUND, DL, ValueVT, Val,
00215                          DAG.getTargetConstant(1, DL, TLI.getPointerTy()));
00216 
00217     return DAG.getNode(ISD::FP_EXTEND, DL, ValueVT, Val);
00218   }
00219 
00220   if (PartEVT.getSizeInBits() == ValueVT.getSizeInBits())
00221     return DAG.getNode(ISD::BITCAST, DL, ValueVT, Val);
00222 
00223   llvm_unreachable("Unknown mismatch!");
00224 }
00225 
00226 static void diagnosePossiblyInvalidConstraint(LLVMContext &Ctx, const Value *V,
00227                                               const Twine &ErrMsg) {
00228   const Instruction *I = dyn_cast_or_null<Instruction>(V);
00229   if (!V)
00230     return Ctx.emitError(ErrMsg);
00231 
00232   const char *AsmError = ", possible invalid constraint for vector type";
00233   if (const CallInst *CI = dyn_cast<CallInst>(I))
00234     if (isa<InlineAsm>(CI->getCalledValue()))
00235       return Ctx.emitError(I, ErrMsg + AsmError);
00236 
00237   return Ctx.emitError(I, ErrMsg);
00238 }
00239 
00240 /// getCopyFromPartsVector - Create a value that contains the specified legal
00241 /// parts combined into the value they represent.  If the parts combine to a
00242 /// type larger then ValueVT then AssertOp can be used to specify whether the
00243 /// extra bits are known to be zero (ISD::AssertZext) or sign extended from
00244 /// ValueVT (ISD::AssertSext).
00245 static SDValue getCopyFromPartsVector(SelectionDAG &DAG, SDLoc DL,
00246                                       const SDValue *Parts, unsigned NumParts,
00247                                       MVT PartVT, EVT ValueVT, const Value *V) {
00248   assert(ValueVT.isVector() && "Not a vector value");
00249   assert(NumParts > 0 && "No parts to assemble!");
00250   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
00251   SDValue Val = Parts[0];
00252 
00253   // Handle a multi-element vector.
00254   if (NumParts > 1) {
00255     EVT IntermediateVT;
00256     MVT RegisterVT;
00257     unsigned NumIntermediates;
00258     unsigned NumRegs =
00259     TLI.getVectorTypeBreakdown(*DAG.getContext(), ValueVT, IntermediateVT,
00260                                NumIntermediates, RegisterVT);
00261     assert(NumRegs == NumParts && "Part count doesn't match vector breakdown!");
00262     NumParts = NumRegs; // Silence a compiler warning.
00263     assert(RegisterVT == PartVT && "Part type doesn't match vector breakdown!");
00264     assert(RegisterVT == Parts[0].getSimpleValueType() &&
00265            "Part type doesn't match part!");
00266 
00267     // Assemble the parts into intermediate operands.
00268     SmallVector<SDValue, 8> Ops(NumIntermediates);
00269     if (NumIntermediates == NumParts) {
00270       // If the register was not expanded, truncate or copy the value,
00271       // as appropriate.
00272       for (unsigned i = 0; i != NumParts; ++i)
00273         Ops[i] = getCopyFromParts(DAG, DL, &Parts[i], 1,
00274                                   PartVT, IntermediateVT, V);
00275     } else if (NumParts > 0) {
00276       // If the intermediate type was expanded, build the intermediate
00277       // operands from the parts.
00278       assert(NumParts % NumIntermediates == 0 &&
00279              "Must expand into a divisible number of parts!");
00280       unsigned Factor = NumParts / NumIntermediates;
00281       for (unsigned i = 0; i != NumIntermediates; ++i)
00282         Ops[i] = getCopyFromParts(DAG, DL, &Parts[i * Factor], Factor,
00283                                   PartVT, IntermediateVT, V);
00284     }
00285 
00286     // Build a vector with BUILD_VECTOR or CONCAT_VECTORS from the
00287     // intermediate operands.
00288     Val = DAG.getNode(IntermediateVT.isVector() ? ISD::CONCAT_VECTORS
00289                                                 : ISD::BUILD_VECTOR,
00290                       DL, ValueVT, Ops);
00291   }
00292 
00293   // There is now one part, held in Val.  Correct it to match ValueVT.
00294   EVT PartEVT = Val.getValueType();
00295 
00296   if (PartEVT == ValueVT)
00297     return Val;
00298 
00299   if (PartEVT.isVector()) {
00300     // If the element type of the source/dest vectors are the same, but the
00301     // parts vector has more elements than the value vector, then we have a
00302     // vector widening case (e.g. <2 x float> -> <4 x float>).  Extract the
00303     // elements we want.
00304     if (PartEVT.getVectorElementType() == ValueVT.getVectorElementType()) {
00305       assert(PartEVT.getVectorNumElements() > ValueVT.getVectorNumElements() &&
00306              "Cannot narrow, it would be a lossy transformation");
00307       return DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, ValueVT, Val,
00308                          DAG.getConstant(0, DL, TLI.getVectorIdxTy()));
00309     }
00310 
00311     // Vector/Vector bitcast.
00312     if (ValueVT.getSizeInBits() == PartEVT.getSizeInBits())
00313       return DAG.getNode(ISD::BITCAST, DL, ValueVT, Val);
00314 
00315     assert(PartEVT.getVectorNumElements() == ValueVT.getVectorNumElements() &&
00316       "Cannot handle this kind of promotion");
00317     // Promoted vector extract
00318     bool Smaller = ValueVT.bitsLE(PartEVT);
00319     return DAG.getNode((Smaller ? ISD::TRUNCATE : ISD::ANY_EXTEND),
00320                        DL, ValueVT, Val);
00321 
00322   }
00323 
00324   // Trivial bitcast if the types are the same size and the destination
00325   // vector type is legal.
00326   if (PartEVT.getSizeInBits() == ValueVT.getSizeInBits() &&
00327       TLI.isTypeLegal(ValueVT))
00328     return DAG.getNode(ISD::BITCAST, DL, ValueVT, Val);
00329 
00330   // Handle cases such as i8 -> <1 x i1>
00331   if (ValueVT.getVectorNumElements() != 1) {
00332     diagnosePossiblyInvalidConstraint(*DAG.getContext(), V,
00333                                       "non-trivial scalar-to-vector conversion");
00334     return DAG.getUNDEF(ValueVT);
00335   }
00336 
00337   if (ValueVT.getVectorNumElements() == 1 &&
00338       ValueVT.getVectorElementType() != PartEVT) {
00339     bool Smaller = ValueVT.bitsLE(PartEVT);
00340     Val = DAG.getNode((Smaller ? ISD::TRUNCATE : ISD::ANY_EXTEND),
00341                        DL, ValueVT.getScalarType(), Val);
00342   }
00343 
00344   return DAG.getNode(ISD::BUILD_VECTOR, DL, ValueVT, Val);
00345 }
00346 
00347 static void getCopyToPartsVector(SelectionDAG &DAG, SDLoc dl,
00348                                  SDValue Val, SDValue *Parts, unsigned NumParts,
00349                                  MVT PartVT, const Value *V);
00350 
00351 /// getCopyToParts - Create a series of nodes that contain the specified value
00352 /// split into legal parts.  If the parts contain more bits than Val, then, for
00353 /// integers, ExtendKind can be used to specify how to generate the extra bits.
00354 static void getCopyToParts(SelectionDAG &DAG, SDLoc DL,
00355                            SDValue Val, SDValue *Parts, unsigned NumParts,
00356                            MVT PartVT, const Value *V,
00357                            ISD::NodeType ExtendKind = ISD::ANY_EXTEND) {
00358   EVT ValueVT = Val.getValueType();
00359 
00360   // Handle the vector case separately.
00361   if (ValueVT.isVector())
00362     return getCopyToPartsVector(DAG, DL, Val, Parts, NumParts, PartVT, V);
00363 
00364   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
00365   unsigned PartBits = PartVT.getSizeInBits();
00366   unsigned OrigNumParts = NumParts;
00367   assert(TLI.isTypeLegal(PartVT) && "Copying to an illegal type!");
00368 
00369   if (NumParts == 0)
00370     return;
00371 
00372   assert(!ValueVT.isVector() && "Vector case handled elsewhere");
00373   EVT PartEVT = PartVT;
00374   if (PartEVT == ValueVT) {
00375     assert(NumParts == 1 && "No-op copy with multiple parts!");
00376     Parts[0] = Val;
00377     return;
00378   }
00379 
00380   if (NumParts * PartBits > ValueVT.getSizeInBits()) {
00381     // If the parts cover more bits than the value has, promote the value.
00382     if (PartVT.isFloatingPoint() && ValueVT.isFloatingPoint()) {
00383       assert(NumParts == 1 && "Do not know what to promote to!");
00384       Val = DAG.getNode(ISD::FP_EXTEND, DL, PartVT, Val);
00385     } else {
00386       assert((PartVT.isInteger() || PartVT == MVT::x86mmx) &&
00387              ValueVT.isInteger() &&
00388              "Unknown mismatch!");
00389       ValueVT = EVT::getIntegerVT(*DAG.getContext(), NumParts * PartBits);
00390       Val = DAG.getNode(ExtendKind, DL, ValueVT, Val);
00391       if (PartVT == MVT::x86mmx)
00392         Val = DAG.getNode(ISD::BITCAST, DL, PartVT, Val);
00393     }
00394   } else if (PartBits == ValueVT.getSizeInBits()) {
00395     // Different types of the same size.
00396     assert(NumParts == 1 && PartEVT != ValueVT);
00397     Val = DAG.getNode(ISD::BITCAST, DL, PartVT, Val);
00398   } else if (NumParts * PartBits < ValueVT.getSizeInBits()) {
00399     // If the parts cover less bits than value has, truncate the value.
00400     assert((PartVT.isInteger() || PartVT == MVT::x86mmx) &&
00401            ValueVT.isInteger() &&
00402            "Unknown mismatch!");
00403     ValueVT = EVT::getIntegerVT(*DAG.getContext(), NumParts * PartBits);
00404     Val = DAG.getNode(ISD::TRUNCATE, DL, ValueVT, Val);
00405     if (PartVT == MVT::x86mmx)
00406       Val = DAG.getNode(ISD::BITCAST, DL, PartVT, Val);
00407   }
00408 
00409   // The value may have changed - recompute ValueVT.
00410   ValueVT = Val.getValueType();
00411   assert(NumParts * PartBits == ValueVT.getSizeInBits() &&
00412          "Failed to tile the value with PartVT!");
00413 
00414   if (NumParts == 1) {
00415     if (PartEVT != ValueVT)
00416       diagnosePossiblyInvalidConstraint(*DAG.getContext(), V,
00417                                         "scalar-to-vector conversion failed");
00418 
00419     Parts[0] = Val;
00420     return;
00421   }
00422 
00423   // Expand the value into multiple parts.
00424   if (NumParts & (NumParts - 1)) {
00425     // The number of parts is not a power of 2.  Split off and copy the tail.
00426     assert(PartVT.isInteger() && ValueVT.isInteger() &&
00427            "Do not know what to expand to!");
00428     unsigned RoundParts = 1 << Log2_32(NumParts);
00429     unsigned RoundBits = RoundParts * PartBits;
00430     unsigned OddParts = NumParts - RoundParts;
00431     SDValue OddVal = DAG.getNode(ISD::SRL, DL, ValueVT, Val,
00432                                  DAG.getIntPtrConstant(RoundBits, DL));
00433     getCopyToParts(DAG, DL, OddVal, Parts + RoundParts, OddParts, PartVT, V);
00434 
00435     if (TLI.isBigEndian())
00436       // The odd parts were reversed by getCopyToParts - unreverse them.
00437       std::reverse(Parts + RoundParts, Parts + NumParts);
00438 
00439     NumParts = RoundParts;
00440     ValueVT = EVT::getIntegerVT(*DAG.getContext(), NumParts * PartBits);
00441     Val = DAG.getNode(ISD::TRUNCATE, DL, ValueVT, Val);
00442   }
00443 
00444   // The number of parts is a power of 2.  Repeatedly bisect the value using
00445   // EXTRACT_ELEMENT.
00446   Parts[0] = DAG.getNode(ISD::BITCAST, DL,
00447                          EVT::getIntegerVT(*DAG.getContext(),
00448                                            ValueVT.getSizeInBits()),
00449                          Val);
00450 
00451   for (unsigned StepSize = NumParts; StepSize > 1; StepSize /= 2) {
00452     for (unsigned i = 0; i < NumParts; i += StepSize) {
00453       unsigned ThisBits = StepSize * PartBits / 2;
00454       EVT ThisVT = EVT::getIntegerVT(*DAG.getContext(), ThisBits);
00455       SDValue &Part0 = Parts[i];
00456       SDValue &Part1 = Parts[i+StepSize/2];
00457 
00458       Part1 = DAG.getNode(ISD::EXTRACT_ELEMENT, DL,
00459                           ThisVT, Part0, DAG.getIntPtrConstant(1, DL));
00460       Part0 = DAG.getNode(ISD::EXTRACT_ELEMENT, DL,
00461                           ThisVT, Part0, DAG.getIntPtrConstant(0, DL));
00462 
00463       if (ThisBits == PartBits && ThisVT != PartVT) {
00464         Part0 = DAG.getNode(ISD::BITCAST, DL, PartVT, Part0);
00465         Part1 = DAG.getNode(ISD::BITCAST, DL, PartVT, Part1);
00466       }
00467     }
00468   }
00469 
00470   if (TLI.isBigEndian())
00471     std::reverse(Parts, Parts + OrigNumParts);
00472 }
00473 
00474 
00475 /// getCopyToPartsVector - Create a series of nodes that contain the specified
00476 /// value split into legal parts.
00477 static void getCopyToPartsVector(SelectionDAG &DAG, SDLoc DL,
00478                                  SDValue Val, SDValue *Parts, unsigned NumParts,
00479                                  MVT PartVT, const Value *V) {
00480   EVT ValueVT = Val.getValueType();
00481   assert(ValueVT.isVector() && "Not a vector");
00482   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
00483 
00484   if (NumParts == 1) {
00485     EVT PartEVT = PartVT;
00486     if (PartEVT == ValueVT) {
00487       // Nothing to do.
00488     } else if (PartVT.getSizeInBits() == ValueVT.getSizeInBits()) {
00489       // Bitconvert vector->vector case.
00490       Val = DAG.getNode(ISD::BITCAST, DL, PartVT, Val);
00491     } else if (PartVT.isVector() &&
00492                PartEVT.getVectorElementType() == ValueVT.getVectorElementType() &&
00493                PartEVT.getVectorNumElements() > ValueVT.getVectorNumElements()) {
00494       EVT ElementVT = PartVT.getVectorElementType();
00495       // Vector widening case, e.g. <2 x float> -> <4 x float>.  Shuffle in
00496       // undef elements.
00497       SmallVector<SDValue, 16> Ops;
00498       for (unsigned i = 0, e = ValueVT.getVectorNumElements(); i != e; ++i)
00499         Ops.push_back(DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL,
00500                                   ElementVT, Val, DAG.getConstant(i, DL,
00501                                                   TLI.getVectorIdxTy())));
00502 
00503       for (unsigned i = ValueVT.getVectorNumElements(),
00504            e = PartVT.getVectorNumElements(); i != e; ++i)
00505         Ops.push_back(DAG.getUNDEF(ElementVT));
00506 
00507       Val = DAG.getNode(ISD::BUILD_VECTOR, DL, PartVT, Ops);
00508 
00509       // FIXME: Use CONCAT for 2x -> 4x.
00510 
00511       //SDValue UndefElts = DAG.getUNDEF(VectorTy);
00512       //Val = DAG.getNode(ISD::CONCAT_VECTORS, DL, PartVT, Val, UndefElts);
00513     } else if (PartVT.isVector() &&
00514                PartEVT.getVectorElementType().bitsGE(
00515                  ValueVT.getVectorElementType()) &&
00516                PartEVT.getVectorNumElements() == ValueVT.getVectorNumElements()) {
00517 
00518       // Promoted vector extract
00519       bool Smaller = PartEVT.bitsLE(ValueVT);
00520       Val = DAG.getNode((Smaller ? ISD::TRUNCATE : ISD::ANY_EXTEND),
00521                         DL, PartVT, Val);
00522     } else{
00523       // Vector -> scalar conversion.
00524       assert(ValueVT.getVectorNumElements() == 1 &&
00525              "Only trivial vector-to-scalar conversions should get here!");
00526       Val = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL,
00527                         PartVT, Val,
00528                         DAG.getConstant(0, DL, TLI.getVectorIdxTy()));
00529 
00530       bool Smaller = ValueVT.bitsLE(PartVT);
00531       Val = DAG.getNode((Smaller ? ISD::TRUNCATE : ISD::ANY_EXTEND),
00532                          DL, PartVT, Val);
00533     }
00534 
00535     Parts[0] = Val;
00536     return;
00537   }
00538 
00539   // Handle a multi-element vector.
00540   EVT IntermediateVT;
00541   MVT RegisterVT;
00542   unsigned NumIntermediates;
00543   unsigned NumRegs = TLI.getVectorTypeBreakdown(*DAG.getContext(), ValueVT,
00544                                                 IntermediateVT,
00545                                                 NumIntermediates, RegisterVT);
00546   unsigned NumElements = ValueVT.getVectorNumElements();
00547 
00548   assert(NumRegs == NumParts && "Part count doesn't match vector breakdown!");
00549   NumParts = NumRegs; // Silence a compiler warning.
00550   assert(RegisterVT == PartVT && "Part type doesn't match vector breakdown!");
00551 
00552   // Split the vector into intermediate operands.
00553   SmallVector<SDValue, 8> Ops(NumIntermediates);
00554   for (unsigned i = 0; i != NumIntermediates; ++i) {
00555     if (IntermediateVT.isVector())
00556       Ops[i] = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL,
00557                            IntermediateVT, Val,
00558                    DAG.getConstant(i * (NumElements / NumIntermediates), DL,
00559                                    TLI.getVectorIdxTy()));
00560     else
00561       Ops[i] = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL,
00562                            IntermediateVT, Val,
00563                            DAG.getConstant(i, DL, TLI.getVectorIdxTy()));
00564   }
00565 
00566   // Split the intermediate operands into legal parts.
00567   if (NumParts == NumIntermediates) {
00568     // If the register was not expanded, promote or copy the value,
00569     // as appropriate.
00570     for (unsigned i = 0; i != NumParts; ++i)
00571       getCopyToParts(DAG, DL, Ops[i], &Parts[i], 1, PartVT, V);
00572   } else if (NumParts > 0) {
00573     // If the intermediate type was expanded, split each the value into
00574     // legal parts.
00575     assert(NumIntermediates != 0 && "division by zero");
00576     assert(NumParts % NumIntermediates == 0 &&
00577            "Must expand into a divisible number of parts!");
00578     unsigned Factor = NumParts / NumIntermediates;
00579     for (unsigned i = 0; i != NumIntermediates; ++i)
00580       getCopyToParts(DAG, DL, Ops[i], &Parts[i*Factor], Factor, PartVT, V);
00581   }
00582 }
00583 
00584 RegsForValue::RegsForValue() {}
00585 
00586 RegsForValue::RegsForValue(const SmallVector<unsigned, 4> &regs, MVT regvt,
00587                            EVT valuevt)
00588     : ValueVTs(1, valuevt), RegVTs(1, regvt), Regs(regs) {}
00589 
00590 RegsForValue::RegsForValue(LLVMContext &Context, const TargetLowering &tli,
00591                            unsigned Reg, Type *Ty) {
00592   ComputeValueVTs(tli, Ty, ValueVTs);
00593 
00594   for (unsigned Value = 0, e = ValueVTs.size(); Value != e; ++Value) {
00595     EVT ValueVT = ValueVTs[Value];
00596     unsigned NumRegs = tli.getNumRegisters(Context, ValueVT);
00597     MVT RegisterVT = tli.getRegisterType(Context, ValueVT);
00598     for (unsigned i = 0; i != NumRegs; ++i)
00599       Regs.push_back(Reg + i);
00600     RegVTs.push_back(RegisterVT);
00601     Reg += NumRegs;
00602   }
00603 }
00604 
00605 /// getCopyFromRegs - Emit a series of CopyFromReg nodes that copies from
00606 /// this value and returns the result as a ValueVT value.  This uses
00607 /// Chain/Flag as the input and updates them for the output Chain/Flag.
00608 /// If the Flag pointer is NULL, no flag is used.
00609 SDValue RegsForValue::getCopyFromRegs(SelectionDAG &DAG,
00610                                       FunctionLoweringInfo &FuncInfo,
00611                                       SDLoc dl,
00612                                       SDValue &Chain, SDValue *Flag,
00613                                       const Value *V) const {
00614   // A Value with type {} or [0 x %t] needs no registers.
00615   if (ValueVTs.empty())
00616     return SDValue();
00617 
00618   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
00619 
00620   // Assemble the legal parts into the final values.
00621   SmallVector<SDValue, 4> Values(ValueVTs.size());
00622   SmallVector<SDValue, 8> Parts;
00623   for (unsigned Value = 0, Part = 0, e = ValueVTs.size(); Value != e; ++Value) {
00624     // Copy the legal parts from the registers.
00625     EVT ValueVT = ValueVTs[Value];
00626     unsigned NumRegs = TLI.getNumRegisters(*DAG.getContext(), ValueVT);
00627     MVT RegisterVT = RegVTs[Value];
00628 
00629     Parts.resize(NumRegs);
00630     for (unsigned i = 0; i != NumRegs; ++i) {
00631       SDValue P;
00632       if (!Flag) {
00633         P = DAG.getCopyFromReg(Chain, dl, Regs[Part+i], RegisterVT);
00634       } else {
00635         P = DAG.getCopyFromReg(Chain, dl, Regs[Part+i], RegisterVT, *Flag);
00636         *Flag = P.getValue(2);
00637       }
00638 
00639       Chain = P.getValue(1);
00640       Parts[i] = P;
00641 
00642       // If the source register was virtual and if we know something about it,
00643       // add an assert node.
00644       if (!TargetRegisterInfo::isVirtualRegister(Regs[Part+i]) ||
00645           !RegisterVT.isInteger() || RegisterVT.isVector())
00646         continue;
00647 
00648       const FunctionLoweringInfo::LiveOutInfo *LOI =
00649         FuncInfo.GetLiveOutRegInfo(Regs[Part+i]);
00650       if (!LOI)
00651         continue;
00652 
00653       unsigned RegSize = RegisterVT.getSizeInBits();
00654       unsigned NumSignBits = LOI->NumSignBits;
00655       unsigned NumZeroBits = LOI->KnownZero.countLeadingOnes();
00656 
00657       if (NumZeroBits == RegSize) {
00658         // The current value is a zero.
00659         // Explicitly express that as it would be easier for
00660         // optimizations to kick in.
00661         Parts[i] = DAG.getConstant(0, dl, RegisterVT);
00662         continue;
00663       }
00664 
00665       // FIXME: We capture more information than the dag can represent.  For
00666       // now, just use the tightest assertzext/assertsext possible.
00667       bool isSExt = true;
00668       EVT FromVT(MVT::Other);
00669       if (NumSignBits == RegSize)
00670         isSExt = true, FromVT = MVT::i1;   // ASSERT SEXT 1
00671       else if (NumZeroBits >= RegSize-1)
00672         isSExt = false, FromVT = MVT::i1;  // ASSERT ZEXT 1
00673       else if (NumSignBits > RegSize-8)
00674         isSExt = true, FromVT = MVT::i8;   // ASSERT SEXT 8
00675       else if (NumZeroBits >= RegSize-8)
00676         isSExt = false, FromVT = MVT::i8;  // ASSERT ZEXT 8
00677       else if (NumSignBits > RegSize-16)
00678         isSExt = true, FromVT = MVT::i16;  // ASSERT SEXT 16
00679       else if (NumZeroBits >= RegSize-16)
00680         isSExt = false, FromVT = MVT::i16; // ASSERT ZEXT 16
00681       else if (NumSignBits > RegSize-32)
00682         isSExt = true, FromVT = MVT::i32;  // ASSERT SEXT 32
00683       else if (NumZeroBits >= RegSize-32)
00684         isSExt = false, FromVT = MVT::i32; // ASSERT ZEXT 32
00685       else
00686         continue;
00687 
00688       // Add an assertion node.
00689       assert(FromVT != MVT::Other);
00690       Parts[i] = DAG.getNode(isSExt ? ISD::AssertSext : ISD::AssertZext, dl,
00691                              RegisterVT, P, DAG.getValueType(FromVT));
00692     }
00693 
00694     Values[Value] = getCopyFromParts(DAG, dl, Parts.begin(),
00695                                      NumRegs, RegisterVT, ValueVT, V);
00696     Part += NumRegs;
00697     Parts.clear();
00698   }
00699 
00700   return DAG.getNode(ISD::MERGE_VALUES, dl, DAG.getVTList(ValueVTs), Values);
00701 }
00702 
00703 /// getCopyToRegs - Emit a series of CopyToReg nodes that copies the
00704 /// specified value into the registers specified by this object.  This uses
00705 /// Chain/Flag as the input and updates them for the output Chain/Flag.
00706 /// If the Flag pointer is NULL, no flag is used.
00707 void RegsForValue::getCopyToRegs(SDValue Val, SelectionDAG &DAG, SDLoc dl,
00708                                  SDValue &Chain, SDValue *Flag, const Value *V,
00709                                  ISD::NodeType PreferredExtendType) const {
00710   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
00711   ISD::NodeType ExtendKind = PreferredExtendType;
00712 
00713   // Get the list of the values's legal parts.
00714   unsigned NumRegs = Regs.size();
00715   SmallVector<SDValue, 8> Parts(NumRegs);
00716   for (unsigned Value = 0, Part = 0, e = ValueVTs.size(); Value != e; ++Value) {
00717     EVT ValueVT = ValueVTs[Value];
00718     unsigned NumParts = TLI.getNumRegisters(*DAG.getContext(), ValueVT);
00719     MVT RegisterVT = RegVTs[Value];
00720 
00721     if (ExtendKind == ISD::ANY_EXTEND && TLI.isZExtFree(Val, RegisterVT))
00722       ExtendKind = ISD::ZERO_EXTEND;
00723 
00724     getCopyToParts(DAG, dl, Val.getValue(Val.getResNo() + Value),
00725                    &Parts[Part], NumParts, RegisterVT, V, ExtendKind);
00726     Part += NumParts;
00727   }
00728 
00729   // Copy the parts into the registers.
00730   SmallVector<SDValue, 8> Chains(NumRegs);
00731   for (unsigned i = 0; i != NumRegs; ++i) {
00732     SDValue Part;
00733     if (!Flag) {
00734       Part = DAG.getCopyToReg(Chain, dl, Regs[i], Parts[i]);
00735     } else {
00736       Part = DAG.getCopyToReg(Chain, dl, Regs[i], Parts[i], *Flag);
00737       *Flag = Part.getValue(1);
00738     }
00739 
00740     Chains[i] = Part.getValue(0);
00741   }
00742 
00743   if (NumRegs == 1 || Flag)
00744     // If NumRegs > 1 && Flag is used then the use of the last CopyToReg is
00745     // flagged to it. That is the CopyToReg nodes and the user are considered
00746     // a single scheduling unit. If we create a TokenFactor and return it as
00747     // chain, then the TokenFactor is both a predecessor (operand) of the
00748     // user as well as a successor (the TF operands are flagged to the user).
00749     // c1, f1 = CopyToReg
00750     // c2, f2 = CopyToReg
00751     // c3     = TokenFactor c1, c2
00752     // ...
00753     //        = op c3, ..., f2
00754     Chain = Chains[NumRegs-1];
00755   else
00756     Chain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Chains);
00757 }
00758 
00759 /// AddInlineAsmOperands - Add this value to the specified inlineasm node
00760 /// operand list.  This adds the code marker and includes the number of
00761 /// values added into it.
00762 void RegsForValue::AddInlineAsmOperands(unsigned Code, bool HasMatching,
00763                                         unsigned MatchingIdx, SDLoc dl,
00764                                         SelectionDAG &DAG,
00765                                         std::vector<SDValue> &Ops) const {
00766   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
00767 
00768   unsigned Flag = InlineAsm::getFlagWord(Code, Regs.size());
00769   if (HasMatching)
00770     Flag = InlineAsm::getFlagWordForMatchingOp(Flag, MatchingIdx);
00771   else if (!Regs.empty() &&
00772            TargetRegisterInfo::isVirtualRegister(Regs.front())) {
00773     // Put the register class of the virtual registers in the flag word.  That
00774     // way, later passes can recompute register class constraints for inline
00775     // assembly as well as normal instructions.
00776     // Don't do this for tied operands that can use the regclass information
00777     // from the def.
00778     const MachineRegisterInfo &MRI = DAG.getMachineFunction().getRegInfo();
00779     const TargetRegisterClass *RC = MRI.getRegClass(Regs.front());
00780     Flag = InlineAsm::getFlagWordForRegClass(Flag, RC->getID());
00781   }
00782 
00783   SDValue Res = DAG.getTargetConstant(Flag, dl, MVT::i32);
00784   Ops.push_back(Res);
00785 
00786   unsigned SP = TLI.getStackPointerRegisterToSaveRestore();
00787   for (unsigned Value = 0, Reg = 0, e = ValueVTs.size(); Value != e; ++Value) {
00788     unsigned NumRegs = TLI.getNumRegisters(*DAG.getContext(), ValueVTs[Value]);
00789     MVT RegisterVT = RegVTs[Value];
00790     for (unsigned i = 0; i != NumRegs; ++i) {
00791       assert(Reg < Regs.size() && "Mismatch in # registers expected");
00792       unsigned TheReg = Regs[Reg++];
00793       Ops.push_back(DAG.getRegister(TheReg, RegisterVT));
00794 
00795       if (TheReg == SP && Code == InlineAsm::Kind_Clobber) {
00796         // If we clobbered the stack pointer, MFI should know about it.
00797         assert(DAG.getMachineFunction().getFrameInfo()->
00798             hasInlineAsmWithSPAdjust());
00799       }
00800     }
00801   }
00802 }
00803 
00804 void SelectionDAGBuilder::init(GCFunctionInfo *gfi, AliasAnalysis &aa,
00805                                const TargetLibraryInfo *li) {
00806   AA = &aa;
00807   GFI = gfi;
00808   LibInfo = li;
00809   DL = DAG.getTarget().getDataLayout();
00810   Context = DAG.getContext();
00811   LPadToCallSiteMap.clear();
00812 }
00813 
00814 /// clear - Clear out the current SelectionDAG and the associated
00815 /// state and prepare this SelectionDAGBuilder object to be used
00816 /// for a new block. This doesn't clear out information about
00817 /// additional blocks that are needed to complete switch lowering
00818 /// or PHI node updating; that information is cleared out as it is
00819 /// consumed.
00820 void SelectionDAGBuilder::clear() {
00821   NodeMap.clear();
00822   UnusedArgNodeMap.clear();
00823   PendingLoads.clear();
00824   PendingExports.clear();
00825   CurInst = nullptr;
00826   HasTailCall = false;
00827   SDNodeOrder = LowestSDNodeOrder;
00828   StatepointLowering.clear();
00829 }
00830 
00831 /// clearDanglingDebugInfo - Clear the dangling debug information
00832 /// map. This function is separated from the clear so that debug
00833 /// information that is dangling in a basic block can be properly
00834 /// resolved in a different basic block. This allows the
00835 /// SelectionDAG to resolve dangling debug information attached
00836 /// to PHI nodes.
00837 void SelectionDAGBuilder::clearDanglingDebugInfo() {
00838   DanglingDebugInfoMap.clear();
00839 }
00840 
00841 /// getRoot - Return the current virtual root of the Selection DAG,
00842 /// flushing any PendingLoad items. This must be done before emitting
00843 /// a store or any other node that may need to be ordered after any
00844 /// prior load instructions.
00845 ///
00846 SDValue SelectionDAGBuilder::getRoot() {
00847   if (PendingLoads.empty())
00848     return DAG.getRoot();
00849 
00850   if (PendingLoads.size() == 1) {
00851     SDValue Root = PendingLoads[0];
00852     DAG.setRoot(Root);
00853     PendingLoads.clear();
00854     return Root;
00855   }
00856 
00857   // Otherwise, we have to make a token factor node.
00858   SDValue Root = DAG.getNode(ISD::TokenFactor, getCurSDLoc(), MVT::Other,
00859                              PendingLoads);
00860   PendingLoads.clear();
00861   DAG.setRoot(Root);
00862   return Root;
00863 }
00864 
00865 /// getControlRoot - Similar to getRoot, but instead of flushing all the
00866 /// PendingLoad items, flush all the PendingExports items. It is necessary
00867 /// to do this before emitting a terminator instruction.
00868 ///
00869 SDValue SelectionDAGBuilder::getControlRoot() {
00870   SDValue Root = DAG.getRoot();
00871 
00872   if (PendingExports.empty())
00873     return Root;
00874 
00875   // Turn all of the CopyToReg chains into one factored node.
00876   if (Root.getOpcode() != ISD::EntryToken) {
00877     unsigned i = 0, e = PendingExports.size();
00878     for (; i != e; ++i) {
00879       assert(PendingExports[i].getNode()->getNumOperands() > 1);
00880       if (PendingExports[i].getNode()->getOperand(0) == Root)
00881         break;  // Don't add the root if we already indirectly depend on it.
00882     }
00883 
00884     if (i == e)
00885       PendingExports.push_back(Root);
00886   }
00887 
00888   Root = DAG.getNode(ISD::TokenFactor, getCurSDLoc(), MVT::Other,
00889                      PendingExports);
00890   PendingExports.clear();
00891   DAG.setRoot(Root);
00892   return Root;
00893 }
00894 
00895 void SelectionDAGBuilder::visit(const Instruction &I) {
00896   // Set up outgoing PHI node register values before emitting the terminator.
00897   if (isa<TerminatorInst>(&I))
00898     HandlePHINodesInSuccessorBlocks(I.getParent());
00899 
00900   ++SDNodeOrder;
00901 
00902   CurInst = &I;
00903 
00904   visit(I.getOpcode(), I);
00905 
00906   if (!isa<TerminatorInst>(&I) && !HasTailCall)
00907     CopyToExportRegsIfNeeded(&I);
00908 
00909   CurInst = nullptr;
00910 }
00911 
00912 void SelectionDAGBuilder::visitPHI(const PHINode &) {
00913   llvm_unreachable("SelectionDAGBuilder shouldn't visit PHI nodes!");
00914 }
00915 
00916 void SelectionDAGBuilder::visit(unsigned Opcode, const User &I) {
00917   // Note: this doesn't use InstVisitor, because it has to work with
00918   // ConstantExpr's in addition to instructions.
00919   switch (Opcode) {
00920   default: llvm_unreachable("Unknown instruction type encountered!");
00921     // Build the switch statement using the Instruction.def file.
00922 #define HANDLE_INST(NUM, OPCODE, CLASS) \
00923     case Instruction::OPCODE: visit##OPCODE((const CLASS&)I); break;
00924 #include "llvm/IR/Instruction.def"
00925   }
00926 }
00927 
00928 // resolveDanglingDebugInfo - if we saw an earlier dbg_value referring to V,
00929 // generate the debug data structures now that we've seen its definition.
00930 void SelectionDAGBuilder::resolveDanglingDebugInfo(const Value *V,
00931                                                    SDValue Val) {
00932   DanglingDebugInfo &DDI = DanglingDebugInfoMap[V];
00933   if (DDI.getDI()) {
00934     const DbgValueInst *DI = DDI.getDI();
00935     DebugLoc dl = DDI.getdl();
00936     unsigned DbgSDNodeOrder = DDI.getSDNodeOrder();
00937     DILocalVariable *Variable = DI->getVariable();
00938     DIExpression *Expr = DI->getExpression();
00939     assert(Variable->isValidLocationForIntrinsic(dl) &&
00940            "Expected inlined-at fields to agree");
00941     uint64_t Offset = DI->getOffset();
00942     // A dbg.value for an alloca is always indirect.
00943     bool IsIndirect = isa<AllocaInst>(V) || Offset != 0;
00944     SDDbgValue *SDV;
00945     if (Val.getNode()) {
00946       if (!EmitFuncArgumentDbgValue(V, Variable, Expr, dl, Offset, IsIndirect,
00947                                     Val)) {
00948         SDV = DAG.getDbgValue(Variable, Expr, Val.getNode(), Val.getResNo(),
00949                               IsIndirect, Offset, dl, DbgSDNodeOrder);
00950         DAG.AddDbgValue(SDV, Val.getNode(), false);
00951       }
00952     } else
00953       DEBUG(dbgs() << "Dropping debug info for " << *DI << "\n");
00954     DanglingDebugInfoMap[V] = DanglingDebugInfo();
00955   }
00956 }
00957 
00958 /// getCopyFromRegs - If there was virtual register allocated for the value V
00959 /// emit CopyFromReg of the specified type Ty. Return empty SDValue() otherwise.
00960 SDValue SelectionDAGBuilder::getCopyFromRegs(const Value *V, Type *Ty) {
00961   DenseMap<const Value *, unsigned>::iterator It = FuncInfo.ValueMap.find(V);
00962   SDValue Result;
00963 
00964   if (It != FuncInfo.ValueMap.end()) {
00965     unsigned InReg = It->second;
00966     RegsForValue RFV(*DAG.getContext(), DAG.getTargetLoweringInfo(), InReg,
00967                      Ty);
00968     SDValue Chain = DAG.getEntryNode();
00969     Result = RFV.getCopyFromRegs(DAG, FuncInfo, getCurSDLoc(), Chain, nullptr, V);
00970     resolveDanglingDebugInfo(V, Result);
00971   }
00972 
00973   return Result;
00974 }
00975 
00976 /// getValue - Return an SDValue for the given Value.
00977 SDValue SelectionDAGBuilder::getValue(const Value *V) {
00978   // If we already have an SDValue for this value, use it. It's important
00979   // to do this first, so that we don't create a CopyFromReg if we already
00980   // have a regular SDValue.
00981   SDValue &N = NodeMap[V];
00982   if (N.getNode()) return N;
00983 
00984   // If there's a virtual register allocated and initialized for this
00985   // value, use it.
00986   SDValue copyFromReg = getCopyFromRegs(V, V->getType());
00987   if (copyFromReg.getNode()) {
00988     return copyFromReg;
00989   }
00990 
00991   // Otherwise create a new SDValue and remember it.
00992   SDValue Val = getValueImpl(V);
00993   NodeMap[V] = Val;
00994   resolveDanglingDebugInfo(V, Val);
00995   return Val;
00996 }
00997 
00998 // Return true if SDValue exists for the given Value
00999 bool SelectionDAGBuilder::findValue(const Value *V) const {
01000   return (NodeMap.find(V) != NodeMap.end()) ||
01001     (FuncInfo.ValueMap.find(V) != FuncInfo.ValueMap.end());
01002 }
01003 
01004 /// getNonRegisterValue - Return an SDValue for the given Value, but
01005 /// don't look in FuncInfo.ValueMap for a virtual register.
01006 SDValue SelectionDAGBuilder::getNonRegisterValue(const Value *V) {
01007   // If we already have an SDValue for this value, use it.
01008   SDValue &N = NodeMap[V];
01009   if (N.getNode()) {
01010     if (isa<ConstantSDNode>(N) || isa<ConstantFPSDNode>(N)) {
01011       // Remove the debug location from the node as the node is about to be used
01012       // in a location which may differ from the original debug location.  This
01013       // is relevant to Constant and ConstantFP nodes because they can appear
01014       // as constant expressions inside PHI nodes.
01015       N->setDebugLoc(DebugLoc());
01016     }
01017     return N;
01018   }
01019 
01020   // Otherwise create a new SDValue and remember it.
01021   SDValue Val = getValueImpl(V);
01022   NodeMap[V] = Val;
01023   resolveDanglingDebugInfo(V, Val);
01024   return Val;
01025 }
01026 
01027 /// getValueImpl - Helper function for getValue and getNonRegisterValue.
01028 /// Create an SDValue for the given value.
01029 SDValue SelectionDAGBuilder::getValueImpl(const Value *V) {
01030   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
01031 
01032   if (const Constant *C = dyn_cast<Constant>(V)) {
01033     EVT VT = TLI.getValueType(V->getType(), true);
01034 
01035     if (const ConstantInt *CI = dyn_cast<ConstantInt>(C))
01036       return DAG.getConstant(*CI, getCurSDLoc(), VT);
01037 
01038     if (const GlobalValue *GV = dyn_cast<GlobalValue>(C))
01039       return DAG.getGlobalAddress(GV, getCurSDLoc(), VT);
01040 
01041     if (isa<ConstantPointerNull>(C)) {
01042       unsigned AS = V->getType()->getPointerAddressSpace();
01043       return DAG.getConstant(0, getCurSDLoc(), TLI.getPointerTy(AS));
01044     }
01045 
01046     if (const ConstantFP *CFP = dyn_cast<ConstantFP>(C))
01047       return DAG.getConstantFP(*CFP, getCurSDLoc(), VT);
01048 
01049     if (isa<UndefValue>(C) && !V->getType()->isAggregateType())
01050       return DAG.getUNDEF(VT);
01051 
01052     if (const ConstantExpr *CE = dyn_cast<ConstantExpr>(C)) {
01053       visit(CE->getOpcode(), *CE);
01054       SDValue N1 = NodeMap[V];
01055       assert(N1.getNode() && "visit didn't populate the NodeMap!");
01056       return N1;
01057     }
01058 
01059     if (isa<ConstantStruct>(C) || isa<ConstantArray>(C)) {
01060       SmallVector<SDValue, 4> Constants;
01061       for (User::const_op_iterator OI = C->op_begin(), OE = C->op_end();
01062            OI != OE; ++OI) {
01063         SDNode *Val = getValue(*OI).getNode();
01064         // If the operand is an empty aggregate, there are no values.
01065         if (!Val) continue;
01066         // Add each leaf value from the operand to the Constants list
01067         // to form a flattened list of all the values.
01068         for (unsigned i = 0, e = Val->getNumValues(); i != e; ++i)
01069           Constants.push_back(SDValue(Val, i));
01070       }
01071 
01072       return DAG.getMergeValues(Constants, getCurSDLoc());
01073     }
01074 
01075     if (const ConstantDataSequential *CDS =
01076           dyn_cast<ConstantDataSequential>(C)) {
01077       SmallVector<SDValue, 4> Ops;
01078       for (unsigned i = 0, e = CDS->getNumElements(); i != e; ++i) {
01079         SDNode *Val = getValue(CDS->getElementAsConstant(i)).getNode();
01080         // Add each leaf value from the operand to the Constants list
01081         // to form a flattened list of all the values.
01082         for (unsigned i = 0, e = Val->getNumValues(); i != e; ++i)
01083           Ops.push_back(SDValue(Val, i));
01084       }
01085 
01086       if (isa<ArrayType>(CDS->getType()))
01087         return DAG.getMergeValues(Ops, getCurSDLoc());
01088       return NodeMap[V] = DAG.getNode(ISD::BUILD_VECTOR, getCurSDLoc(),
01089                                       VT, Ops);
01090     }
01091 
01092     if (C->getType()->isStructTy() || C->getType()->isArrayTy()) {
01093       assert((isa<ConstantAggregateZero>(C) || isa<UndefValue>(C)) &&
01094              "Unknown struct or array constant!");
01095 
01096       SmallVector<EVT, 4> ValueVTs;
01097       ComputeValueVTs(TLI, C->getType(), ValueVTs);
01098       unsigned NumElts = ValueVTs.size();
01099       if (NumElts == 0)
01100         return SDValue(); // empty struct
01101       SmallVector<SDValue, 4> Constants(NumElts);
01102       for (unsigned i = 0; i != NumElts; ++i) {
01103         EVT EltVT = ValueVTs[i];
01104         if (isa<UndefValue>(C))
01105           Constants[i] = DAG.getUNDEF(EltVT);
01106         else if (EltVT.isFloatingPoint())
01107           Constants[i] = DAG.getConstantFP(0, getCurSDLoc(), EltVT);
01108         else
01109           Constants[i] = DAG.getConstant(0, getCurSDLoc(), EltVT);
01110       }
01111 
01112       return DAG.getMergeValues(Constants, getCurSDLoc());
01113     }
01114 
01115     if (const BlockAddress *BA = dyn_cast<BlockAddress>(C))
01116       return DAG.getBlockAddress(BA, VT);
01117 
01118     VectorType *VecTy = cast<VectorType>(V->getType());
01119     unsigned NumElements = VecTy->getNumElements();
01120 
01121     // Now that we know the number and type of the elements, get that number of
01122     // elements into the Ops array based on what kind of constant it is.
01123     SmallVector<SDValue, 16> Ops;
01124     if (const ConstantVector *CV = dyn_cast<ConstantVector>(C)) {
01125       for (unsigned i = 0; i != NumElements; ++i)
01126         Ops.push_back(getValue(CV->getOperand(i)));
01127     } else {
01128       assert(isa<ConstantAggregateZero>(C) && "Unknown vector constant!");
01129       EVT EltVT = TLI.getValueType(VecTy->getElementType());
01130 
01131       SDValue Op;
01132       if (EltVT.isFloatingPoint())
01133         Op = DAG.getConstantFP(0, getCurSDLoc(), EltVT);
01134       else
01135         Op = DAG.getConstant(0, getCurSDLoc(), EltVT);
01136       Ops.assign(NumElements, Op);
01137     }
01138 
01139     // Create a BUILD_VECTOR node.
01140     return NodeMap[V] = DAG.getNode(ISD::BUILD_VECTOR, getCurSDLoc(), VT, Ops);
01141   }
01142 
01143   // If this is a static alloca, generate it as the frameindex instead of
01144   // computation.
01145   if (const AllocaInst *AI = dyn_cast<AllocaInst>(V)) {
01146     DenseMap<const AllocaInst*, int>::iterator SI =
01147       FuncInfo.StaticAllocaMap.find(AI);
01148     if (SI != FuncInfo.StaticAllocaMap.end())
01149       return DAG.getFrameIndex(SI->second, TLI.getPointerTy());
01150   }
01151 
01152   // If this is an instruction which fast-isel has deferred, select it now.
01153   if (const Instruction *Inst = dyn_cast<Instruction>(V)) {
01154     unsigned InReg = FuncInfo.InitializeRegForValue(Inst);
01155     RegsForValue RFV(*DAG.getContext(), TLI, InReg, Inst->getType());
01156     SDValue Chain = DAG.getEntryNode();
01157     return RFV.getCopyFromRegs(DAG, FuncInfo, getCurSDLoc(), Chain, nullptr, V);
01158   }
01159 
01160   llvm_unreachable("Can't get register for value!");
01161 }
01162 
01163 void SelectionDAGBuilder::visitRet(const ReturnInst &I) {
01164   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
01165   SDValue Chain = getControlRoot();
01166   SmallVector<ISD::OutputArg, 8> Outs;
01167   SmallVector<SDValue, 8> OutVals;
01168 
01169   if (!FuncInfo.CanLowerReturn) {
01170     unsigned DemoteReg = FuncInfo.DemoteRegister;
01171     const Function *F = I.getParent()->getParent();
01172 
01173     // Emit a store of the return value through the virtual register.
01174     // Leave Outs empty so that LowerReturn won't try to load return
01175     // registers the usual way.
01176     SmallVector<EVT, 1> PtrValueVTs;
01177     ComputeValueVTs(TLI, PointerType::getUnqual(F->getReturnType()),
01178                     PtrValueVTs);
01179 
01180     SDValue RetPtr = DAG.getRegister(DemoteReg, PtrValueVTs[0]);
01181     SDValue RetOp = getValue(I.getOperand(0));
01182 
01183     SmallVector<EVT, 4> ValueVTs;
01184     SmallVector<uint64_t, 4> Offsets;
01185     ComputeValueVTs(TLI, I.getOperand(0)->getType(), ValueVTs, &Offsets);
01186     unsigned NumValues = ValueVTs.size();
01187 
01188     SmallVector<SDValue, 4> Chains(NumValues);
01189     for (unsigned i = 0; i != NumValues; ++i) {
01190       SDValue Add = DAG.getNode(ISD::ADD, getCurSDLoc(),
01191                                 RetPtr.getValueType(), RetPtr,
01192                                 DAG.getIntPtrConstant(Offsets[i],
01193                                                       getCurSDLoc()));
01194       Chains[i] =
01195         DAG.getStore(Chain, getCurSDLoc(),
01196                      SDValue(RetOp.getNode(), RetOp.getResNo() + i),
01197                      // FIXME: better loc info would be nice.
01198                      Add, MachinePointerInfo(), false, false, 0);
01199     }
01200 
01201     Chain = DAG.getNode(ISD::TokenFactor, getCurSDLoc(),
01202                         MVT::Other, Chains);
01203   } else if (I.getNumOperands() != 0) {
01204     SmallVector<EVT, 4> ValueVTs;
01205     ComputeValueVTs(TLI, I.getOperand(0)->getType(), ValueVTs);
01206     unsigned NumValues = ValueVTs.size();
01207     if (NumValues) {
01208       SDValue RetOp = getValue(I.getOperand(0));
01209 
01210       const Function *F = I.getParent()->getParent();
01211 
01212       ISD::NodeType ExtendKind = ISD::ANY_EXTEND;
01213       if (F->getAttributes().hasAttribute(AttributeSet::ReturnIndex,
01214                                           Attribute::SExt))
01215         ExtendKind = ISD::SIGN_EXTEND;
01216       else if (F->getAttributes().hasAttribute(AttributeSet::ReturnIndex,
01217                                                Attribute::ZExt))
01218         ExtendKind = ISD::ZERO_EXTEND;
01219 
01220       LLVMContext &Context = F->getContext();
01221       bool RetInReg = F->getAttributes().hasAttribute(AttributeSet::ReturnIndex,
01222                                                       Attribute::InReg);
01223 
01224       for (unsigned j = 0; j != NumValues; ++j) {
01225         EVT VT = ValueVTs[j];
01226 
01227         if (ExtendKind != ISD::ANY_EXTEND && VT.isInteger())
01228           VT = TLI.getTypeForExtArgOrReturn(Context, VT, ExtendKind);
01229 
01230         unsigned NumParts = TLI.getNumRegisters(Context, VT);
01231         MVT PartVT = TLI.getRegisterType(Context, VT);
01232         SmallVector<SDValue, 4> Parts(NumParts);
01233         getCopyToParts(DAG, getCurSDLoc(),
01234                        SDValue(RetOp.getNode(), RetOp.getResNo() + j),
01235                        &Parts[0], NumParts, PartVT, &I, ExtendKind);
01236 
01237         // 'inreg' on function refers to return value
01238         ISD::ArgFlagsTy Flags = ISD::ArgFlagsTy();
01239         if (RetInReg)
01240           Flags.setInReg();
01241 
01242         // Propagate extension type if any
01243         if (ExtendKind == ISD::SIGN_EXTEND)
01244           Flags.setSExt();
01245         else if (ExtendKind == ISD::ZERO_EXTEND)
01246           Flags.setZExt();
01247 
01248         for (unsigned i = 0; i < NumParts; ++i) {
01249           Outs.push_back(ISD::OutputArg(Flags, Parts[i].getValueType(),
01250                                         VT, /*isfixed=*/true, 0, 0));
01251           OutVals.push_back(Parts[i]);
01252         }
01253       }
01254     }
01255   }
01256 
01257   bool isVarArg = DAG.getMachineFunction().getFunction()->isVarArg();
01258   CallingConv::ID CallConv =
01259     DAG.getMachineFunction().getFunction()->getCallingConv();
01260   Chain = DAG.getTargetLoweringInfo().LowerReturn(
01261       Chain, CallConv, isVarArg, Outs, OutVals, getCurSDLoc(), DAG);
01262 
01263   // Verify that the target's LowerReturn behaved as expected.
01264   assert(Chain.getNode() && Chain.getValueType() == MVT::Other &&
01265          "LowerReturn didn't return a valid chain!");
01266 
01267   // Update the DAG with the new chain value resulting from return lowering.
01268   DAG.setRoot(Chain);
01269 }
01270 
01271 /// CopyToExportRegsIfNeeded - If the given value has virtual registers
01272 /// created for it, emit nodes to copy the value into the virtual
01273 /// registers.
01274 void SelectionDAGBuilder::CopyToExportRegsIfNeeded(const Value *V) {
01275   // Skip empty types
01276   if (V->getType()->isEmptyTy())
01277     return;
01278 
01279   DenseMap<const Value *, unsigned>::iterator VMI = FuncInfo.ValueMap.find(V);
01280   if (VMI != FuncInfo.ValueMap.end()) {
01281     assert(!V->use_empty() && "Unused value assigned virtual registers!");
01282     CopyValueToVirtualRegister(V, VMI->second);
01283   }
01284 }
01285 
01286 /// ExportFromCurrentBlock - If this condition isn't known to be exported from
01287 /// the current basic block, add it to ValueMap now so that we'll get a
01288 /// CopyTo/FromReg.
01289 void SelectionDAGBuilder::ExportFromCurrentBlock(const Value *V) {
01290   // No need to export constants.
01291   if (!isa<Instruction>(V) && !isa<Argument>(V)) return;
01292 
01293   // Already exported?
01294   if (FuncInfo.isExportedInst(V)) return;
01295 
01296   unsigned Reg = FuncInfo.InitializeRegForValue(V);
01297   CopyValueToVirtualRegister(V, Reg);
01298 }
01299 
01300 bool SelectionDAGBuilder::isExportableFromCurrentBlock(const Value *V,
01301                                                      const BasicBlock *FromBB) {
01302   // The operands of the setcc have to be in this block.  We don't know
01303   // how to export them from some other block.
01304   if (const Instruction *VI = dyn_cast<Instruction>(V)) {
01305     // Can export from current BB.
01306     if (VI->getParent() == FromBB)
01307       return true;
01308 
01309     // Is already exported, noop.
01310     return FuncInfo.isExportedInst(V);
01311   }
01312 
01313   // If this is an argument, we can export it if the BB is the entry block or
01314   // if it is already exported.
01315   if (isa<Argument>(V)) {
01316     if (FromBB == &FromBB->getParent()->getEntryBlock())
01317       return true;
01318 
01319     // Otherwise, can only export this if it is already exported.
01320     return FuncInfo.isExportedInst(V);
01321   }
01322 
01323   // Otherwise, constants can always be exported.
01324   return true;
01325 }
01326 
01327 /// Return branch probability calculated by BranchProbabilityInfo for IR blocks.
01328 uint32_t SelectionDAGBuilder::getEdgeWeight(const MachineBasicBlock *Src,
01329                                             const MachineBasicBlock *Dst) const {
01330   BranchProbabilityInfo *BPI = FuncInfo.BPI;
01331   if (!BPI)
01332     return 0;
01333   const BasicBlock *SrcBB = Src->getBasicBlock();
01334   const BasicBlock *DstBB = Dst->getBasicBlock();
01335   return BPI->getEdgeWeight(SrcBB, DstBB);
01336 }
01337 
01338 void SelectionDAGBuilder::
01339 addSuccessorWithWeight(MachineBasicBlock *Src, MachineBasicBlock *Dst,
01340                        uint32_t Weight /* = 0 */) {
01341   if (!Weight)
01342     Weight = getEdgeWeight(Src, Dst);
01343   Src->addSuccessor(Dst, Weight);
01344 }
01345 
01346 
01347 static bool InBlock(const Value *V, const BasicBlock *BB) {
01348   if (const Instruction *I = dyn_cast<Instruction>(V))
01349     return I->getParent() == BB;
01350   return true;
01351 }
01352 
01353 /// EmitBranchForMergedCondition - Helper method for FindMergedConditions.
01354 /// This function emits a branch and is used at the leaves of an OR or an
01355 /// AND operator tree.
01356 ///
01357 void
01358 SelectionDAGBuilder::EmitBranchForMergedCondition(const Value *Cond,
01359                                                   MachineBasicBlock *TBB,
01360                                                   MachineBasicBlock *FBB,
01361                                                   MachineBasicBlock *CurBB,
01362                                                   MachineBasicBlock *SwitchBB,
01363                                                   uint32_t TWeight,
01364                                                   uint32_t FWeight) {
01365   const BasicBlock *BB = CurBB->getBasicBlock();
01366 
01367   // If the leaf of the tree is a comparison, merge the condition into
01368   // the caseblock.
01369   if (const CmpInst *BOp = dyn_cast<CmpInst>(Cond)) {
01370     // The operands of the cmp have to be in this block.  We don't know
01371     // how to export them from some other block.  If this is the first block
01372     // of the sequence, no exporting is needed.
01373     if (CurBB == SwitchBB ||
01374         (isExportableFromCurrentBlock(BOp->getOperand(0), BB) &&
01375          isExportableFromCurrentBlock(BOp->getOperand(1), BB))) {
01376       ISD::CondCode Condition;
01377       if (const ICmpInst *IC = dyn_cast<ICmpInst>(Cond)) {
01378         Condition = getICmpCondCode(IC->getPredicate());
01379       } else if (const FCmpInst *FC = dyn_cast<FCmpInst>(Cond)) {
01380         Condition = getFCmpCondCode(FC->getPredicate());
01381         if (TM.Options.NoNaNsFPMath)
01382           Condition = getFCmpCodeWithoutNaN(Condition);
01383       } else {
01384         (void)Condition; // silence warning.
01385         llvm_unreachable("Unknown compare instruction");
01386       }
01387 
01388       CaseBlock CB(Condition, BOp->getOperand(0), BOp->getOperand(1), nullptr,
01389                    TBB, FBB, CurBB, TWeight, FWeight);
01390       SwitchCases.push_back(CB);
01391       return;
01392     }
01393   }
01394 
01395   // Create a CaseBlock record representing this branch.
01396   CaseBlock CB(ISD::SETEQ, Cond, ConstantInt::getTrue(*DAG.getContext()),
01397                nullptr, TBB, FBB, CurBB, TWeight, FWeight);
01398   SwitchCases.push_back(CB);
01399 }
01400 
01401 /// Scale down both weights to fit into uint32_t.
01402 static void ScaleWeights(uint64_t &NewTrue, uint64_t &NewFalse) {
01403   uint64_t NewMax = (NewTrue > NewFalse) ? NewTrue : NewFalse;
01404   uint32_t Scale = (NewMax / UINT32_MAX) + 1;
01405   NewTrue = NewTrue / Scale;
01406   NewFalse = NewFalse / Scale;
01407 }
01408 
01409 /// FindMergedConditions - If Cond is an expression like
01410 void SelectionDAGBuilder::FindMergedConditions(const Value *Cond,
01411                                                MachineBasicBlock *TBB,
01412                                                MachineBasicBlock *FBB,
01413                                                MachineBasicBlock *CurBB,
01414                                                MachineBasicBlock *SwitchBB,
01415                                                unsigned Opc, uint32_t TWeight,
01416                                                uint32_t FWeight) {
01417   // If this node is not part of the or/and tree, emit it as a branch.
01418   const Instruction *BOp = dyn_cast<Instruction>(Cond);
01419   if (!BOp || !(isa<BinaryOperator>(BOp) || isa<CmpInst>(BOp)) ||
01420       (unsigned)BOp->getOpcode() != Opc || !BOp->hasOneUse() ||
01421       BOp->getParent() != CurBB->getBasicBlock() ||
01422       !InBlock(BOp->getOperand(0), CurBB->getBasicBlock()) ||
01423       !InBlock(BOp->getOperand(1), CurBB->getBasicBlock())) {
01424     EmitBranchForMergedCondition(Cond, TBB, FBB, CurBB, SwitchBB,
01425                                  TWeight, FWeight);
01426     return;
01427   }
01428 
01429   //  Create TmpBB after CurBB.
01430   MachineFunction::iterator BBI = CurBB;
01431   MachineFunction &MF = DAG.getMachineFunction();
01432   MachineBasicBlock *TmpBB = MF.CreateMachineBasicBlock(CurBB->getBasicBlock());
01433   CurBB->getParent()->insert(++BBI, TmpBB);
01434 
01435   if (Opc == Instruction::Or) {
01436     // Codegen X | Y as:
01437     // BB1:
01438     //   jmp_if_X TBB
01439     //   jmp TmpBB
01440     // TmpBB:
01441     //   jmp_if_Y TBB
01442     //   jmp FBB
01443     //
01444 
01445     // We have flexibility in setting Prob for BB1 and Prob for TmpBB.
01446     // The requirement is that
01447     //   TrueProb for BB1 + (FalseProb for BB1 * TrueProb for TmpBB)
01448     //     = TrueProb for orignal BB.
01449     // Assuming the orignal weights are A and B, one choice is to set BB1's
01450     // weights to A and A+2B, and set TmpBB's weights to A and 2B. This choice
01451     // assumes that
01452     //   TrueProb for BB1 == FalseProb for BB1 * TrueProb for TmpBB.
01453     // Another choice is to assume TrueProb for BB1 equals to TrueProb for
01454     // TmpBB, but the math is more complicated.
01455 
01456     uint64_t NewTrueWeight = TWeight;
01457     uint64_t NewFalseWeight = (uint64_t)TWeight + 2 * (uint64_t)FWeight;
01458     ScaleWeights(NewTrueWeight, NewFalseWeight);
01459     // Emit the LHS condition.
01460     FindMergedConditions(BOp->getOperand(0), TBB, TmpBB, CurBB, SwitchBB, Opc,
01461                          NewTrueWeight, NewFalseWeight);
01462 
01463     NewTrueWeight = TWeight;
01464     NewFalseWeight = 2 * (uint64_t)FWeight;
01465     ScaleWeights(NewTrueWeight, NewFalseWeight);
01466     // Emit the RHS condition into TmpBB.
01467     FindMergedConditions(BOp->getOperand(1), TBB, FBB, TmpBB, SwitchBB, Opc,
01468                          NewTrueWeight, NewFalseWeight);
01469   } else {
01470     assert(Opc == Instruction::And && "Unknown merge op!");
01471     // Codegen X & Y as:
01472     // BB1:
01473     //   jmp_if_X TmpBB
01474     //   jmp FBB
01475     // TmpBB:
01476     //   jmp_if_Y TBB
01477     //   jmp FBB
01478     //
01479     //  This requires creation of TmpBB after CurBB.
01480 
01481     // We have flexibility in setting Prob for BB1 and Prob for TmpBB.
01482     // The requirement is that
01483     //   FalseProb for BB1 + (TrueProb for BB1 * FalseProb for TmpBB)
01484     //     = FalseProb for orignal BB.
01485     // Assuming the orignal weights are A and B, one choice is to set BB1's
01486     // weights to 2A+B and B, and set TmpBB's weights to 2A and B. This choice
01487     // assumes that
01488     //   FalseProb for BB1 == TrueProb for BB1 * FalseProb for TmpBB.
01489 
01490     uint64_t NewTrueWeight = 2 * (uint64_t)TWeight + (uint64_t)FWeight;
01491     uint64_t NewFalseWeight = FWeight;
01492     ScaleWeights(NewTrueWeight, NewFalseWeight);
01493     // Emit the LHS condition.
01494     FindMergedConditions(BOp->getOperand(0), TmpBB, FBB, CurBB, SwitchBB, Opc,
01495                          NewTrueWeight, NewFalseWeight);
01496 
01497     NewTrueWeight = 2 * (uint64_t)TWeight;
01498     NewFalseWeight = FWeight;
01499     ScaleWeights(NewTrueWeight, NewFalseWeight);
01500     // Emit the RHS condition into TmpBB.
01501     FindMergedConditions(BOp->getOperand(1), TBB, FBB, TmpBB, SwitchBB, Opc,
01502                          NewTrueWeight, NewFalseWeight);
01503   }
01504 }
01505 
01506 /// If the set of cases should be emitted as a series of branches, return true.
01507 /// If we should emit this as a bunch of and/or'd together conditions, return
01508 /// false.
01509 bool
01510 SelectionDAGBuilder::ShouldEmitAsBranches(const std::vector<CaseBlock> &Cases) {
01511   if (Cases.size() != 2) return true;
01512 
01513   // If this is two comparisons of the same values or'd or and'd together, they
01514   // will get folded into a single comparison, so don't emit two blocks.
01515   if ((Cases[0].CmpLHS == Cases[1].CmpLHS &&
01516        Cases[0].CmpRHS == Cases[1].CmpRHS) ||
01517       (Cases[0].CmpRHS == Cases[1].CmpLHS &&
01518        Cases[0].CmpLHS == Cases[1].CmpRHS)) {
01519     return false;
01520   }
01521 
01522   // Handle: (X != null) | (Y != null) --> (X|Y) != 0
01523   // Handle: (X == null) & (Y == null) --> (X|Y) == 0
01524   if (Cases[0].CmpRHS == Cases[1].CmpRHS &&
01525       Cases[0].CC == Cases[1].CC &&
01526       isa<Constant>(Cases[0].CmpRHS) &&
01527       cast<Constant>(Cases[0].CmpRHS)->isNullValue()) {
01528     if (Cases[0].CC == ISD::SETEQ && Cases[0].TrueBB == Cases[1].ThisBB)
01529       return false;
01530     if (Cases[0].CC == ISD::SETNE && Cases[0].FalseBB == Cases[1].ThisBB)
01531       return false;
01532   }
01533 
01534   return true;
01535 }
01536 
01537 void SelectionDAGBuilder::visitBr(const BranchInst &I) {
01538   MachineBasicBlock *BrMBB = FuncInfo.MBB;
01539 
01540   // Update machine-CFG edges.
01541   MachineBasicBlock *Succ0MBB = FuncInfo.MBBMap[I.getSuccessor(0)];
01542 
01543   if (I.isUnconditional()) {
01544     // Update machine-CFG edges.
01545     BrMBB->addSuccessor(Succ0MBB);
01546 
01547     // If this is not a fall-through branch or optimizations are switched off,
01548     // emit the branch.
01549     if (Succ0MBB != NextBlock(BrMBB) || TM.getOptLevel() == CodeGenOpt::None)
01550       DAG.setRoot(DAG.getNode(ISD::BR, getCurSDLoc(),
01551                               MVT::Other, getControlRoot(),
01552                               DAG.getBasicBlock(Succ0MBB)));
01553 
01554     return;
01555   }
01556 
01557   // If this condition is one of the special cases we handle, do special stuff
01558   // now.
01559   const Value *CondVal = I.getCondition();
01560   MachineBasicBlock *Succ1MBB = FuncInfo.MBBMap[I.getSuccessor(1)];
01561 
01562   // If this is a series of conditions that are or'd or and'd together, emit
01563   // this as a sequence of branches instead of setcc's with and/or operations.
01564   // As long as jumps are not expensive, this should improve performance.
01565   // For example, instead of something like:
01566   //     cmp A, B
01567   //     C = seteq
01568   //     cmp D, E
01569   //     F = setle
01570   //     or C, F
01571   //     jnz foo
01572   // Emit:
01573   //     cmp A, B
01574   //     je foo
01575   //     cmp D, E
01576   //     jle foo
01577   //
01578   if (const BinaryOperator *BOp = dyn_cast<BinaryOperator>(CondVal)) {
01579     if (!DAG.getTargetLoweringInfo().isJumpExpensive() &&
01580         BOp->hasOneUse() && (BOp->getOpcode() == Instruction::And ||
01581                              BOp->getOpcode() == Instruction::Or)) {
01582       FindMergedConditions(BOp, Succ0MBB, Succ1MBB, BrMBB, BrMBB,
01583                            BOp->getOpcode(), getEdgeWeight(BrMBB, Succ0MBB),
01584                            getEdgeWeight(BrMBB, Succ1MBB));
01585       // If the compares in later blocks need to use values not currently
01586       // exported from this block, export them now.  This block should always
01587       // be the first entry.
01588       assert(SwitchCases[0].ThisBB == BrMBB && "Unexpected lowering!");
01589 
01590       // Allow some cases to be rejected.
01591       if (ShouldEmitAsBranches(SwitchCases)) {
01592         for (unsigned i = 1, e = SwitchCases.size(); i != e; ++i) {
01593           ExportFromCurrentBlock(SwitchCases[i].CmpLHS);
01594           ExportFromCurrentBlock(SwitchCases[i].CmpRHS);
01595         }
01596 
01597         // Emit the branch for this block.
01598         visitSwitchCase(SwitchCases[0], BrMBB);
01599         SwitchCases.erase(SwitchCases.begin());
01600         return;
01601       }
01602 
01603       // Okay, we decided not to do this, remove any inserted MBB's and clear
01604       // SwitchCases.
01605       for (unsigned i = 1, e = SwitchCases.size(); i != e; ++i)
01606         FuncInfo.MF->erase(SwitchCases[i].ThisBB);
01607 
01608       SwitchCases.clear();
01609     }
01610   }
01611 
01612   // Create a CaseBlock record representing this branch.
01613   CaseBlock CB(ISD::SETEQ, CondVal, ConstantInt::getTrue(*DAG.getContext()),
01614                nullptr, Succ0MBB, Succ1MBB, BrMBB);
01615 
01616   // Use visitSwitchCase to actually insert the fast branch sequence for this
01617   // cond branch.
01618   visitSwitchCase(CB, BrMBB);
01619 }
01620 
01621 /// visitSwitchCase - Emits the necessary code to represent a single node in
01622 /// the binary search tree resulting from lowering a switch instruction.
01623 void SelectionDAGBuilder::visitSwitchCase(CaseBlock &CB,
01624                                           MachineBasicBlock *SwitchBB) {
01625   SDValue Cond;
01626   SDValue CondLHS = getValue(CB.CmpLHS);
01627   SDLoc dl = getCurSDLoc();
01628 
01629   // Build the setcc now.
01630   if (!CB.CmpMHS) {
01631     // Fold "(X == true)" to X and "(X == false)" to !X to
01632     // handle common cases produced by branch lowering.
01633     if (CB.CmpRHS == ConstantInt::getTrue(*DAG.getContext()) &&
01634         CB.CC == ISD::SETEQ)
01635       Cond = CondLHS;
01636     else if (CB.CmpRHS == ConstantInt::getFalse(*DAG.getContext()) &&
01637              CB.CC == ISD::SETEQ) {
01638       SDValue True = DAG.getConstant(1, dl, CondLHS.getValueType());
01639       Cond = DAG.getNode(ISD::XOR, dl, CondLHS.getValueType(), CondLHS, True);
01640     } else
01641       Cond = DAG.getSetCC(dl, MVT::i1, CondLHS, getValue(CB.CmpRHS), CB.CC);
01642   } else {
01643     assert(CB.CC == ISD::SETLE && "Can handle only LE ranges now");
01644 
01645     const APInt& Low = cast<ConstantInt>(CB.CmpLHS)->getValue();
01646     const APInt& High = cast<ConstantInt>(CB.CmpRHS)->getValue();
01647 
01648     SDValue CmpOp = getValue(CB.CmpMHS);
01649     EVT VT = CmpOp.getValueType();
01650 
01651     if (cast<ConstantInt>(CB.CmpLHS)->isMinValue(true)) {
01652       Cond = DAG.getSetCC(dl, MVT::i1, CmpOp, DAG.getConstant(High, dl, VT),
01653                           ISD::SETLE);
01654     } else {
01655       SDValue SUB = DAG.getNode(ISD::SUB, dl,
01656                                 VT, CmpOp, DAG.getConstant(Low, dl, VT));
01657       Cond = DAG.getSetCC(dl, MVT::i1, SUB,
01658                           DAG.getConstant(High-Low, dl, VT), ISD::SETULE);
01659     }
01660   }
01661 
01662   // Update successor info
01663   addSuccessorWithWeight(SwitchBB, CB.TrueBB, CB.TrueWeight);
01664   // TrueBB and FalseBB are always different unless the incoming IR is
01665   // degenerate. This only happens when running llc on weird IR.
01666   if (CB.TrueBB != CB.FalseBB)
01667     addSuccessorWithWeight(SwitchBB, CB.FalseBB, CB.FalseWeight);
01668 
01669   // If the lhs block is the next block, invert the condition so that we can
01670   // fall through to the lhs instead of the rhs block.
01671   if (CB.TrueBB == NextBlock(SwitchBB)) {
01672     std::swap(CB.TrueBB, CB.FalseBB);
01673     SDValue True = DAG.getConstant(1, dl, Cond.getValueType());
01674     Cond = DAG.getNode(ISD::XOR, dl, Cond.getValueType(), Cond, True);
01675   }
01676 
01677   SDValue BrCond = DAG.getNode(ISD::BRCOND, dl,
01678                                MVT::Other, getControlRoot(), Cond,
01679                                DAG.getBasicBlock(CB.TrueBB));
01680 
01681   // Insert the false branch. Do this even if it's a fall through branch,
01682   // this makes it easier to do DAG optimizations which require inverting
01683   // the branch condition.
01684   BrCond = DAG.getNode(ISD::BR, dl, MVT::Other, BrCond,
01685                        DAG.getBasicBlock(CB.FalseBB));
01686 
01687   DAG.setRoot(BrCond);
01688 }
01689 
01690 /// visitJumpTable - Emit JumpTable node in the current MBB
01691 void SelectionDAGBuilder::visitJumpTable(JumpTable &JT) {
01692   // Emit the code for the jump table
01693   assert(JT.Reg != -1U && "Should lower JT Header first!");
01694   EVT PTy = DAG.getTargetLoweringInfo().getPointerTy();
01695   SDValue Index = DAG.getCopyFromReg(getControlRoot(), getCurSDLoc(),
01696                                      JT.Reg, PTy);
01697   SDValue Table = DAG.getJumpTable(JT.JTI, PTy);
01698   SDValue BrJumpTable = DAG.getNode(ISD::BR_JT, getCurSDLoc(),
01699                                     MVT::Other, Index.getValue(1),
01700                                     Table, Index);
01701   DAG.setRoot(BrJumpTable);
01702 }
01703 
01704 /// visitJumpTableHeader - This function emits necessary code to produce index
01705 /// in the JumpTable from switch case.
01706 void SelectionDAGBuilder::visitJumpTableHeader(JumpTable &JT,
01707                                                JumpTableHeader &JTH,
01708                                                MachineBasicBlock *SwitchBB) {
01709   SDLoc dl = getCurSDLoc();
01710 
01711   // Subtract the lowest switch case value from the value being switched on and
01712   // conditional branch to default mbb if the result is greater than the
01713   // difference between smallest and largest cases.
01714   SDValue SwitchOp = getValue(JTH.SValue);
01715   EVT VT = SwitchOp.getValueType();
01716   SDValue Sub = DAG.getNode(ISD::SUB, dl, VT, SwitchOp,
01717                             DAG.getConstant(JTH.First, dl, VT));
01718 
01719   // The SDNode we just created, which holds the value being switched on minus
01720   // the smallest case value, needs to be copied to a virtual register so it
01721   // can be used as an index into the jump table in a subsequent basic block.
01722   // This value may be smaller or larger than the target's pointer type, and
01723   // therefore require extension or truncating.
01724   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
01725   SwitchOp = DAG.getZExtOrTrunc(Sub, dl, TLI.getPointerTy());
01726 
01727   unsigned JumpTableReg = FuncInfo.CreateReg(TLI.getPointerTy());
01728   SDValue CopyTo = DAG.getCopyToReg(getControlRoot(), dl,
01729                                     JumpTableReg, SwitchOp);
01730   JT.Reg = JumpTableReg;
01731 
01732   // Emit the range check for the jump table, and branch to the default block
01733   // for the switch statement if the value being switched on exceeds the largest
01734   // case in the switch.
01735   SDValue CMP =
01736       DAG.getSetCC(dl, TLI.getSetCCResultType(*DAG.getContext(),
01737                                               Sub.getValueType()),
01738                    Sub, DAG.getConstant(JTH.Last - JTH.First, dl, VT),
01739                    ISD::SETUGT);
01740 
01741   SDValue BrCond = DAG.getNode(ISD::BRCOND, dl,
01742                                MVT::Other, CopyTo, CMP,
01743                                DAG.getBasicBlock(JT.Default));
01744 
01745   // Avoid emitting unnecessary branches to the next block.
01746   if (JT.MBB != NextBlock(SwitchBB))
01747     BrCond = DAG.getNode(ISD::BR, dl, MVT::Other, BrCond,
01748                          DAG.getBasicBlock(JT.MBB));
01749 
01750   DAG.setRoot(BrCond);
01751 }
01752 
01753 /// Codegen a new tail for a stack protector check ParentMBB which has had its
01754 /// tail spliced into a stack protector check success bb.
01755 ///
01756 /// For a high level explanation of how this fits into the stack protector
01757 /// generation see the comment on the declaration of class
01758 /// StackProtectorDescriptor.
01759 void SelectionDAGBuilder::visitSPDescriptorParent(StackProtectorDescriptor &SPD,
01760                                                   MachineBasicBlock *ParentBB) {
01761 
01762   // First create the loads to the guard/stack slot for the comparison.
01763   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
01764   EVT PtrTy = TLI.getPointerTy();
01765 
01766   MachineFrameInfo *MFI = ParentBB->getParent()->getFrameInfo();
01767   int FI = MFI->getStackProtectorIndex();
01768 
01769   const Value *IRGuard = SPD.getGuard();
01770   SDValue GuardPtr = getValue(IRGuard);
01771   SDValue StackSlotPtr = DAG.getFrameIndex(FI, PtrTy);
01772 
01773   unsigned Align =
01774     TLI.getDataLayout()->getPrefTypeAlignment(IRGuard->getType());
01775 
01776   SDValue Guard;
01777   SDLoc dl = getCurSDLoc();
01778 
01779   // If GuardReg is set and useLoadStackGuardNode returns true, retrieve the
01780   // guard value from the virtual register holding the value. Otherwise, emit a
01781   // volatile load to retrieve the stack guard value.
01782   unsigned GuardReg = SPD.getGuardReg();
01783 
01784   if (GuardReg && TLI.useLoadStackGuardNode())
01785     Guard = DAG.getCopyFromReg(DAG.getEntryNode(), dl, GuardReg,
01786                                PtrTy);
01787   else
01788     Guard = DAG.getLoad(PtrTy, dl, DAG.getEntryNode(),
01789                         GuardPtr, MachinePointerInfo(IRGuard, 0),
01790                         true, false, false, Align);
01791 
01792   SDValue StackSlot = DAG.getLoad(PtrTy, dl, DAG.getEntryNode(),
01793                                   StackSlotPtr,
01794                                   MachinePointerInfo::getFixedStack(FI),
01795                                   true, false, false, Align);
01796 
01797   // Perform the comparison via a subtract/getsetcc.
01798   EVT VT = Guard.getValueType();
01799   SDValue Sub = DAG.getNode(ISD::SUB, dl, VT, Guard, StackSlot);
01800 
01801   SDValue Cmp =
01802       DAG.getSetCC(dl, TLI.getSetCCResultType(*DAG.getContext(),
01803                                                          Sub.getValueType()),
01804                    Sub, DAG.getConstant(0, dl, VT), ISD::SETNE);
01805 
01806   // If the sub is not 0, then we know the guard/stackslot do not equal, so
01807   // branch to failure MBB.
01808   SDValue BrCond = DAG.getNode(ISD::BRCOND, dl,
01809                                MVT::Other, StackSlot.getOperand(0),
01810                                Cmp, DAG.getBasicBlock(SPD.getFailureMBB()));
01811   // Otherwise branch to success MBB.
01812   SDValue Br = DAG.getNode(ISD::BR, dl,
01813                            MVT::Other, BrCond,
01814                            DAG.getBasicBlock(SPD.getSuccessMBB()));
01815 
01816   DAG.setRoot(Br);
01817 }
01818 
01819 /// Codegen the failure basic block for a stack protector check.
01820 ///
01821 /// A failure stack protector machine basic block consists simply of a call to
01822 /// __stack_chk_fail().
01823 ///
01824 /// For a high level explanation of how this fits into the stack protector
01825 /// generation see the comment on the declaration of class
01826 /// StackProtectorDescriptor.
01827 void
01828 SelectionDAGBuilder::visitSPDescriptorFailure(StackProtectorDescriptor &SPD) {
01829   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
01830   SDValue Chain =
01831       TLI.makeLibCall(DAG, RTLIB::STACKPROTECTOR_CHECK_FAIL, MVT::isVoid,
01832                       nullptr, 0, false, getCurSDLoc(), false, false).second;
01833   DAG.setRoot(Chain);
01834 }
01835 
01836 /// visitBitTestHeader - This function emits necessary code to produce value
01837 /// suitable for "bit tests"
01838 void SelectionDAGBuilder::visitBitTestHeader(BitTestBlock &B,
01839                                              MachineBasicBlock *SwitchBB) {
01840   SDLoc dl = getCurSDLoc();
01841 
01842   // Subtract the minimum value
01843   SDValue SwitchOp = getValue(B.SValue);
01844   EVT VT = SwitchOp.getValueType();
01845   SDValue Sub = DAG.getNode(ISD::SUB, dl, VT, SwitchOp,
01846                             DAG.getConstant(B.First, dl, VT));
01847 
01848   // Check range
01849   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
01850   SDValue RangeCmp =
01851       DAG.getSetCC(dl, TLI.getSetCCResultType(*DAG.getContext(),
01852                                               Sub.getValueType()),
01853                    Sub, DAG.getConstant(B.Range, dl, VT), ISD::SETUGT);
01854 
01855   // Determine the type of the test operands.
01856   bool UsePtrType = false;
01857   if (!TLI.isTypeLegal(VT))
01858     UsePtrType = true;
01859   else {
01860     for (unsigned i = 0, e = B.Cases.size(); i != e; ++i)
01861       if (!isUIntN(VT.getSizeInBits(), B.Cases[i].Mask)) {
01862         // Switch table case range are encoded into series of masks.
01863         // Just use pointer type, it's guaranteed to fit.
01864         UsePtrType = true;
01865         break;
01866       }
01867   }
01868   if (UsePtrType) {
01869     VT = TLI.getPointerTy();
01870     Sub = DAG.getZExtOrTrunc(Sub, dl, VT);
01871   }
01872 
01873   B.RegVT = VT.getSimpleVT();
01874   B.Reg = FuncInfo.CreateReg(B.RegVT);
01875   SDValue CopyTo = DAG.getCopyToReg(getControlRoot(), dl, B.Reg, Sub);
01876 
01877   MachineBasicBlock* MBB = B.Cases[0].ThisBB;
01878 
01879   addSuccessorWithWeight(SwitchBB, B.Default);
01880   addSuccessorWithWeight(SwitchBB, MBB);
01881 
01882   SDValue BrRange = DAG.getNode(ISD::BRCOND, dl,
01883                                 MVT::Other, CopyTo, RangeCmp,
01884                                 DAG.getBasicBlock(B.Default));
01885 
01886   // Avoid emitting unnecessary branches to the next block.
01887   if (MBB != NextBlock(SwitchBB))
01888     BrRange = DAG.getNode(ISD::BR, dl, MVT::Other, BrRange,
01889                           DAG.getBasicBlock(MBB));
01890 
01891   DAG.setRoot(BrRange);
01892 }
01893 
01894 /// visitBitTestCase - this function produces one "bit test"
01895 void SelectionDAGBuilder::visitBitTestCase(BitTestBlock &BB,
01896                                            MachineBasicBlock* NextMBB,
01897                                            uint32_t BranchWeightToNext,
01898                                            unsigned Reg,
01899                                            BitTestCase &B,
01900                                            MachineBasicBlock *SwitchBB) {
01901   SDLoc dl = getCurSDLoc();
01902   MVT VT = BB.RegVT;
01903   SDValue ShiftOp = DAG.getCopyFromReg(getControlRoot(), dl, Reg, VT);
01904   SDValue Cmp;
01905   unsigned PopCount = countPopulation(B.Mask);
01906   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
01907   if (PopCount == 1) {
01908     // Testing for a single bit; just compare the shift count with what it
01909     // would need to be to shift a 1 bit in that position.
01910     Cmp = DAG.getSetCC(
01911         dl, TLI.getSetCCResultType(*DAG.getContext(), VT), ShiftOp,
01912         DAG.getConstant(countTrailingZeros(B.Mask), dl, VT), ISD::SETEQ);
01913   } else if (PopCount == BB.Range) {
01914     // There is only one zero bit in the range, test for it directly.
01915     Cmp = DAG.getSetCC(
01916         dl, TLI.getSetCCResultType(*DAG.getContext(), VT), ShiftOp,
01917         DAG.getConstant(countTrailingOnes(B.Mask), dl, VT), ISD::SETNE);
01918   } else {
01919     // Make desired shift
01920     SDValue SwitchVal = DAG.getNode(ISD::SHL, dl, VT,
01921                                     DAG.getConstant(1, dl, VT), ShiftOp);
01922 
01923     // Emit bit tests and jumps
01924     SDValue AndOp = DAG.getNode(ISD::AND, dl,
01925                                 VT, SwitchVal, DAG.getConstant(B.Mask, dl, VT));
01926     Cmp = DAG.getSetCC(dl, TLI.getSetCCResultType(*DAG.getContext(), VT), AndOp,
01927                        DAG.getConstant(0, dl, VT), ISD::SETNE);
01928   }
01929 
01930   // The branch weight from SwitchBB to B.TargetBB is B.ExtraWeight.
01931   addSuccessorWithWeight(SwitchBB, B.TargetBB, B.ExtraWeight);
01932   // The branch weight from SwitchBB to NextMBB is BranchWeightToNext.
01933   addSuccessorWithWeight(SwitchBB, NextMBB, BranchWeightToNext);
01934 
01935   SDValue BrAnd = DAG.getNode(ISD::BRCOND, dl,
01936                               MVT::Other, getControlRoot(),
01937                               Cmp, DAG.getBasicBlock(B.TargetBB));
01938 
01939   // Avoid emitting unnecessary branches to the next block.
01940   if (NextMBB != NextBlock(SwitchBB))
01941     BrAnd = DAG.getNode(ISD::BR, dl, MVT::Other, BrAnd,
01942                         DAG.getBasicBlock(NextMBB));
01943 
01944   DAG.setRoot(BrAnd);
01945 }
01946 
01947 void SelectionDAGBuilder::visitInvoke(const InvokeInst &I) {
01948   MachineBasicBlock *InvokeMBB = FuncInfo.MBB;
01949 
01950   // Retrieve successors.
01951   MachineBasicBlock *Return = FuncInfo.MBBMap[I.getSuccessor(0)];
01952   MachineBasicBlock *LandingPad = FuncInfo.MBBMap[I.getSuccessor(1)];
01953 
01954   const Value *Callee(I.getCalledValue());
01955   const Function *Fn = dyn_cast<Function>(Callee);
01956   if (isa<InlineAsm>(Callee))
01957     visitInlineAsm(&I);
01958   else if (Fn && Fn->isIntrinsic()) {
01959     switch (Fn->getIntrinsicID()) {
01960     default:
01961       llvm_unreachable("Cannot invoke this intrinsic");
01962     case Intrinsic::donothing:
01963       // Ignore invokes to @llvm.donothing: jump directly to the next BB.
01964       break;
01965     case Intrinsic::experimental_patchpoint_void:
01966     case Intrinsic::experimental_patchpoint_i64:
01967       visitPatchpoint(&I, LandingPad);
01968       break;
01969     case Intrinsic::experimental_gc_statepoint:
01970       LowerStatepoint(ImmutableStatepoint(&I), LandingPad);
01971       break;
01972     }
01973   } else
01974     LowerCallTo(&I, getValue(Callee), false, LandingPad);
01975 
01976   // If the value of the invoke is used outside of its defining block, make it
01977   // available as a virtual register.
01978   // We already took care of the exported value for the statepoint instruction
01979   // during call to the LowerStatepoint.
01980   if (!isStatepoint(I)) {
01981     CopyToExportRegsIfNeeded(&I);
01982   }
01983 
01984   // Update successor info
01985   addSuccessorWithWeight(InvokeMBB, Return);
01986   addSuccessorWithWeight(InvokeMBB, LandingPad);
01987 
01988   // Drop into normal successor.
01989   DAG.setRoot(DAG.getNode(ISD::BR, getCurSDLoc(),
01990                           MVT::Other, getControlRoot(),
01991                           DAG.getBasicBlock(Return)));
01992 }
01993 
01994 void SelectionDAGBuilder::visitResume(const ResumeInst &RI) {
01995   llvm_unreachable("SelectionDAGBuilder shouldn't visit resume instructions!");
01996 }
01997 
01998 void SelectionDAGBuilder::visitLandingPad(const LandingPadInst &LP) {
01999   assert(FuncInfo.MBB->isLandingPad() &&
02000          "Call to landingpad not in landing pad!");
02001 
02002   MachineBasicBlock *MBB = FuncInfo.MBB;
02003   MachineModuleInfo &MMI = DAG.getMachineFunction().getMMI();
02004   AddLandingPadInfo(LP, MMI, MBB);
02005 
02006   // If there aren't registers to copy the values into (e.g., during SjLj
02007   // exceptions), then don't bother to create these DAG nodes.
02008   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
02009   if (TLI.getExceptionPointerRegister() == 0 &&
02010       TLI.getExceptionSelectorRegister() == 0)
02011     return;
02012 
02013   SmallVector<EVT, 2> ValueVTs;
02014   SDLoc dl = getCurSDLoc();
02015   ComputeValueVTs(TLI, LP.getType(), ValueVTs);
02016   assert(ValueVTs.size() == 2 && "Only two-valued landingpads are supported");
02017 
02018   // Get the two live-in registers as SDValues. The physregs have already been
02019   // copied into virtual registers.
02020   SDValue Ops[2];
02021   if (FuncInfo.ExceptionPointerVirtReg) {
02022     Ops[0] = DAG.getZExtOrTrunc(
02023         DAG.getCopyFromReg(DAG.getEntryNode(), dl,
02024                            FuncInfo.ExceptionPointerVirtReg, TLI.getPointerTy()),
02025         dl, ValueVTs[0]);
02026   } else {
02027     Ops[0] = DAG.getConstant(0, dl, TLI.getPointerTy());
02028   }
02029   Ops[1] = DAG.getZExtOrTrunc(
02030       DAG.getCopyFromReg(DAG.getEntryNode(), dl,
02031                          FuncInfo.ExceptionSelectorVirtReg, TLI.getPointerTy()),
02032       dl, ValueVTs[1]);
02033 
02034   // Merge into one.
02035   SDValue Res = DAG.getNode(ISD::MERGE_VALUES, dl,
02036                             DAG.getVTList(ValueVTs), Ops);
02037   setValue(&LP, Res);
02038 }
02039 
02040 unsigned
02041 SelectionDAGBuilder::visitLandingPadClauseBB(GlobalValue *ClauseGV,
02042                                              MachineBasicBlock *LPadBB) {
02043   SDValue Chain = getControlRoot();
02044   SDLoc dl = getCurSDLoc();
02045 
02046   // Get the typeid that we will dispatch on later.
02047   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
02048   const TargetRegisterClass *RC = TLI.getRegClassFor(TLI.getPointerTy());
02049   unsigned VReg = FuncInfo.MF->getRegInfo().createVirtualRegister(RC);
02050   unsigned TypeID = DAG.getMachineFunction().getMMI().getTypeIDFor(ClauseGV);
02051   SDValue Sel = DAG.getConstant(TypeID, dl, TLI.getPointerTy());
02052   Chain = DAG.getCopyToReg(Chain, dl, VReg, Sel);
02053 
02054   // Branch to the main landing pad block.
02055   MachineBasicBlock *ClauseMBB = FuncInfo.MBB;
02056   ClauseMBB->addSuccessor(LPadBB);
02057   DAG.setRoot(DAG.getNode(ISD::BR, dl, MVT::Other, Chain,
02058                           DAG.getBasicBlock(LPadBB)));
02059   return VReg;
02060 }
02061 
02062 void SelectionDAGBuilder::sortAndRangeify(CaseClusterVector &Clusters) {
02063 #ifndef NDEBUG
02064   for (const CaseCluster &CC : Clusters)
02065     assert(CC.Low == CC.High && "Input clusters must be single-case");
02066 #endif
02067 
02068   std::sort(Clusters.begin(), Clusters.end(),
02069             [](const CaseCluster &a, const CaseCluster &b) {
02070     return a.Low->getValue().slt(b.Low->getValue());
02071   });
02072 
02073   // Merge adjacent clusters with the same destination.
02074   const unsigned N = Clusters.size();
02075   unsigned DstIndex = 0;
02076   for (unsigned SrcIndex = 0; SrcIndex < N; ++SrcIndex) {
02077     CaseCluster &CC = Clusters[SrcIndex];
02078     const ConstantInt *CaseVal = CC.Low;
02079     MachineBasicBlock *Succ = CC.MBB;
02080 
02081     if (DstIndex != 0 && Clusters[DstIndex - 1].MBB == Succ &&
02082         (CaseVal->getValue() - Clusters[DstIndex - 1].High->getValue()) == 1) {
02083       // If this case has the same successor and is a neighbour, merge it into
02084       // the previous cluster.
02085       Clusters[DstIndex - 1].High = CaseVal;
02086       Clusters[DstIndex - 1].Weight += CC.Weight;
02087       assert(Clusters[DstIndex - 1].Weight >= CC.Weight && "Weight overflow!");
02088     } else {
02089       std::memmove(&Clusters[DstIndex++], &Clusters[SrcIndex],
02090                    sizeof(Clusters[SrcIndex]));
02091     }
02092   }
02093   Clusters.resize(DstIndex);
02094 }
02095 
02096 void SelectionDAGBuilder::UpdateSplitBlock(MachineBasicBlock *First,
02097                                            MachineBasicBlock *Last) {
02098   // Update JTCases.
02099   for (unsigned i = 0, e = JTCases.size(); i != e; ++i)
02100     if (JTCases[i].first.HeaderBB == First)
02101       JTCases[i].first.HeaderBB = Last;
02102 
02103   // Update BitTestCases.
02104   for (unsigned i = 0, e = BitTestCases.size(); i != e; ++i)
02105     if (BitTestCases[i].Parent == First)
02106       BitTestCases[i].Parent = Last;
02107 }
02108 
02109 void SelectionDAGBuilder::visitIndirectBr(const IndirectBrInst &I) {
02110   MachineBasicBlock *IndirectBrMBB = FuncInfo.MBB;
02111 
02112   // Update machine-CFG edges with unique successors.
02113   SmallSet<BasicBlock*, 32> Done;
02114   for (unsigned i = 0, e = I.getNumSuccessors(); i != e; ++i) {
02115     BasicBlock *BB = I.getSuccessor(i);
02116     bool Inserted = Done.insert(BB).second;
02117     if (!Inserted)
02118         continue;
02119 
02120     MachineBasicBlock *Succ = FuncInfo.MBBMap[BB];
02121     addSuccessorWithWeight(IndirectBrMBB, Succ);
02122   }
02123 
02124   DAG.setRoot(DAG.getNode(ISD::BRIND, getCurSDLoc(),
02125                           MVT::Other, getControlRoot(),
02126                           getValue(I.getAddress())));
02127 }
02128 
02129 void SelectionDAGBuilder::visitUnreachable(const UnreachableInst &I) {
02130   if (DAG.getTarget().Options.TrapUnreachable)
02131     DAG.setRoot(DAG.getNode(ISD::TRAP, getCurSDLoc(), MVT::Other, DAG.getRoot()));
02132 }
02133 
02134 void SelectionDAGBuilder::visitFSub(const User &I) {
02135   // -0.0 - X --> fneg
02136   Type *Ty = I.getType();
02137   if (isa<Constant>(I.getOperand(0)) &&
02138       I.getOperand(0) == ConstantFP::getZeroValueForNegation(Ty)) {
02139     SDValue Op2 = getValue(I.getOperand(1));
02140     setValue(&I, DAG.getNode(ISD::FNEG, getCurSDLoc(),
02141                              Op2.getValueType(), Op2));
02142     return;
02143   }
02144 
02145   visitBinary(I, ISD::FSUB);
02146 }
02147 
02148 void SelectionDAGBuilder::visitBinary(const User &I, unsigned OpCode) {
02149   SDValue Op1 = getValue(I.getOperand(0));
02150   SDValue Op2 = getValue(I.getOperand(1));
02151 
02152   bool nuw = false;
02153   bool nsw = false;
02154   bool exact = false;
02155   FastMathFlags FMF;
02156 
02157   if (const OverflowingBinaryOperator *OFBinOp =
02158           dyn_cast<const OverflowingBinaryOperator>(&I)) {
02159     nuw = OFBinOp->hasNoUnsignedWrap();
02160     nsw = OFBinOp->hasNoSignedWrap();
02161   }
02162   if (const PossiblyExactOperator *ExactOp =
02163           dyn_cast<const PossiblyExactOperator>(&I))
02164     exact = ExactOp->isExact();
02165   if (const FPMathOperator *FPOp = dyn_cast<const FPMathOperator>(&I))
02166     FMF = FPOp->getFastMathFlags();
02167 
02168   SDNodeFlags Flags;
02169   Flags.setExact(exact);
02170   Flags.setNoSignedWrap(nsw);
02171   Flags.setNoUnsignedWrap(nuw);
02172   if (EnableFMFInDAG) {
02173     Flags.setAllowReciprocal(FMF.allowReciprocal());
02174     Flags.setNoInfs(FMF.noInfs());
02175     Flags.setNoNaNs(FMF.noNaNs());
02176     Flags.setNoSignedZeros(FMF.noSignedZeros());
02177     Flags.setUnsafeAlgebra(FMF.unsafeAlgebra());
02178   }
02179   SDValue BinNodeValue = DAG.getNode(OpCode, getCurSDLoc(), Op1.getValueType(),
02180                                      Op1, Op2, &Flags);
02181   setValue(&I, BinNodeValue);
02182 }
02183 
02184 void SelectionDAGBuilder::visitShift(const User &I, unsigned Opcode) {
02185   SDValue Op1 = getValue(I.getOperand(0));
02186   SDValue Op2 = getValue(I.getOperand(1));
02187 
02188   EVT ShiftTy =
02189       DAG.getTargetLoweringInfo().getShiftAmountTy(Op2.getValueType());
02190 
02191   // Coerce the shift amount to the right type if we can.
02192   if (!I.getType()->isVectorTy() && Op2.getValueType() != ShiftTy) {
02193     unsigned ShiftSize = ShiftTy.getSizeInBits();
02194     unsigned Op2Size = Op2.getValueType().getSizeInBits();
02195     SDLoc DL = getCurSDLoc();
02196 
02197     // If the operand is smaller than the shift count type, promote it.
02198     if (ShiftSize > Op2Size)
02199       Op2 = DAG.getNode(ISD::ZERO_EXTEND, DL, ShiftTy, Op2);
02200 
02201     // If the operand is larger than the shift count type but the shift
02202     // count type has enough bits to represent any shift value, truncate
02203     // it now. This is a common case and it exposes the truncate to
02204     // optimization early.
02205     else if (ShiftSize >= Log2_32_Ceil(Op2.getValueType().getSizeInBits()))
02206       Op2 = DAG.getNode(ISD::TRUNCATE, DL, ShiftTy, Op2);
02207     // Otherwise we'll need to temporarily settle for some other convenient
02208     // type.  Type legalization will make adjustments once the shiftee is split.
02209     else
02210       Op2 = DAG.getZExtOrTrunc(Op2, DL, MVT::i32);
02211   }
02212 
02213   bool nuw = false;
02214   bool nsw = false;
02215   bool exact = false;
02216 
02217   if (Opcode == ISD::SRL || Opcode == ISD::SRA || Opcode == ISD::SHL) {
02218 
02219     if (const OverflowingBinaryOperator *OFBinOp =
02220             dyn_cast<const OverflowingBinaryOperator>(&I)) {
02221       nuw = OFBinOp->hasNoUnsignedWrap();
02222       nsw = OFBinOp->hasNoSignedWrap();
02223     }
02224     if (const PossiblyExactOperator *ExactOp =
02225             dyn_cast<const PossiblyExactOperator>(&I))
02226       exact = ExactOp->isExact();
02227   }
02228   SDNodeFlags Flags;
02229   Flags.setExact(exact);
02230   Flags.setNoSignedWrap(nsw);
02231   Flags.setNoUnsignedWrap(nuw);
02232   SDValue Res = DAG.getNode(Opcode, getCurSDLoc(), Op1.getValueType(), Op1, Op2,
02233                             &Flags);
02234   setValue(&I, Res);
02235 }
02236 
02237 void SelectionDAGBuilder::visitSDiv(const User &I) {
02238   SDValue Op1 = getValue(I.getOperand(0));
02239   SDValue Op2 = getValue(I.getOperand(1));
02240 
02241   // Turn exact SDivs into multiplications.
02242   // FIXME: This should be in DAGCombiner, but it doesn't have access to the
02243   // exact bit.
02244   if (isa<BinaryOperator>(&I) && cast<BinaryOperator>(&I)->isExact() &&
02245       !isa<ConstantSDNode>(Op1) &&
02246       isa<ConstantSDNode>(Op2) && !cast<ConstantSDNode>(Op2)->isNullValue())
02247     setValue(&I, DAG.getTargetLoweringInfo()
02248                      .BuildExactSDIV(Op1, Op2, getCurSDLoc(), DAG));
02249   else
02250     setValue(&I, DAG.getNode(ISD::SDIV, getCurSDLoc(), Op1.getValueType(),
02251                              Op1, Op2));
02252 }
02253 
02254 void SelectionDAGBuilder::visitICmp(const User &I) {
02255   ICmpInst::Predicate predicate = ICmpInst::BAD_ICMP_PREDICATE;
02256   if (const ICmpInst *IC = dyn_cast<ICmpInst>(&I))
02257     predicate = IC->getPredicate();
02258   else if (const ConstantExpr *IC = dyn_cast<ConstantExpr>(&I))
02259     predicate = ICmpInst::Predicate(IC->getPredicate());
02260   SDValue Op1 = getValue(I.getOperand(0));
02261   SDValue Op2 = getValue(I.getOperand(1));
02262   ISD::CondCode Opcode = getICmpCondCode(predicate);
02263 
02264   EVT DestVT = DAG.getTargetLoweringInfo().getValueType(I.getType());
02265   setValue(&I, DAG.getSetCC(getCurSDLoc(), DestVT, Op1, Op2, Opcode));
02266 }
02267 
02268 void SelectionDAGBuilder::visitFCmp(const User &I) {
02269   FCmpInst::Predicate predicate = FCmpInst::BAD_FCMP_PREDICATE;
02270   if (const FCmpInst *FC = dyn_cast<FCmpInst>(&I))
02271     predicate = FC->getPredicate();
02272   else if (const ConstantExpr *FC = dyn_cast<ConstantExpr>(&I))
02273     predicate = FCmpInst::Predicate(FC->getPredicate());
02274   SDValue Op1 = getValue(I.getOperand(0));
02275   SDValue Op2 = getValue(I.getOperand(1));
02276   ISD::CondCode Condition = getFCmpCondCode(predicate);
02277   if (TM.Options.NoNaNsFPMath)
02278     Condition = getFCmpCodeWithoutNaN(Condition);
02279   EVT DestVT = DAG.getTargetLoweringInfo().getValueType(I.getType());
02280   setValue(&I, DAG.getSetCC(getCurSDLoc(), DestVT, Op1, Op2, Condition));
02281 }
02282 
02283 void SelectionDAGBuilder::visitSelect(const User &I) {
02284   SmallVector<EVT, 4> ValueVTs;
02285   ComputeValueVTs(DAG.getTargetLoweringInfo(), I.getType(), ValueVTs);
02286   unsigned NumValues = ValueVTs.size();
02287   if (NumValues == 0) return;
02288 
02289   SmallVector<SDValue, 4> Values(NumValues);
02290   SDValue Cond     = getValue(I.getOperand(0));
02291   SDValue LHSVal   = getValue(I.getOperand(1));
02292   SDValue RHSVal   = getValue(I.getOperand(2));
02293   auto BaseOps = {Cond};
02294   ISD::NodeType OpCode = Cond.getValueType().isVector() ?
02295     ISD::VSELECT : ISD::SELECT;
02296 
02297   // Min/max matching is only viable if all output VTs are the same.
02298   if (std::equal(ValueVTs.begin(), ValueVTs.end(), ValueVTs.begin())) {
02299     Value *LHS, *RHS;
02300     SelectPatternFlavor SPF = matchSelectPattern(const_cast<User*>(&I), LHS, RHS);
02301     ISD::NodeType Opc = ISD::DELETED_NODE;
02302     switch (SPF) {
02303     case SPF_UMAX: Opc = ISD::UMAX; break;
02304     case SPF_UMIN: Opc = ISD::UMIN; break;
02305     case SPF_SMAX: Opc = ISD::SMAX; break;
02306     case SPF_SMIN: Opc = ISD::SMIN; break;
02307     default: break;
02308     }
02309 
02310     EVT VT = ValueVTs[0];
02311     LLVMContext &Ctx = *DAG.getContext();
02312     auto &TLI = DAG.getTargetLoweringInfo();
02313     while (TLI.getTypeAction(Ctx, VT) == TargetLoweringBase::TypeSplitVector)
02314       VT = TLI.getTypeToTransformTo(Ctx, VT);
02315 
02316     if (Opc != ISD::DELETED_NODE && TLI.isOperationLegalOrCustom(Opc, VT) &&
02317         // If the underlying comparison instruction is used by any other instruction,
02318         // the consumed instructions won't be destroyed, so it is not profitable
02319         // to convert to a min/max.
02320         cast<SelectInst>(&I)->getCondition()->hasOneUse()) {
02321       OpCode = Opc;
02322       LHSVal = getValue(LHS);
02323       RHSVal = getValue(RHS);
02324       BaseOps = {};
02325     }
02326   }
02327 
02328   for (unsigned i = 0; i != NumValues; ++i) {
02329     SmallVector<SDValue, 3> Ops(BaseOps.begin(), BaseOps.end());
02330     Ops.push_back(SDValue(LHSVal.getNode(), LHSVal.getResNo() + i));
02331     Ops.push_back(SDValue(RHSVal.getNode(), RHSVal.getResNo() + i));
02332     Values[i] = DAG.getNode(OpCode, getCurSDLoc(),
02333                             LHSVal.getNode()->getValueType(LHSVal.getResNo()+i),
02334                             Ops);
02335   }
02336 
02337   setValue(&I, DAG.getNode(ISD::MERGE_VALUES, getCurSDLoc(),
02338                            DAG.getVTList(ValueVTs), Values));
02339 }
02340 
02341 void SelectionDAGBuilder::visitTrunc(const User &I) {
02342   // TruncInst cannot be a no-op cast because sizeof(src) > sizeof(dest).
02343   SDValue N = getValue(I.getOperand(0));
02344   EVT DestVT = DAG.getTargetLoweringInfo().getValueType(I.getType());
02345   setValue(&I, DAG.getNode(ISD::TRUNCATE, getCurSDLoc(), DestVT, N));
02346 }
02347 
02348 void SelectionDAGBuilder::visitZExt(const User &I) {
02349   // ZExt cannot be a no-op cast because sizeof(src) < sizeof(dest).
02350   // ZExt also can't be a cast to bool for same reason. So, nothing much to do
02351   SDValue N = getValue(I.getOperand(0));
02352   EVT DestVT = DAG.getTargetLoweringInfo().getValueType(I.getType());
02353   setValue(&I, DAG.getNode(ISD::ZERO_EXTEND, getCurSDLoc(), DestVT, N));
02354 }
02355 
02356 void SelectionDAGBuilder::visitSExt(const User &I) {
02357   // SExt cannot be a no-op cast because sizeof(src) < sizeof(dest).
02358   // SExt also can't be a cast to bool for same reason. So, nothing much to do
02359   SDValue N = getValue(I.getOperand(0));
02360   EVT DestVT = DAG.getTargetLoweringInfo().getValueType(I.getType());
02361   setValue(&I, DAG.getNode(ISD::SIGN_EXTEND, getCurSDLoc(), DestVT, N));
02362 }
02363 
02364 void SelectionDAGBuilder::visitFPTrunc(const User &I) {
02365   // FPTrunc is never a no-op cast, no need to check
02366   SDValue N = getValue(I.getOperand(0));
02367   SDLoc dl = getCurSDLoc();
02368   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
02369   EVT DestVT = TLI.getValueType(I.getType());
02370   setValue(&I, DAG.getNode(ISD::FP_ROUND, dl, DestVT, N,
02371                            DAG.getTargetConstant(0, dl, TLI.getPointerTy())));
02372 }
02373 
02374 void SelectionDAGBuilder::visitFPExt(const User &I) {
02375   // FPExt is never a no-op cast, no need to check
02376   SDValue N = getValue(I.getOperand(0));
02377   EVT DestVT = DAG.getTargetLoweringInfo().getValueType(I.getType());
02378   setValue(&I, DAG.getNode(ISD::FP_EXTEND, getCurSDLoc(), DestVT, N));
02379 }
02380 
02381 void SelectionDAGBuilder::visitFPToUI(const User &I) {
02382   // FPToUI is never a no-op cast, no need to check
02383   SDValue N = getValue(I.getOperand(0));
02384   EVT DestVT = DAG.getTargetLoweringInfo().getValueType(I.getType());
02385   setValue(&I, DAG.getNode(ISD::FP_TO_UINT, getCurSDLoc(), DestVT, N));
02386 }
02387 
02388 void SelectionDAGBuilder::visitFPToSI(const User &I) {
02389   // FPToSI is never a no-op cast, no need to check
02390   SDValue N = getValue(I.getOperand(0));
02391   EVT DestVT = DAG.getTargetLoweringInfo().getValueType(I.getType());
02392   setValue(&I, DAG.getNode(ISD::FP_TO_SINT, getCurSDLoc(), DestVT, N));
02393 }
02394 
02395 void SelectionDAGBuilder::visitUIToFP(const User &I) {
02396   // UIToFP is never a no-op cast, no need to check
02397   SDValue N = getValue(I.getOperand(0));
02398   EVT DestVT = DAG.getTargetLoweringInfo().getValueType(I.getType());
02399   setValue(&I, DAG.getNode(ISD::UINT_TO_FP, getCurSDLoc(), DestVT, N));
02400 }
02401 
02402 void SelectionDAGBuilder::visitSIToFP(const User &I) {
02403   // SIToFP is never a no-op cast, no need to check
02404   SDValue N = getValue(I.getOperand(0));
02405   EVT DestVT = DAG.getTargetLoweringInfo().getValueType(I.getType());
02406   setValue(&I, DAG.getNode(ISD::SINT_TO_FP, getCurSDLoc(), DestVT, N));
02407 }
02408 
02409 void SelectionDAGBuilder::visitPtrToInt(const User &I) {
02410   // What to do depends on the size of the integer and the size of the pointer.
02411   // We can either truncate, zero extend, or no-op, accordingly.
02412   SDValue N = getValue(I.getOperand(0));
02413   EVT DestVT = DAG.getTargetLoweringInfo().getValueType(I.getType());
02414   setValue(&I, DAG.getZExtOrTrunc(N, getCurSDLoc(), DestVT));
02415 }
02416 
02417 void SelectionDAGBuilder::visitIntToPtr(const User &I) {
02418   // What to do depends on the size of the integer and the size of the pointer.
02419   // We can either truncate, zero extend, or no-op, accordingly.
02420   SDValue N = getValue(I.getOperand(0));
02421   EVT DestVT = DAG.getTargetLoweringInfo().getValueType(I.getType());
02422   setValue(&I, DAG.getZExtOrTrunc(N, getCurSDLoc(), DestVT));
02423 }
02424 
02425 void SelectionDAGBuilder::visitBitCast(const User &I) {
02426   SDValue N = getValue(I.getOperand(0));
02427   SDLoc dl = getCurSDLoc();
02428   EVT DestVT = DAG.getTargetLoweringInfo().getValueType(I.getType());
02429 
02430   // BitCast assures us that source and destination are the same size so this is
02431   // either a BITCAST or a no-op.
02432   if (DestVT != N.getValueType())
02433     setValue(&I, DAG.getNode(ISD::BITCAST, dl,
02434                              DestVT, N)); // convert types.
02435   // Check if the original LLVM IR Operand was a ConstantInt, because getValue()
02436   // might fold any kind of constant expression to an integer constant and that
02437   // is not what we are looking for. Only regcognize a bitcast of a genuine
02438   // constant integer as an opaque constant.
02439   else if(ConstantInt *C = dyn_cast<ConstantInt>(I.getOperand(0)))
02440     setValue(&I, DAG.getConstant(C->getValue(), dl, DestVT, /*isTarget=*/false,
02441                                  /*isOpaque*/true));
02442   else
02443     setValue(&I, N);            // noop cast.
02444 }
02445 
02446 void SelectionDAGBuilder::visitAddrSpaceCast(const User &I) {
02447   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
02448   const Value *SV = I.getOperand(0);
02449   SDValue N = getValue(SV);
02450   EVT DestVT = TLI.getValueType(I.getType());
02451 
02452   unsigned SrcAS = SV->getType()->getPointerAddressSpace();
02453   unsigned DestAS = I.getType()->getPointerAddressSpace();
02454 
02455   if (!TLI.isNoopAddrSpaceCast(SrcAS, DestAS))
02456     N = DAG.getAddrSpaceCast(getCurSDLoc(), DestVT, N, SrcAS, DestAS);
02457 
02458   setValue(&I, N);
02459 }
02460 
02461 void SelectionDAGBuilder::visitInsertElement(const User &I) {
02462   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
02463   SDValue InVec = getValue(I.getOperand(0));
02464   SDValue InVal = getValue(I.getOperand(1));
02465   SDValue InIdx = DAG.getSExtOrTrunc(getValue(I.getOperand(2)),
02466                                      getCurSDLoc(), TLI.getVectorIdxTy());
02467   setValue(&I, DAG.getNode(ISD::INSERT_VECTOR_ELT, getCurSDLoc(),
02468                            TLI.getValueType(I.getType()), InVec, InVal, InIdx));
02469 }
02470 
02471 void SelectionDAGBuilder::visitExtractElement(const User &I) {
02472   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
02473   SDValue InVec = getValue(I.getOperand(0));
02474   SDValue InIdx = DAG.getSExtOrTrunc(getValue(I.getOperand(1)),
02475                                      getCurSDLoc(), TLI.getVectorIdxTy());
02476   setValue(&I, DAG.getNode(ISD::EXTRACT_VECTOR_ELT, getCurSDLoc(),
02477                            TLI.getValueType(I.getType()), InVec, InIdx));
02478 }
02479 
02480 // Utility for visitShuffleVector - Return true if every element in Mask,
02481 // beginning from position Pos and ending in Pos+Size, falls within the
02482 // specified sequential range [L, L+Pos). or is undef.
02483 static bool isSequentialInRange(const SmallVectorImpl<int> &Mask,
02484                                 unsigned Pos, unsigned Size, int Low) {
02485   for (unsigned i = Pos, e = Pos+Size; i != e; ++i, ++Low)
02486     if (Mask[i] >= 0 && Mask[i] != Low)
02487       return false;
02488   return true;
02489 }
02490 
02491 void SelectionDAGBuilder::visitShuffleVector(const User &I) {
02492   SDValue Src1 = getValue(I.getOperand(0));
02493   SDValue Src2 = getValue(I.getOperand(1));
02494 
02495   SmallVector<int, 8> Mask;
02496   ShuffleVectorInst::getShuffleMask(cast<Constant>(I.getOperand(2)), Mask);
02497   unsigned MaskNumElts = Mask.size();
02498 
02499   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
02500   EVT VT = TLI.getValueType(I.getType());
02501   EVT SrcVT = Src1.getValueType();
02502   unsigned SrcNumElts = SrcVT.getVectorNumElements();
02503 
02504   if (SrcNumElts == MaskNumElts) {
02505     setValue(&I, DAG.getVectorShuffle(VT, getCurSDLoc(), Src1, Src2,
02506                                       &Mask[0]));
02507     return;
02508   }
02509 
02510   // Normalize the shuffle vector since mask and vector length don't match.
02511   if (SrcNumElts < MaskNumElts && MaskNumElts % SrcNumElts == 0) {
02512     // Mask is longer than the source vectors and is a multiple of the source
02513     // vectors.  We can use concatenate vector to make the mask and vectors
02514     // lengths match.
02515     if (SrcNumElts*2 == MaskNumElts) {
02516       // First check for Src1 in low and Src2 in high
02517       if (isSequentialInRange(Mask, 0, SrcNumElts, 0) &&
02518           isSequentialInRange(Mask, SrcNumElts, SrcNumElts, SrcNumElts)) {
02519         // The shuffle is concatenating two vectors together.
02520         setValue(&I, DAG.getNode(ISD::CONCAT_VECTORS, getCurSDLoc(),
02521                                  VT, Src1, Src2));
02522         return;
02523       }
02524       // Then check for Src2 in low and Src1 in high
02525       if (isSequentialInRange(Mask, 0, SrcNumElts, SrcNumElts) &&
02526           isSequentialInRange(Mask, SrcNumElts, SrcNumElts, 0)) {
02527         // The shuffle is concatenating two vectors together.
02528         setValue(&I, DAG.getNode(ISD::CONCAT_VECTORS, getCurSDLoc(),
02529                                  VT, Src2, Src1));
02530         return;
02531       }
02532     }
02533 
02534     // Pad both vectors with undefs to make them the same length as the mask.
02535     unsigned NumConcat = MaskNumElts / SrcNumElts;
02536     bool Src1U = Src1.getOpcode() == ISD::UNDEF;
02537     bool Src2U = Src2.getOpcode() == ISD::UNDEF;
02538     SDValue UndefVal = DAG.getUNDEF(SrcVT);
02539 
02540     SmallVector<SDValue, 8> MOps1(NumConcat, UndefVal);
02541     SmallVector<SDValue, 8> MOps2(NumConcat, UndefVal);
02542     MOps1[0] = Src1;
02543     MOps2[0] = Src2;
02544 
02545     Src1 = Src1U ? DAG.getUNDEF(VT) : DAG.getNode(ISD::CONCAT_VECTORS,
02546                                                   getCurSDLoc(), VT, MOps1);
02547     Src2 = Src2U ? DAG.getUNDEF(VT) : DAG.getNode(ISD::CONCAT_VECTORS,
02548                                                   getCurSDLoc(), VT, MOps2);
02549 
02550     // Readjust mask for new input vector length.
02551     SmallVector<int, 8> MappedOps;
02552     for (unsigned i = 0; i != MaskNumElts; ++i) {
02553       int Idx = Mask[i];
02554       if (Idx >= (int)SrcNumElts)
02555         Idx -= SrcNumElts - MaskNumElts;
02556       MappedOps.push_back(Idx);
02557     }
02558 
02559     setValue(&I, DAG.getVectorShuffle(VT, getCurSDLoc(), Src1, Src2,
02560                                       &MappedOps[0]));
02561     return;
02562   }
02563 
02564   if (SrcNumElts > MaskNumElts) {
02565     // Analyze the access pattern of the vector to see if we can extract
02566     // two subvectors and do the shuffle. The analysis is done by calculating
02567     // the range of elements the mask access on both vectors.
02568     int MinRange[2] = { static_cast<int>(SrcNumElts),
02569                         static_cast<int>(SrcNumElts)};
02570     int MaxRange[2] = {-1, -1};
02571 
02572     for (unsigned i = 0; i != MaskNumElts; ++i) {
02573       int Idx = Mask[i];
02574       unsigned Input = 0;
02575       if (Idx < 0)
02576         continue;
02577 
02578       if (Idx >= (int)SrcNumElts) {
02579         Input = 1;
02580         Idx -= SrcNumElts;
02581       }
02582       if (Idx > MaxRange[Input])
02583         MaxRange[Input] = Idx;
02584       if (Idx < MinRange[Input])
02585         MinRange[Input] = Idx;
02586     }
02587 
02588     // Check if the access is smaller than the vector size and can we find
02589     // a reasonable extract index.
02590     int RangeUse[2] = { -1, -1 };  // 0 = Unused, 1 = Extract, -1 = Can not
02591                                    // Extract.
02592     int StartIdx[2];  // StartIdx to extract from
02593     for (unsigned Input = 0; Input < 2; ++Input) {
02594       if (MinRange[Input] >= (int)SrcNumElts && MaxRange[Input] < 0) {
02595         RangeUse[Input] = 0; // Unused
02596         StartIdx[Input] = 0;
02597         continue;
02598       }
02599 
02600       // Find a good start index that is a multiple of the mask length. Then
02601       // see if the rest of the elements are in range.
02602       StartIdx[Input] = (MinRange[Input]/MaskNumElts)*MaskNumElts;
02603       if (MaxRange[Input] - StartIdx[Input] < (int)MaskNumElts &&
02604           StartIdx[Input] + MaskNumElts <= SrcNumElts)
02605         RangeUse[Input] = 1; // Extract from a multiple of the mask length.
02606     }
02607 
02608     if (RangeUse[0] == 0 && RangeUse[1] == 0) {
02609       setValue(&I, DAG.getUNDEF(VT)); // Vectors are not used.
02610       return;
02611     }
02612     if (RangeUse[0] >= 0 && RangeUse[1] >= 0) {
02613       // Extract appropriate subvector and generate a vector shuffle
02614       for (unsigned Input = 0; Input < 2; ++Input) {
02615         SDValue &Src = Input == 0 ? Src1 : Src2;
02616         if (RangeUse[Input] == 0)
02617           Src = DAG.getUNDEF(VT);
02618         else {
02619           SDLoc dl = getCurSDLoc();
02620           Src = DAG.getNode(
02621               ISD::EXTRACT_SUBVECTOR, dl, VT, Src,
02622               DAG.getConstant(StartIdx[Input], dl, TLI.getVectorIdxTy()));
02623         }
02624       }
02625 
02626       // Calculate new mask.
02627       SmallVector<int, 8> MappedOps;
02628       for (unsigned i = 0; i != MaskNumElts; ++i) {
02629         int Idx = Mask[i];
02630         if (Idx >= 0) {
02631           if (Idx < (int)SrcNumElts)
02632             Idx -= StartIdx[0];
02633           else
02634             Idx -= SrcNumElts + StartIdx[1] - MaskNumElts;
02635         }
02636         MappedOps.push_back(Idx);
02637       }
02638 
02639       setValue(&I, DAG.getVectorShuffle(VT, getCurSDLoc(), Src1, Src2,
02640                                         &MappedOps[0]));
02641       return;
02642     }
02643   }
02644 
02645   // We can't use either concat vectors or extract subvectors so fall back to
02646   // replacing the shuffle with extract and build vector.
02647   // to insert and build vector.
02648   EVT EltVT = VT.getVectorElementType();
02649   EVT IdxVT = TLI.getVectorIdxTy();
02650   SDLoc dl = getCurSDLoc();
02651   SmallVector<SDValue,8> Ops;
02652   for (unsigned i = 0; i != MaskNumElts; ++i) {
02653     int Idx = Mask[i];
02654     SDValue Res;
02655 
02656     if (Idx < 0) {
02657       Res = DAG.getUNDEF(EltVT);
02658     } else {
02659       SDValue &Src = Idx < (int)SrcNumElts ? Src1 : Src2;
02660       if (Idx >= (int)SrcNumElts) Idx -= SrcNumElts;
02661 
02662       Res = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl,
02663                         EltVT, Src, DAG.getConstant(Idx, dl, IdxVT));
02664     }
02665 
02666     Ops.push_back(Res);
02667   }
02668 
02669   setValue(&I, DAG.getNode(ISD::BUILD_VECTOR, dl, VT, Ops));
02670 }
02671 
02672 void SelectionDAGBuilder::visitInsertValue(const InsertValueInst &I) {
02673   const Value *Op0 = I.getOperand(0);
02674   const Value *Op1 = I.getOperand(1);
02675   Type *AggTy = I.getType();
02676   Type *ValTy = Op1->getType();
02677   bool IntoUndef = isa<UndefValue>(Op0);
02678   bool FromUndef = isa<UndefValue>(Op1);
02679 
02680   unsigned LinearIndex = ComputeLinearIndex(AggTy, I.getIndices());
02681 
02682   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
02683   SmallVector<EVT, 4> AggValueVTs;
02684   ComputeValueVTs(TLI, AggTy, AggValueVTs);
02685   SmallVector<EVT, 4> ValValueVTs;
02686   ComputeValueVTs(TLI, ValTy, ValValueVTs);
02687 
02688   unsigned NumAggValues = AggValueVTs.size();
02689   unsigned NumValValues = ValValueVTs.size();
02690   SmallVector<SDValue, 4> Values(NumAggValues);
02691 
02692   // Ignore an insertvalue that produces an empty object
02693   if (!NumAggValues) {
02694     setValue(&I, DAG.getUNDEF(MVT(MVT::Other)));
02695     return;
02696   }
02697 
02698   SDValue Agg = getValue(Op0);
02699   unsigned i = 0;
02700   // Copy the beginning value(s) from the original aggregate.
02701   for (; i != LinearIndex; ++i)
02702     Values[i] = IntoUndef ? DAG.getUNDEF(AggValueVTs[i]) :
02703                 SDValue(Agg.getNode(), Agg.getResNo() + i);
02704   // Copy values from the inserted value(s).
02705   if (NumValValues) {
02706     SDValue Val = getValue(Op1);
02707     for (; i != LinearIndex + NumValValues; ++i)
02708       Values[i] = FromUndef ? DAG.getUNDEF(AggValueVTs[i]) :
02709                   SDValue(Val.getNode(), Val.getResNo() + i - LinearIndex);
02710   }
02711   // Copy remaining value(s) from the original aggregate.
02712   for (; i != NumAggValues; ++i)
02713     Values[i] = IntoUndef ? DAG.getUNDEF(AggValueVTs[i]) :
02714                 SDValue(Agg.getNode(), Agg.getResNo() + i);
02715 
02716   setValue(&I, DAG.getNode(ISD::MERGE_VALUES, getCurSDLoc(),
02717                            DAG.getVTList(AggValueVTs), Values));
02718 }
02719 
02720 void SelectionDAGBuilder::visitExtractValue(const ExtractValueInst &I) {
02721   const Value *Op0 = I.getOperand(0);
02722   Type *AggTy = Op0->getType();
02723   Type *ValTy = I.getType();
02724   bool OutOfUndef = isa<UndefValue>(Op0);
02725 
02726   unsigned LinearIndex = ComputeLinearIndex(AggTy, I.getIndices());
02727 
02728   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
02729   SmallVector<EVT, 4> ValValueVTs;
02730   ComputeValueVTs(TLI, ValTy, ValValueVTs);
02731 
02732   unsigned NumValValues = ValValueVTs.size();
02733 
02734   // Ignore a extractvalue that produces an empty object
02735   if (!NumValValues) {
02736     setValue(&I, DAG.getUNDEF(MVT(MVT::Other)));
02737     return;
02738   }
02739 
02740   SmallVector<SDValue, 4> Values(NumValValues);
02741 
02742   SDValue Agg = getValue(Op0);
02743   // Copy out the selected value(s).
02744   for (unsigned i = LinearIndex; i != LinearIndex + NumValValues; ++i)
02745     Values[i - LinearIndex] =
02746       OutOfUndef ?
02747         DAG.getUNDEF(Agg.getNode()->getValueType(Agg.getResNo() + i)) :
02748         SDValue(Agg.getNode(), Agg.getResNo() + i);
02749 
02750   setValue(&I, DAG.getNode(ISD::MERGE_VALUES, getCurSDLoc(),
02751                            DAG.getVTList(ValValueVTs), Values));
02752 }
02753 
02754 void SelectionDAGBuilder::visitGetElementPtr(const User &I) {
02755   Value *Op0 = I.getOperand(0);
02756   // Note that the pointer operand may be a vector of pointers. Take the scalar
02757   // element which holds a pointer.
02758   Type *Ty = Op0->getType()->getScalarType();
02759   unsigned AS = Ty->getPointerAddressSpace();
02760   SDValue N = getValue(Op0);
02761   SDLoc dl = getCurSDLoc();
02762 
02763   for (GetElementPtrInst::const_op_iterator OI = I.op_begin()+1, E = I.op_end();
02764        OI != E; ++OI) {
02765     const Value *Idx = *OI;
02766     if (StructType *StTy = dyn_cast<StructType>(Ty)) {
02767       unsigned Field = cast<Constant>(Idx)->getUniqueInteger().getZExtValue();
02768       if (Field) {
02769         // N = N + Offset
02770         uint64_t Offset = DL->getStructLayout(StTy)->getElementOffset(Field);
02771         N = DAG.getNode(ISD::ADD, dl, N.getValueType(), N,
02772                         DAG.getConstant(Offset, dl, N.getValueType()));
02773       }
02774 
02775       Ty = StTy->getElementType(Field);
02776     } else {
02777       Ty = cast<SequentialType>(Ty)->getElementType();
02778       MVT PtrTy = DAG.getTargetLoweringInfo().getPointerTy(AS);
02779       unsigned PtrSize = PtrTy.getSizeInBits();
02780       APInt ElementSize(PtrSize, DL->getTypeAllocSize(Ty));
02781 
02782       // If this is a constant subscript, handle it quickly.
02783       if (const auto *CI = dyn_cast<ConstantInt>(Idx)) {
02784         if (CI->isZero())
02785           continue;
02786         APInt Offs = ElementSize * CI->getValue().sextOrTrunc(PtrSize);
02787         SDValue OffsVal = DAG.getConstant(Offs, dl, PtrTy);
02788         N = DAG.getNode(ISD::ADD, dl, N.getValueType(), N, OffsVal);
02789         continue;
02790       }
02791 
02792       // N = N + Idx * ElementSize;
02793       SDValue IdxN = getValue(Idx);
02794 
02795       // If the index is smaller or larger than intptr_t, truncate or extend
02796       // it.
02797       IdxN = DAG.getSExtOrTrunc(IdxN, dl, N.getValueType());
02798 
02799       // If this is a multiply by a power of two, turn it into a shl
02800       // immediately.  This is a very common case.
02801       if (ElementSize != 1) {
02802         if (ElementSize.isPowerOf2()) {
02803           unsigned Amt = ElementSize.logBase2();
02804           IdxN = DAG.getNode(ISD::SHL, dl,
02805                              N.getValueType(), IdxN,
02806                              DAG.getConstant(Amt, dl, IdxN.getValueType()));
02807         } else {
02808           SDValue Scale = DAG.getConstant(ElementSize, dl, IdxN.getValueType());
02809           IdxN = DAG.getNode(ISD::MUL, dl,
02810                              N.getValueType(), IdxN, Scale);
02811         }
02812       }
02813 
02814       N = DAG.getNode(ISD::ADD, dl,
02815                       N.getValueType(), N, IdxN);
02816     }
02817   }
02818 
02819   setValue(&I, N);
02820 }
02821 
02822 void SelectionDAGBuilder::visitAlloca(const AllocaInst &I) {
02823   // If this is a fixed sized alloca in the entry block of the function,
02824   // allocate it statically on the stack.
02825   if (FuncInfo.StaticAllocaMap.count(&I))
02826     return;   // getValue will auto-populate this.
02827 
02828   SDLoc dl = getCurSDLoc();
02829   Type *Ty = I.getAllocatedType();
02830   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
02831   uint64_t TySize = TLI.getDataLayout()->getTypeAllocSize(Ty);
02832   unsigned Align =
02833       std::max((unsigned)TLI.getDataLayout()->getPrefTypeAlignment(Ty),
02834                I.getAlignment());
02835 
02836   SDValue AllocSize = getValue(I.getArraySize());
02837 
02838   EVT IntPtr = TLI.getPointerTy();
02839   if (AllocSize.getValueType() != IntPtr)
02840     AllocSize = DAG.getZExtOrTrunc(AllocSize, dl, IntPtr);
02841 
02842   AllocSize = DAG.getNode(ISD::MUL, dl, IntPtr,
02843                           AllocSize,
02844                           DAG.getConstant(TySize, dl, IntPtr));
02845 
02846   // Handle alignment.  If the requested alignment is less than or equal to
02847   // the stack alignment, ignore it.  If the size is greater than or equal to
02848   // the stack alignment, we note this in the DYNAMIC_STACKALLOC node.
02849   unsigned StackAlign =
02850       DAG.getSubtarget().getFrameLowering()->getStackAlignment();
02851   if (Align <= StackAlign)
02852     Align = 0;
02853 
02854   // Round the size of the allocation up to the stack alignment size
02855   // by add SA-1 to the size.
02856   AllocSize = DAG.getNode(ISD::ADD, dl,
02857                           AllocSize.getValueType(), AllocSize,
02858                           DAG.getIntPtrConstant(StackAlign - 1, dl));
02859 
02860   // Mask out the low bits for alignment purposes.
02861   AllocSize = DAG.getNode(ISD::AND, dl,
02862                           AllocSize.getValueType(), AllocSize,
02863                           DAG.getIntPtrConstant(~(uint64_t)(StackAlign - 1),
02864                                                 dl));
02865 
02866   SDValue Ops[] = { getRoot(), AllocSize, DAG.getIntPtrConstant(Align, dl) };
02867   SDVTList VTs = DAG.getVTList(AllocSize.getValueType(), MVT::Other);
02868   SDValue DSA = DAG.getNode(ISD::DYNAMIC_STACKALLOC, dl, VTs, Ops);
02869   setValue(&I, DSA);
02870   DAG.setRoot(DSA.getValue(1));
02871 
02872   assert(FuncInfo.MF->getFrameInfo()->hasVarSizedObjects());
02873 }
02874 
02875 void SelectionDAGBuilder::visitLoad(const LoadInst &I) {
02876   if (I.isAtomic())
02877     return visitAtomicLoad(I);
02878 
02879   const Value *SV = I.getOperand(0);
02880   SDValue Ptr = getValue(SV);
02881 
02882   Type *Ty = I.getType();
02883 
02884   bool isVolatile = I.isVolatile();
02885   bool isNonTemporal = I.getMetadata(LLVMContext::MD_nontemporal) != nullptr;
02886 
02887   // The IR notion of invariant_load only guarantees that all *non-faulting*
02888   // invariant loads result in the same value.  The MI notion of invariant load
02889   // guarantees that the load can be legally moved to any location within its
02890   // containing function.  The MI notion of invariant_load is stronger than the
02891   // IR notion of invariant_load -- an MI invariant_load is an IR invariant_load
02892   // with a guarantee that the location being loaded from is dereferenceable
02893   // throughout the function's lifetime.
02894 
02895   bool isInvariant = I.getMetadata(LLVMContext::MD_invariant_load) != nullptr &&
02896     isDereferenceablePointer(SV, *DAG.getTarget().getDataLayout());
02897   unsigned Alignment = I.getAlignment();
02898 
02899   AAMDNodes AAInfo;
02900   I.getAAMetadata(AAInfo);
02901   const MDNode *Ranges = I.getMetadata(LLVMContext::MD_range);
02902 
02903   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
02904   SmallVector<EVT, 4> ValueVTs;
02905   SmallVector<uint64_t, 4> Offsets;
02906   ComputeValueVTs(TLI, Ty, ValueVTs, &Offsets);
02907   unsigned NumValues = ValueVTs.size();
02908   if (NumValues == 0)
02909     return;
02910 
02911   SDValue Root;
02912   bool ConstantMemory = false;
02913   if (isVolatile || NumValues > MaxParallelChains)
02914     // Serialize volatile loads with other side effects.
02915     Root = getRoot();
02916   else if (AA->pointsToConstantMemory(
02917                MemoryLocation(SV, AA->getTypeStoreSize(Ty), AAInfo))) {
02918     // Do not serialize (non-volatile) loads of constant memory with anything.
02919     Root = DAG.getEntryNode();
02920     ConstantMemory = true;
02921   } else {
02922     // Do not serialize non-volatile loads against each other.
02923     Root = DAG.getRoot();
02924   }
02925 
02926   SDLoc dl = getCurSDLoc();
02927 
02928   if (isVolatile)
02929     Root = TLI.prepareVolatileOrAtomicLoad(Root, dl, DAG);
02930 
02931   SmallVector<SDValue, 4> Values(NumValues);
02932   SmallVector<SDValue, 4> Chains(std::min(MaxParallelChains, NumValues));
02933   EVT PtrVT = Ptr.getValueType();
02934   unsigned ChainI = 0;
02935   for (unsigned i = 0; i != NumValues; ++i, ++ChainI) {
02936     // Serializing loads here may result in excessive register pressure, and
02937     // TokenFactor places arbitrary choke points on the scheduler. SD scheduling
02938     // could recover a bit by hoisting nodes upward in the chain by recognizing
02939     // they are side-effect free or do not alias. The optimizer should really
02940     // avoid this case by converting large object/array copies to llvm.memcpy
02941     // (MaxParallelChains should always remain as failsafe).
02942     if (ChainI == MaxParallelChains) {
02943       assert(PendingLoads.empty() && "PendingLoads must be serialized first");
02944       SDValue Chain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other,
02945                                   makeArrayRef(Chains.data(), ChainI));
02946       Root = Chain;
02947       ChainI = 0;
02948     }
02949     SDValue A = DAG.getNode(ISD::ADD, dl,
02950                             PtrVT, Ptr,
02951                             DAG.getConstant(Offsets[i], dl, PtrVT));
02952     SDValue L = DAG.getLoad(ValueVTs[i], dl, Root,
02953                             A, MachinePointerInfo(SV, Offsets[i]), isVolatile,
02954                             isNonTemporal, isInvariant, Alignment, AAInfo,
02955                             Ranges);
02956 
02957     Values[i] = L;
02958     Chains[ChainI] = L.getValue(1);
02959   }
02960 
02961   if (!ConstantMemory) {
02962     SDValue Chain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other,
02963                                 makeArrayRef(Chains.data(), ChainI));
02964     if (isVolatile)
02965       DAG.setRoot(Chain);
02966     else
02967       PendingLoads.push_back(Chain);
02968   }
02969 
02970   setValue(&I, DAG.getNode(ISD::MERGE_VALUES, dl,
02971                            DAG.getVTList(ValueVTs), Values));
02972 }
02973 
02974 void SelectionDAGBuilder::visitStore(const StoreInst &I) {
02975   if (I.isAtomic())
02976     return visitAtomicStore(I);
02977 
02978   const Value *SrcV = I.getOperand(0);
02979   const Value *PtrV = I.getOperand(1);
02980 
02981   SmallVector<EVT, 4> ValueVTs;
02982   SmallVector<uint64_t, 4> Offsets;
02983   ComputeValueVTs(DAG.getTargetLoweringInfo(), SrcV->getType(),
02984                   ValueVTs, &Offsets);
02985   unsigned NumValues = ValueVTs.size();
02986   if (NumValues == 0)
02987     return;
02988 
02989   // Get the lowered operands. Note that we do this after
02990   // checking if NumResults is zero, because with zero results
02991   // the operands won't have values in the map.
02992   SDValue Src = getValue(SrcV);
02993   SDValue Ptr = getValue(PtrV);
02994 
02995   SDValue Root = getRoot();
02996   SmallVector<SDValue, 4> Chains(std::min(MaxParallelChains, NumValues));
02997   EVT PtrVT = Ptr.getValueType();
02998   bool isVolatile = I.isVolatile();
02999   bool isNonTemporal = I.getMetadata(LLVMContext::MD_nontemporal) != nullptr;
03000   unsigned Alignment = I.getAlignment();
03001   SDLoc dl = getCurSDLoc();
03002 
03003   AAMDNodes AAInfo;
03004   I.getAAMetadata(AAInfo);
03005 
03006   unsigned ChainI = 0;
03007   for (unsigned i = 0; i != NumValues; ++i, ++ChainI) {
03008     // See visitLoad comments.
03009     if (ChainI == MaxParallelChains) {
03010       SDValue Chain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other,
03011                                   makeArrayRef(Chains.data(), ChainI));
03012       Root = Chain;
03013       ChainI = 0;
03014     }
03015     SDValue Add = DAG.getNode(ISD::ADD, dl, PtrVT, Ptr,
03016                               DAG.getConstant(Offsets[i], dl, PtrVT));
03017     SDValue St = DAG.getStore(Root, dl,
03018                               SDValue(Src.getNode(), Src.getResNo() + i),
03019                               Add, MachinePointerInfo(PtrV, Offsets[i]),
03020                               isVolatile, isNonTemporal, Alignment, AAInfo);
03021     Chains[ChainI] = St;
03022   }
03023 
03024   SDValue StoreNode = DAG.getNode(ISD::TokenFactor, dl, MVT::Other,
03025                                   makeArrayRef(Chains.data(), ChainI));
03026   DAG.setRoot(StoreNode);
03027 }
03028 
03029 void SelectionDAGBuilder::visitMaskedStore(const CallInst &I) {
03030   SDLoc sdl = getCurSDLoc();
03031 
03032   // llvm.masked.store.*(Src0, Ptr, alignemt, Mask)
03033   Value  *PtrOperand = I.getArgOperand(1);
03034   SDValue Ptr = getValue(PtrOperand);
03035   SDValue Src0 = getValue(I.getArgOperand(0));
03036   SDValue Mask = getValue(I.getArgOperand(3));
03037   EVT VT = Src0.getValueType();
03038   unsigned Alignment = (cast<ConstantInt>(I.getArgOperand(2)))->getZExtValue();
03039   if (!Alignment)
03040     Alignment = DAG.getEVTAlignment(VT);
03041 
03042   AAMDNodes AAInfo;
03043   I.getAAMetadata(AAInfo);
03044 
03045   MachineMemOperand *MMO =
03046     DAG.getMachineFunction().
03047     getMachineMemOperand(MachinePointerInfo(PtrOperand),
03048                           MachineMemOperand::MOStore,  VT.getStoreSize(),
03049                           Alignment, AAInfo);
03050   SDValue StoreNode = DAG.getMaskedStore(getRoot(), sdl, Src0, Ptr, Mask, VT,
03051                                          MMO, false);
03052   DAG.setRoot(StoreNode);
03053   setValue(&I, StoreNode);
03054 }
03055 
03056 // Gather/scatter receive a vector of pointers.
03057 // This vector of pointers may be represented as a base pointer + vector of 
03058 // indices, it depends on GEP and instruction preceeding GEP
03059 // that calculates indices
03060 static bool getUniformBase(Value *& Ptr, SDValue& Base, SDValue& Index,
03061                            SelectionDAGBuilder* SDB) {
03062 
03063   assert (Ptr->getType()->isVectorTy() && "Uexpected pointer type");
03064   GetElementPtrInst *Gep = dyn_cast<GetElementPtrInst>(Ptr);
03065   if (!Gep || Gep->getNumOperands() > 2)
03066     return false;
03067   ShuffleVectorInst *ShuffleInst = 
03068     dyn_cast<ShuffleVectorInst>(Gep->getPointerOperand());
03069   if (!ShuffleInst || !ShuffleInst->getMask()->isNullValue() ||
03070       cast<Instruction>(ShuffleInst->getOperand(0))->getOpcode() !=
03071       Instruction::InsertElement)
03072     return false;
03073 
03074   Ptr = cast<InsertElementInst>(ShuffleInst->getOperand(0))->getOperand(1);
03075 
03076   SelectionDAG& DAG = SDB->DAG;
03077   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
03078   // Check is the Ptr is inside current basic block
03079   // If not, look for the shuffle instruction
03080   if (SDB->findValue(Ptr))
03081     Base = SDB->getValue(Ptr);
03082   else if (SDB->findValue(ShuffleInst)) {
03083     SDValue ShuffleNode = SDB->getValue(ShuffleInst);
03084     SDLoc sdl = ShuffleNode;
03085     Base = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, sdl,
03086                        ShuffleNode.getValueType().getScalarType(), ShuffleNode,
03087                        DAG.getConstant(0, sdl, TLI.getVectorIdxTy()));
03088     SDB->setValue(Ptr, Base);
03089   }
03090   else
03091     return false;
03092 
03093   Value *IndexVal = Gep->getOperand(1);
03094   if (SDB->findValue(IndexVal)) {
03095     Index = SDB->getValue(IndexVal);
03096 
03097     if (SExtInst* Sext = dyn_cast<SExtInst>(IndexVal)) {
03098       IndexVal = Sext->getOperand(0);
03099       if (SDB->findValue(IndexVal))
03100         Index = SDB->getValue(IndexVal);
03101     }
03102     return true;
03103   }
03104   return false;
03105 }
03106 
03107 void SelectionDAGBuilder::visitMaskedScatter(const CallInst &I) {
03108   SDLoc sdl = getCurSDLoc();
03109 
03110   // llvm.masked.scatter.*(Src0, Ptrs, alignemt, Mask)
03111   Value  *Ptr = I.getArgOperand(1);
03112   SDValue Src0 = getValue(I.getArgOperand(0));
03113   SDValue Mask = getValue(I.getArgOperand(3));
03114   EVT VT = Src0.getValueType();
03115   unsigned Alignment = (cast<ConstantInt>(I.getArgOperand(2)))->getZExtValue();
03116   if (!Alignment)
03117     Alignment = DAG.getEVTAlignment(VT);
03118   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
03119 
03120   AAMDNodes AAInfo;
03121   I.getAAMetadata(AAInfo);
03122 
03123   SDValue Base;
03124   SDValue Index;
03125   Value *BasePtr = Ptr;
03126   bool UniformBase = getUniformBase(BasePtr, Base, Index, this);
03127 
03128   Value *MemOpBasePtr = UniformBase ? BasePtr : nullptr;
03129   MachineMemOperand *MMO = DAG.getMachineFunction().
03130     getMachineMemOperand(MachinePointerInfo(MemOpBasePtr),
03131                          MachineMemOperand::MOStore,  VT.getStoreSize(),
03132                          Alignment, AAInfo);
03133   if (!UniformBase) {
03134     Base = DAG.getTargetConstant(0, sdl, TLI.getPointerTy());
03135     Index = getValue(Ptr);
03136   }
03137   SDValue Ops[] = { getRoot(), Src0, Mask, Base, Index };
03138   SDValue Scatter = DAG.getMaskedScatter(DAG.getVTList(MVT::Other), VT, sdl,
03139                                          Ops, MMO);
03140   DAG.setRoot(Scatter);
03141   setValue(&I, Scatter);
03142 }
03143 
03144 void SelectionDAGBuilder::visitMaskedLoad(const CallInst &I) {
03145   SDLoc sdl = getCurSDLoc();
03146 
03147   // @llvm.masked.load.*(Ptr, alignment, Mask, Src0)
03148   Value  *PtrOperand = I.getArgOperand(0);
03149   SDValue Ptr = getValue(PtrOperand);
03150   SDValue Src0 = getValue(I.getArgOperand(3));
03151   SDValue Mask = getValue(I.getArgOperand(2));
03152 
03153   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
03154   EVT VT = TLI.getValueType(I.getType());
03155   unsigned Alignment = (cast<ConstantInt>(I.getArgOperand(1)))->getZExtValue();
03156   if (!Alignment)
03157     Alignment = DAG.getEVTAlignment(VT);
03158 
03159   AAMDNodes AAInfo;
03160   I.getAAMetadata(AAInfo);
03161   const MDNode *Ranges = I.getMetadata(LLVMContext::MD_range);
03162 
03163   SDValue InChain = DAG.getRoot();
03164   if (AA->pointsToConstantMemory(MemoryLocation(
03165           PtrOperand, AA->getTypeStoreSize(I.getType()), AAInfo))) {
03166     // Do not serialize (non-volatile) loads of constant memory with anything.
03167     InChain = DAG.getEntryNode();
03168   }
03169 
03170   MachineMemOperand *MMO =
03171     DAG.getMachineFunction().
03172     getMachineMemOperand(MachinePointerInfo(PtrOperand),
03173                           MachineMemOperand::MOLoad,  VT.getStoreSize(),
03174                           Alignment, AAInfo, Ranges);
03175 
03176   SDValue Load = DAG.getMaskedLoad(VT, sdl, InChain, Ptr, Mask, Src0, VT, MMO,
03177                                    ISD::NON_EXTLOAD);
03178   SDValue OutChain = Load.getValue(1);
03179   DAG.setRoot(OutChain);
03180   setValue(&I, Load);
03181 }
03182 
03183 void SelectionDAGBuilder::visitMaskedGather(const CallInst &I) {
03184   SDLoc sdl = getCurSDLoc();
03185 
03186   // @llvm.masked.gather.*(Ptrs, alignment, Mask, Src0)
03187   Value  *Ptr = I.getArgOperand(0);
03188   SDValue Src0 = getValue(I.getArgOperand(3));
03189   SDValue Mask = getValue(I.getArgOperand(2));
03190 
03191   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
03192   EVT VT = TLI.getValueType(I.getType());
03193   unsigned Alignment = (cast<ConstantInt>(I.getArgOperand(1)))->getZExtValue();
03194   if (!Alignment)
03195     Alignment = DAG.getEVTAlignment(VT);
03196 
03197   AAMDNodes AAInfo;
03198   I.getAAMetadata(AAInfo);
03199   const MDNode *Ranges = I.getMetadata(LLVMContext::MD_range);
03200 
03201   SDValue Root = DAG.getRoot();
03202   SDValue Base;
03203   SDValue Index;
03204   Value *BasePtr = Ptr;
03205   bool UniformBase = getUniformBase(BasePtr, Base, Index, this);
03206   bool ConstantMemory = false;
03207   if (UniformBase &&
03208       AA->pointsToConstantMemory(
03209           MemoryLocation(BasePtr, AA->getTypeStoreSize(I.getType()), AAInfo))) {
03210     // Do not serialize (non-volatile) loads of constant memory with anything.
03211     Root = DAG.getEntryNode();
03212     ConstantMemory = true;
03213   }
03214 
03215   MachineMemOperand *MMO =
03216     DAG.getMachineFunction().
03217     getMachineMemOperand(MachinePointerInfo(UniformBase ? BasePtr : nullptr),
03218                          MachineMemOperand::MOLoad,  VT.getStoreSize(),
03219                          Alignment, AAInfo, Ranges);
03220 
03221   if (!UniformBase) {
03222     Base = DAG.getTargetConstant(0, sdl, TLI.getPointerTy());
03223     Index = getValue(Ptr);
03224   }
03225   SDValue Ops[] = { Root, Src0, Mask, Base, Index };
03226   SDValue Gather = DAG.getMaskedGather(DAG.getVTList(VT, MVT::Other), VT, sdl,
03227                                        Ops, MMO);
03228 
03229   SDValue OutChain = Gather.getValue(1);
03230   if (!ConstantMemory)
03231     PendingLoads.push_back(OutChain);
03232   setValue(&I, Gather);
03233 }
03234 
03235 void SelectionDAGBuilder::visitAtomicCmpXchg(const AtomicCmpXchgInst &I) {
03236   SDLoc dl = getCurSDLoc();
03237   AtomicOrdering SuccessOrder = I.getSuccessOrdering();
03238   AtomicOrdering FailureOrder = I.getFailureOrdering();
03239   SynchronizationScope Scope = I.getSynchScope();
03240 
03241   SDValue InChain = getRoot();
03242 
03243   MVT MemVT = getValue(I.getCompareOperand()).getSimpleValueType();
03244   SDVTList VTs = DAG.getVTList(MemVT, MVT::i1, MVT::Other);
03245   SDValue L = DAG.getAtomicCmpSwap(
03246       ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS, dl, MemVT, VTs, InChain,
03247       getValue(I.getPointerOperand()), getValue(I.getCompareOperand()),
03248       getValue(I.getNewValOperand()), MachinePointerInfo(I.getPointerOperand()),
03249       /*Alignment=*/ 0, SuccessOrder, FailureOrder, Scope);
03250 
03251   SDValue OutChain = L.getValue(2);
03252 
03253   setValue(&I, L);
03254   DAG.setRoot(OutChain);
03255 }
03256 
03257 void SelectionDAGBuilder::visitAtomicRMW(const AtomicRMWInst &I) {
03258   SDLoc dl = getCurSDLoc();
03259   ISD::NodeType NT;
03260   switch (I.getOperation()) {
03261   default: llvm_unreachable("Unknown atomicrmw operation");
03262   case AtomicRMWInst::Xchg: NT = ISD::ATOMIC_SWAP; break;
03263   case AtomicRMWInst::Add:  NT = ISD::ATOMIC_LOAD_ADD; break;
03264   case AtomicRMWInst::Sub:  NT = ISD::ATOMIC_LOAD_SUB; break;
03265   case AtomicRMWInst::And:  NT = ISD::ATOMIC_LOAD_AND; break;
03266   case AtomicRMWInst::Nand: NT = ISD::ATOMIC_LOAD_NAND; break;
03267   case AtomicRMWInst::Or:   NT = ISD::ATOMIC_LOAD_OR; break;
03268   case AtomicRMWInst::Xor:  NT = ISD::ATOMIC_LOAD_XOR; break;
03269   case AtomicRMWInst::Max:  NT = ISD::ATOMIC_LOAD_MAX; break;
03270   case AtomicRMWInst::Min:  NT = ISD::ATOMIC_LOAD_MIN; break;
03271   case AtomicRMWInst::UMax: NT = ISD::ATOMIC_LOAD_UMAX; break;
03272   case AtomicRMWInst::UMin: NT = ISD::ATOMIC_LOAD_UMIN; break;
03273   }
03274   AtomicOrdering Order = I.getOrdering();
03275   SynchronizationScope Scope = I.getSynchScope();
03276 
03277   SDValue InChain = getRoot();
03278 
03279   SDValue L =
03280     DAG.getAtomic(NT, dl,
03281                   getValue(I.getValOperand()).getSimpleValueType(),
03282                   InChain,
03283                   getValue(I.getPointerOperand()),
03284                   getValue(I.getValOperand()),
03285                   I.getPointerOperand(),
03286                   /* Alignment=*/ 0, Order, Scope);
03287 
03288   SDValue OutChain = L.getValue(1);
03289 
03290   setValue(&I, L);
03291   DAG.setRoot(OutChain);
03292 }
03293 
03294 void SelectionDAGBuilder::visitFence(const FenceInst &I) {
03295   SDLoc dl = getCurSDLoc();
03296   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
03297   SDValue Ops[3];
03298   Ops[0] = getRoot();
03299   Ops[1] = DAG.getConstant(I.getOrdering(), dl, TLI.getPointerTy());
03300   Ops[2] = DAG.getConstant(I.getSynchScope(), dl, TLI.getPointerTy());
03301   DAG.setRoot(DAG.getNode(ISD::ATOMIC_FENCE, dl, MVT::Other, Ops));
03302 }
03303 
03304 void SelectionDAGBuilder::visitAtomicLoad(const LoadInst &I) {
03305   SDLoc dl = getCurSDLoc();
03306   AtomicOrdering Order = I.getOrdering();
03307   SynchronizationScope Scope = I.getSynchScope();
03308 
03309   SDValue InChain = getRoot();
03310 
03311   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
03312   EVT VT = TLI.getValueType(I.getType());
03313 
03314   if (I.getAlignment() < VT.getSizeInBits() / 8)
03315     report_fatal_error("Cannot generate unaligned atomic load");
03316 
03317   MachineMemOperand *MMO =
03318       DAG.getMachineFunction().
03319       getMachineMemOperand(MachinePointerInfo(I.getPointerOperand()),
03320                            MachineMemOperand::MOVolatile |
03321                            MachineMemOperand::MOLoad,
03322                            VT.getStoreSize(),
03323                            I.getAlignment() ? I.getAlignment() :
03324                                               DAG.getEVTAlignment(VT));
03325 
03326   InChain = TLI.prepareVolatileOrAtomicLoad(InChain, dl, DAG);
03327   SDValue L =
03328       DAG.getAtomic(ISD::ATOMIC_LOAD, dl, VT, VT, InChain,
03329                     getValue(I.getPointerOperand()), MMO,
03330                     Order, Scope);
03331 
03332   SDValue OutChain = L.getValue(1);
03333 
03334   setValue(&I, L);
03335   DAG.setRoot(OutChain);
03336 }
03337 
03338 void SelectionDAGBuilder::visitAtomicStore(const StoreInst &I) {
03339   SDLoc dl = getCurSDLoc();
03340 
03341   AtomicOrdering Order = I.getOrdering();
03342   SynchronizationScope Scope = I.getSynchScope();
03343 
03344   SDValue InChain = getRoot();
03345 
03346   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
03347   EVT VT = TLI.getValueType(I.getValueOperand()->getType());
03348 
03349   if (I.getAlignment() < VT.getSizeInBits() / 8)
03350     report_fatal_error("Cannot generate unaligned atomic store");
03351 
03352   SDValue OutChain =
03353     DAG.getAtomic(ISD::ATOMIC_STORE, dl, VT,
03354                   InChain,
03355                   getValue(I.getPointerOperand()),
03356                   getValue(I.getValueOperand()),
03357                   I.getPointerOperand(), I.getAlignment(),
03358                   Order, Scope);
03359 
03360   DAG.setRoot(OutChain);
03361 }
03362 
03363 /// visitTargetIntrinsic - Lower a call of a target intrinsic to an INTRINSIC
03364 /// node.
03365 void SelectionDAGBuilder::visitTargetIntrinsic(const CallInst &I,
03366                                                unsigned Intrinsic) {
03367   bool HasChain = !I.doesNotAccessMemory();
03368   bool OnlyLoad = HasChain && I.onlyReadsMemory();
03369 
03370   // Build the operand list.
03371   SmallVector<SDValue, 8> Ops;
03372   if (HasChain) {  // If this intrinsic has side-effects, chainify it.
03373     if (OnlyLoad) {
03374       // We don't need to serialize loads against other loads.
03375       Ops.push_back(DAG.getRoot());
03376     } else {
03377       Ops.push_back(getRoot());
03378     }
03379   }
03380 
03381   // Info is set by getTgtMemInstrinsic
03382   TargetLowering::IntrinsicInfo Info;
03383   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
03384   bool IsTgtIntrinsic = TLI.getTgtMemIntrinsic(Info, I, Intrinsic);
03385 
03386   // Add the intrinsic ID as an integer operand if it's not a target intrinsic.
03387   if (!IsTgtIntrinsic || Info.opc == ISD::INTRINSIC_VOID ||
03388       Info.opc == ISD::INTRINSIC_W_CHAIN)
03389     Ops.push_back(DAG.getTargetConstant(Intrinsic, getCurSDLoc(),
03390                                         TLI.getPointerTy()));
03391 
03392   // Add all operands of the call to the operand list.
03393   for (unsigned i = 0, e = I.getNumArgOperands(); i != e; ++i) {
03394     SDValue Op = getValue(I.getArgOperand(i));
03395     Ops.push_back(Op);
03396   }
03397 
03398   SmallVector<EVT, 4> ValueVTs;
03399   ComputeValueVTs(TLI, I.getType(), ValueVTs);
03400 
03401   if (HasChain)
03402     ValueVTs.push_back(MVT::Other);
03403 
03404   SDVTList VTs = DAG.getVTList(ValueVTs);
03405 
03406   // Create the node.
03407   SDValue Result;
03408   if (IsTgtIntrinsic) {
03409     // This is target intrinsic that touches memory
03410     Result = DAG.getMemIntrinsicNode(Info.opc, getCurSDLoc(),
03411                                      VTs, Ops, Info.memVT,
03412                                    MachinePointerInfo(Info.ptrVal, Info.offset),
03413                                      Info.align, Info.vol,
03414                                      Info.readMem, Info.writeMem, Info.size);
03415   } else if (!HasChain) {
03416     Result = DAG.getNode(ISD::INTRINSIC_WO_CHAIN, getCurSDLoc(), VTs, Ops);
03417   } else if (!I.getType()->isVoidTy()) {
03418     Result = DAG.getNode(ISD::INTRINSIC_W_CHAIN, getCurSDLoc(), VTs, Ops);
03419   } else {
03420     Result = DAG.getNode(ISD::INTRINSIC_VOID, getCurSDLoc(), VTs, Ops);
03421   }
03422 
03423   if (HasChain) {
03424     SDValue Chain = Result.getValue(Result.getNode()->getNumValues()-1);
03425     if (OnlyLoad)
03426       PendingLoads.push_back(Chain);
03427     else
03428       DAG.setRoot(Chain);
03429   }
03430 
03431   if (!I.getType()->isVoidTy()) {
03432     if (VectorType *PTy = dyn_cast<VectorType>(I.getType())) {
03433       EVT VT = TLI.getValueType(PTy);
03434       Result = DAG.getNode(ISD::BITCAST, getCurSDLoc(), VT, Result);
03435     }
03436 
03437     setValue(&I, Result);
03438   }
03439 }
03440 
03441 /// GetSignificand - Get the significand and build it into a floating-point
03442 /// number with exponent of 1:
03443 ///
03444 ///   Op = (Op & 0x007fffff) | 0x3f800000;
03445 ///
03446 /// where Op is the hexadecimal representation of floating point value.
03447 static SDValue
03448 GetSignificand(SelectionDAG &DAG, SDValue Op, SDLoc dl) {
03449   SDValue t1 = DAG.getNode(ISD::AND, dl, MVT::i32, Op,
03450                            DAG.getConstant(0x007fffff, dl, MVT::i32));
03451   SDValue t2 = DAG.getNode(ISD::OR, dl, MVT::i32, t1,
03452                            DAG.getConstant(0x3f800000, dl, MVT::i32));
03453   return DAG.getNode(ISD::BITCAST, dl, MVT::f32, t2);
03454 }
03455 
03456 /// GetExponent - Get the exponent:
03457 ///
03458 ///   (float)(int)(((Op & 0x7f800000) >> 23) - 127);
03459 ///
03460 /// where Op is the hexadecimal representation of floating point value.
03461 static SDValue
03462 GetExponent(SelectionDAG &DAG, SDValue Op, const TargetLowering &TLI,
03463             SDLoc dl) {
03464   SDValue t0 = DAG.getNode(ISD::AND, dl, MVT::i32, Op,
03465                            DAG.getConstant(0x7f800000, dl, MVT::i32));
03466   SDValue t1 = DAG.getNode(ISD::SRL, dl, MVT::i32, t0,
03467                            DAG.getConstant(23, dl, TLI.getPointerTy()));
03468   SDValue t2 = DAG.getNode(ISD::SUB, dl, MVT::i32, t1,
03469                            DAG.getConstant(127, dl, MVT::i32));
03470   return DAG.getNode(ISD::SINT_TO_FP, dl, MVT::f32, t2);
03471 }
03472 
03473 /// getF32Constant - Get 32-bit floating point constant.
03474 static SDValue
03475 getF32Constant(SelectionDAG &DAG, unsigned Flt, SDLoc dl) {
03476   return DAG.getConstantFP(APFloat(APFloat::IEEEsingle, APInt(32, Flt)), dl,
03477                            MVT::f32);
03478 }
03479 
03480 static SDValue getLimitedPrecisionExp2(SDValue t0, SDLoc dl,
03481                                        SelectionDAG &DAG) {
03482   //   IntegerPartOfX = ((int32_t)(t0);
03483   SDValue IntegerPartOfX = DAG.getNode(ISD::FP_TO_SINT, dl, MVT::i32, t0);
03484 
03485   //   FractionalPartOfX = t0 - (float)IntegerPartOfX;
03486   SDValue t1 = DAG.getNode(ISD::SINT_TO_FP, dl, MVT::f32, IntegerPartOfX);
03487   SDValue X = DAG.getNode(ISD::FSUB, dl, MVT::f32, t0, t1);
03488 
03489   //   IntegerPartOfX <<= 23;
03490   IntegerPartOfX = DAG.getNode(
03491       ISD::SHL, dl, MVT::i32, IntegerPartOfX,
03492       DAG.getConstant(23, dl, DAG.getTargetLoweringInfo().getPointerTy()));
03493 
03494   SDValue TwoToFractionalPartOfX;
03495   if (LimitFloatPrecision <= 6) {
03496     // For floating-point precision of 6:
03497     //
03498     //   TwoToFractionalPartOfX =
03499     //     0.997535578f +
03500     //       (0.735607626f + 0.252464424f * x) * x;
03501     //
03502     // error 0.0144103317, which is 6 bits
03503     SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X,
03504                              getF32Constant(DAG, 0x3e814304, dl));
03505     SDValue t3 = DAG.getNode(ISD::FADD, dl, MVT::f32, t2,
03506                              getF32Constant(DAG, 0x3f3c50c8, dl));
03507     SDValue t4 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t3, X);
03508     TwoToFractionalPartOfX = DAG.getNode(ISD::FADD, dl, MVT::f32, t4,
03509                                          getF32Constant(DAG, 0x3f7f5e7e, dl));
03510   } else if (LimitFloatPrecision <= 12) {
03511     // For floating-point precision of 12:
03512     //
03513     //   TwoToFractionalPartOfX =
03514     //     0.999892986f +
03515     //       (0.696457318f +
03516     //         (0.224338339f + 0.792043434e-1f * x) * x) * x;
03517     //
03518     // error 0.000107046256, which is 13 to 14 bits
03519     SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X,
03520                              getF32Constant(DAG, 0x3da235e3, dl));
03521     SDValue t3 = DAG.getNode(ISD::FADD, dl, MVT::f32, t2,
03522                              getF32Constant(DAG, 0x3e65b8f3, dl));
03523     SDValue t4 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t3, X);
03524     SDValue t5 = DAG.getNode(ISD::FADD, dl, MVT::f32, t4,
03525                              getF32Constant(DAG, 0x3f324b07, dl));
03526     SDValue t6 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t5, X);
03527     TwoToFractionalPartOfX = DAG.getNode(ISD::FADD, dl, MVT::f32, t6,
03528                                          getF32Constant(DAG, 0x3f7ff8fd, dl));
03529   } else { // LimitFloatPrecision <= 18
03530     // For floating-point precision of 18:
03531     //
03532     //   TwoToFractionalPartOfX =
03533     //     0.999999982f +
03534     //       (0.693148872f +
03535     //         (0.240227044f +
03536     //           (0.554906021e-1f +
03537     //             (0.961591928e-2f +
03538     //               (0.136028312e-2f + 0.157059148e-3f *x)*x)*x)*x)*x)*x;
03539     // error 2.47208000*10^(-7), which is better than 18 bits
03540     SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X,
03541                              getF32Constant(DAG, 0x3924b03e, dl));
03542     SDValue t3 = DAG.getNode(ISD::FADD, dl, MVT::f32, t2,
03543                              getF32Constant(DAG, 0x3ab24b87, dl));
03544     SDValue t4 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t3, X);
03545     SDValue t5 = DAG.getNode(ISD::FADD, dl, MVT::f32, t4,
03546                              getF32Constant(DAG, 0x3c1d8c17, dl));
03547     SDValue t6 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t5, X);
03548     SDValue t7 = DAG.getNode(ISD::FADD, dl, MVT::f32, t6,
03549                              getF32Constant(DAG, 0x3d634a1d, dl));
03550     SDValue t8 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t7, X);
03551     SDValue t9 = DAG.getNode(ISD::FADD, dl, MVT::f32, t8,
03552                              getF32Constant(DAG, 0x3e75fe14, dl));
03553     SDValue t10 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t9, X);
03554     SDValue t11 = DAG.getNode(ISD::FADD, dl, MVT::f32, t10,
03555                               getF32Constant(DAG, 0x3f317234, dl));
03556     SDValue t12 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t11, X);
03557     TwoToFractionalPartOfX = DAG.getNode(ISD::FADD, dl, MVT::f32, t12,
03558                                          getF32Constant(DAG, 0x3f800000, dl));
03559   }
03560 
03561   // Add the exponent into the result in integer domain.
03562   SDValue t13 = DAG.getNode(ISD::BITCAST, dl, MVT::i32, TwoToFractionalPartOfX);
03563   return DAG.getNode(ISD::BITCAST, dl, MVT::f32,
03564                      DAG.getNode(ISD::ADD, dl, MVT::i32, t13, IntegerPartOfX));
03565 }
03566 
03567 /// expandExp - Lower an exp intrinsic. Handles the special sequences for
03568 /// limited-precision mode.
03569 static SDValue expandExp(SDLoc dl, SDValue Op, SelectionDAG &DAG,
03570                          const TargetLowering &TLI) {
03571   if (Op.getValueType() == MVT::f32 &&
03572       LimitFloatPrecision > 0 && LimitFloatPrecision <= 18) {
03573 
03574     // Put the exponent in the right bit position for later addition to the
03575     // final result:
03576     //
03577     //   #define LOG2OFe 1.4426950f
03578     //   t0 = Op * LOG2OFe
03579     SDValue t0 = DAG.getNode(ISD::FMUL, dl, MVT::f32, Op,
03580                              getF32Constant(DAG, 0x3fb8aa3b, dl));
03581     return getLimitedPrecisionExp2(t0, dl, DAG);
03582   }
03583 
03584   // No special expansion.
03585   return DAG.getNode(ISD::FEXP, dl, Op.getValueType(), Op);
03586 }
03587 
03588 /// expandLog - Lower a log intrinsic. Handles the special sequences for
03589 /// limited-precision mode.
03590 static SDValue expandLog(SDLoc dl, SDValue Op, SelectionDAG &DAG,
03591                          const TargetLowering &TLI) {
03592   if (Op.getValueType() == MVT::f32 &&
03593       LimitFloatPrecision > 0 && LimitFloatPrecision <= 18) {
03594     SDValue Op1 = DAG.getNode(ISD::BITCAST, dl, MVT::i32, Op);
03595 
03596     // Scale the exponent by log(2) [0.69314718f].
03597     SDValue Exp = GetExponent(DAG, Op1, TLI, dl);
03598     SDValue LogOfExponent = DAG.getNode(ISD::FMUL, dl, MVT::f32, Exp,
03599                                         getF32Constant(DAG, 0x3f317218, dl));
03600 
03601     // Get the significand and build it into a floating-point number with
03602     // exponent of 1.
03603     SDValue X = GetSignificand(DAG, Op1, dl);
03604 
03605     SDValue LogOfMantissa;
03606     if (LimitFloatPrecision <= 6) {
03607       // For floating-point precision of 6:
03608       //
03609       //   LogofMantissa =
03610       //     -1.1609546f +
03611       //       (1.4034025f - 0.23903021f * x) * x;
03612       //
03613       // error 0.0034276066, which is better than 8 bits
03614       SDValue t0 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X,
03615                                getF32Constant(DAG, 0xbe74c456, dl));
03616       SDValue t1 = DAG.getNode(ISD::FADD, dl, MVT::f32, t0,
03617                                getF32Constant(DAG, 0x3fb3a2b1, dl));
03618       SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t1, X);
03619       LogOfMantissa = DAG.getNode(ISD::FSUB, dl, MVT::f32, t2,
03620                                   getF32Constant(DAG, 0x3f949a29, dl));
03621     } else if (LimitFloatPrecision <= 12) {
03622       // For floating-point precision of 12:
03623       //
03624       //   LogOfMantissa =
03625       //     -1.7417939f +
03626       //       (2.8212026f +
03627       //         (-1.4699568f +
03628       //           (0.44717955f - 0.56570851e-1f * x) * x) * x) * x;
03629       //
03630       // error 0.000061011436, which is 14 bits
03631       SDValue t0 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X,
03632                                getF32Constant(DAG, 0xbd67b6d6, dl));
03633       SDValue t1 = DAG.getNode(ISD::FADD, dl, MVT::f32, t0,
03634                                getF32Constant(DAG, 0x3ee4f4b8, dl));
03635       SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t1, X);
03636       SDValue t3 = DAG.getNode(ISD::FSUB, dl, MVT::f32, t2,
03637                                getF32Constant(DAG, 0x3fbc278b, dl));
03638       SDValue t4 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t3, X);
03639       SDValue t5 = DAG.getNode(ISD::FADD, dl, MVT::f32, t4,
03640                                getF32Constant(DAG, 0x40348e95, dl));
03641       SDValue t6 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t5, X);
03642       LogOfMantissa = DAG.getNode(ISD::FSUB, dl, MVT::f32, t6,
03643                                   getF32Constant(DAG, 0x3fdef31a, dl));
03644     } else { // LimitFloatPrecision <= 18
03645       // For floating-point precision of 18:
03646       //
03647       //   LogOfMantissa =
03648       //     -2.1072184f +
03649       //       (4.2372794f +
03650       //         (-3.7029485f +
03651       //           (2.2781945f +
03652       //             (-0.87823314f +
03653       //               (0.19073739f - 0.17809712e-1f * x) * x) * x) * x) * x)*x;
03654       //
03655       // error 0.0000023660568, which is better than 18 bits
03656       SDValue t0 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X,
03657                                getF32Constant(DAG, 0xbc91e5ac, dl));
03658       SDValue t1 = DAG.getNode(ISD::FADD, dl, MVT::f32, t0,
03659                                getF32Constant(DAG, 0x3e4350aa, dl));
03660       SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t1, X);
03661       SDValue t3 = DAG.getNode(ISD::FSUB, dl, MVT::f32, t2,
03662                                getF32Constant(DAG, 0x3f60d3e3, dl));
03663       SDValue t4 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t3, X);
03664       SDValue t5 = DAG.getNode(ISD::FADD, dl, MVT::f32, t4,
03665                                getF32Constant(DAG, 0x4011cdf0, dl));
03666       SDValue t6 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t5, X);
03667       SDValue t7 = DAG.getNode(ISD::FSUB, dl, MVT::f32, t6,
03668                                getF32Constant(DAG, 0x406cfd1c, dl));
03669       SDValue t8 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t7, X);
03670       SDValue t9 = DAG.getNode(ISD::FADD, dl, MVT::f32, t8,
03671                                getF32Constant(DAG, 0x408797cb, dl));
03672       SDValue t10 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t9, X);
03673       LogOfMantissa = DAG.getNode(ISD::FSUB, dl, MVT::f32, t10,
03674                                   getF32Constant(DAG, 0x4006dcab, dl));
03675     }
03676 
03677     return DAG.getNode(ISD::FADD, dl, MVT::f32, LogOfExponent, LogOfMantissa);
03678   }
03679 
03680   // No special expansion.
03681   return DAG.getNode(ISD::FLOG, dl, Op.getValueType(), Op);
03682 }
03683 
03684 /// expandLog2 - Lower a log2 intrinsic. Handles the special sequences for
03685 /// limited-precision mode.
03686 static SDValue expandLog2(SDLoc dl, SDValue Op, SelectionDAG &DAG,
03687                           const TargetLowering &TLI) {
03688   if (Op.getValueType() == MVT::f32 &&
03689       LimitFloatPrecision > 0 && LimitFloatPrecision <= 18) {
03690     SDValue Op1 = DAG.getNode(ISD::BITCAST, dl, MVT::i32, Op);
03691 
03692     // Get the exponent.
03693     SDValue LogOfExponent = GetExponent(DAG, Op1, TLI, dl);
03694 
03695     // Get the significand and build it into a floating-point number with
03696     // exponent of 1.
03697     SDValue X = GetSignificand(DAG, Op1, dl);
03698 
03699     // Different possible minimax approximations of significand in
03700     // floating-point for various degrees of accuracy over [1,2].
03701     SDValue Log2ofMantissa;
03702     if (LimitFloatPrecision <= 6) {
03703       // For floating-point precision of 6:
03704       //
03705       //   Log2ofMantissa = -1.6749035f + (2.0246817f - .34484768f * x) * x;
03706       //
03707       // error 0.0049451742, which is more than 7 bits
03708       SDValue t0 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X,
03709                                getF32Constant(DAG, 0xbeb08fe0, dl));
03710       SDValue t1 = DAG.getNode(ISD::FADD, dl, MVT::f32, t0,
03711                                getF32Constant(DAG, 0x40019463, dl));
03712       SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t1, X);
03713       Log2ofMantissa = DAG.getNode(ISD::FSUB, dl, MVT::f32, t2,
03714                                    getF32Constant(DAG, 0x3fd6633d, dl));
03715     } else if (LimitFloatPrecision <= 12) {
03716       // For floating-point precision of 12:
03717       //
03718       //   Log2ofMantissa =
03719       //     -2.51285454f +
03720       //       (4.07009056f +
03721       //         (-2.12067489f +
03722       //           (.645142248f - 0.816157886e-1f * x) * x) * x) * x;
03723       //
03724       // error 0.0000876136000, which is better than 13 bits
03725       SDValue t0 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X,
03726                                getF32Constant(DAG, 0xbda7262e, dl));
03727       SDValue t1 = DAG.getNode(ISD::FADD, dl, MVT::f32, t0,
03728                                getF32Constant(DAG, 0x3f25280b, dl));
03729       SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t1, X);
03730       SDValue t3 = DAG.getNode(ISD::FSUB, dl, MVT::f32, t2,
03731                                getF32Constant(DAG, 0x4007b923, dl));
03732       SDValue t4 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t3, X);
03733       SDValue t5 = DAG.getNode(ISD::FADD, dl, MVT::f32, t4,
03734                                getF32Constant(DAG, 0x40823e2f, dl));
03735       SDValue t6 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t5, X);
03736       Log2ofMantissa = DAG.getNode(ISD::FSUB, dl, MVT::f32, t6,
03737                                    getF32Constant(DAG, 0x4020d29c, dl));
03738     } else { // LimitFloatPrecision <= 18
03739       // For floating-point precision of 18:
03740       //
03741       //   Log2ofMantissa =
03742       //     -3.0400495f +
03743       //       (6.1129976f +
03744       //         (-5.3420409f +
03745       //           (3.2865683f +
03746       //             (-1.2669343f +
03747       //               (0.27515199f -
03748       //                 0.25691327e-1f * x) * x) * x) * x) * x) * x;
03749       //
03750       // error 0.0000018516, which is better than 18 bits
03751       SDValue t0 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X,
03752                                getF32Constant(DAG, 0xbcd2769e, dl));
03753       SDValue t1 = DAG.getNode(ISD::FADD, dl, MVT::f32, t0,
03754                                getF32Constant(DAG, 0x3e8ce0b9, dl));
03755       SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t1, X);
03756       SDValue t3 = DAG.getNode(ISD::FSUB, dl, MVT::f32, t2,
03757                                getF32Constant(DAG, 0x3fa22ae7, dl));
03758       SDValue t4 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t3, X);
03759       SDValue t5 = DAG.getNode(ISD::FADD, dl, MVT::f32, t4,
03760                                getF32Constant(DAG, 0x40525723, dl));
03761       SDValue t6 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t5, X);
03762       SDValue t7 = DAG.getNode(ISD::FSUB, dl, MVT::f32, t6,
03763                                getF32Constant(DAG, 0x40aaf200, dl));
03764       SDValue t8 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t7, X);
03765       SDValue t9 = DAG.getNode(ISD::FADD, dl, MVT::f32, t8,
03766                                getF32Constant(DAG, 0x40c39dad, dl));
03767       SDValue t10 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t9, X);
03768       Log2ofMantissa = DAG.getNode(ISD::FSUB, dl, MVT::f32, t10,
03769                                    getF32Constant(DAG, 0x4042902c, dl));
03770     }
03771 
03772     return DAG.getNode(ISD::FADD, dl, MVT::f32, LogOfExponent, Log2ofMantissa);
03773   }
03774 
03775   // No special expansion.
03776   return DAG.getNode(ISD::FLOG2, dl, Op.getValueType(), Op);
03777 }
03778 
03779 /// expandLog10 - Lower a log10 intrinsic. Handles the special sequences for
03780 /// limited-precision mode.
03781 static SDValue expandLog10(SDLoc dl, SDValue Op, SelectionDAG &DAG,
03782                            const TargetLowering &TLI) {
03783   if (Op.getValueType() == MVT::f32 &&
03784       LimitFloatPrecision > 0 && LimitFloatPrecision <= 18) {
03785     SDValue Op1 = DAG.getNode(ISD::BITCAST, dl, MVT::i32, Op);
03786 
03787     // Scale the exponent by log10(2) [0.30102999f].
03788     SDValue Exp = GetExponent(DAG, Op1, TLI, dl);
03789     SDValue LogOfExponent = DAG.getNode(ISD::FMUL, dl, MVT::f32, Exp,
03790                                         getF32Constant(DAG, 0x3e9a209a, dl));
03791 
03792     // Get the significand and build it into a floating-point number with
03793     // exponent of 1.
03794     SDValue X = GetSignificand(DAG, Op1, dl);
03795 
03796     SDValue Log10ofMantissa;
03797     if (LimitFloatPrecision <= 6) {
03798       // For floating-point precision of 6:
03799       //
03800       //   Log10ofMantissa =
03801       //     -0.50419619f +
03802       //       (0.60948995f - 0.10380950f * x) * x;
03803       //
03804       // error 0.0014886165, which is 6 bits
03805       SDValue t0 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X,
03806                                getF32Constant(DAG, 0xbdd49a13, dl));
03807       SDValue t1 = DAG.getNode(ISD::FADD, dl, MVT::f32, t0,
03808                                getF32Constant(DAG, 0x3f1c0789, dl));
03809       SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t1, X);
03810       Log10ofMantissa = DAG.getNode(ISD::FSUB, dl, MVT::f32, t2,
03811                                     getF32Constant(DAG, 0x3f011300, dl));
03812     } else if (LimitFloatPrecision <= 12) {
03813       // For floating-point precision of 12:
03814       //
03815       //   Log10ofMantissa =
03816       //     -0.64831180f +
03817       //       (0.91751397f +
03818       //         (-0.31664806f + 0.47637168e-1f * x) * x) * x;
03819       //
03820       // error 0.00019228036, which is better than 12 bits
03821       SDValue t0 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X,
03822                                getF32Constant(DAG, 0x3d431f31, dl));
03823       SDValue t1 = DAG.getNode(ISD::FSUB, dl, MVT::f32, t0,
03824                                getF32Constant(DAG, 0x3ea21fb2, dl));
03825       SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t1, X);
03826       SDValue t3 = DAG.getNode(ISD::FADD, dl, MVT::f32, t2,
03827                                getF32Constant(DAG, 0x3f6ae232, dl));
03828       SDValue t4 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t3, X);
03829       Log10ofMantissa = DAG.getNode(ISD::FSUB, dl, MVT::f32, t4,
03830                                     getF32Constant(DAG, 0x3f25f7c3, dl));
03831     } else { // LimitFloatPrecision <= 18
03832       // For floating-point precision of 18:
03833       //
03834       //   Log10ofMantissa =
03835       //     -0.84299375f +
03836       //       (1.5327582f +
03837       //         (-1.0688956f +
03838       //           (0.49102474f +
03839       //             (-0.12539807f + 0.13508273e-1f * x) * x) * x) * x) * x;
03840       //
03841       // error 0.0000037995730, which is better than 18 bits
03842       SDValue t0 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X,
03843                                getF32Constant(DAG, 0x3c5d51ce, dl));
03844       SDValue t1 = DAG.getNode(ISD::FSUB, dl, MVT::f32, t0,
03845                                getF32Constant(DAG, 0x3e00685a, dl));
03846       SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t1, X);
03847       SDValue t3 = DAG.getNode(ISD::FADD, dl, MVT::f32, t2,
03848                                getF32Constant(DAG, 0x3efb6798, dl));
03849       SDValue t4 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t3, X);
03850       SDValue t5 = DAG.getNode(ISD::FSUB, dl, MVT::f32, t4,
03851                                getF32Constant(DAG, 0x3f88d192, dl));
03852       SDValue t6 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t5, X);
03853       SDValue t7 = DAG.getNode(ISD::FADD, dl, MVT::f32, t6,
03854                                getF32Constant(DAG, 0x3fc4316c, dl));
03855       SDValue t8 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t7, X);
03856       Log10ofMantissa = DAG.getNode(ISD::FSUB, dl, MVT::f32, t8,
03857                                     getF32Constant(DAG, 0x3f57ce70, dl));
03858     }
03859 
03860     return DAG.getNode(ISD::FADD, dl, MVT::f32, LogOfExponent, Log10ofMantissa);
03861   }
03862 
03863   // No special expansion.
03864   return DAG.getNode(ISD::FLOG10, dl, Op.getValueType(), Op);
03865 }
03866 
03867 /// expandExp2 - Lower an exp2 intrinsic. Handles the special sequences for
03868 /// limited-precision mode.
03869 static SDValue expandExp2(SDLoc dl, SDValue Op, SelectionDAG &DAG,
03870                           const TargetLowering &TLI) {
03871   if (Op.getValueType() == MVT::f32 &&
03872       LimitFloatPrecision > 0 && LimitFloatPrecision <= 18)
03873     return getLimitedPrecisionExp2(Op, dl, DAG);
03874 
03875   // No special expansion.
03876   return DAG.getNode(ISD::FEXP2, dl, Op.getValueType(), Op);
03877 }
03878 
03879 /// visitPow - Lower a pow intrinsic. Handles the special sequences for
03880 /// limited-precision mode with x == 10.0f.
03881 static SDValue expandPow(SDLoc dl, SDValue LHS, SDValue RHS,
03882                          SelectionDAG &DAG, const TargetLowering &TLI) {
03883   bool IsExp10 = false;
03884   if (LHS.getValueType() == MVT::f32 && RHS.getValueType() == MVT::f32 &&
03885       LimitFloatPrecision > 0 && LimitFloatPrecision <= 18) {
03886     if (ConstantFPSDNode *LHSC = dyn_cast<ConstantFPSDNode>(LHS)) {
03887       APFloat Ten(10.0f);
03888       IsExp10 = LHSC->isExactlyValue(Ten);
03889     }
03890   }
03891 
03892   if (IsExp10) {
03893     // Put the exponent in the right bit position for later addition to the
03894     // final result:
03895     //
03896     //   #define LOG2OF10 3.3219281f
03897     //   t0 = Op * LOG2OF10;
03898     SDValue t0 = DAG.getNode(ISD::FMUL, dl, MVT::f32, RHS,
03899                              getF32Constant(DAG, 0x40549a78, dl));
03900     return getLimitedPrecisionExp2(t0, dl, DAG);
03901   }
03902 
03903   // No special expansion.
03904   return DAG.getNode(ISD::FPOW, dl, LHS.getValueType(), LHS, RHS);
03905 }
03906 
03907 
03908 /// ExpandPowI - Expand a llvm.powi intrinsic.
03909 static SDValue ExpandPowI(SDLoc DL, SDValue LHS, SDValue RHS,
03910                           SelectionDAG &DAG) {
03911   // If RHS is a constant, we can expand this out to a multiplication tree,
03912   // otherwise we end up lowering to a call to __powidf2 (for example).  When
03913   // optimizing for size, we only want to do this if the expansion would produce
03914   // a small number of multiplies, otherwise we do the full expansion.
03915   if (ConstantSDNode *RHSC = dyn_cast<ConstantSDNode>(RHS)) {
03916     // Get the exponent as a positive value.
03917     unsigned Val = RHSC->getSExtValue();
03918     if ((int)Val < 0) Val = -Val;
03919 
03920     // powi(x, 0) -> 1.0
03921     if (Val == 0)
03922       return DAG.getConstantFP(1.0, DL, LHS.getValueType());
03923 
03924     const Function *F = DAG.getMachineFunction().getFunction();
03925     if (!F->hasFnAttribute(Attribute::OptimizeForSize) ||
03926         // If optimizing for size, don't insert too many multiplies.  This
03927         // inserts up to 5 multiplies.
03928         countPopulation(Val) + Log2_32(Val) < 7) {
03929       // We use the simple binary decomposition method to generate the multiply
03930       // sequence.  There are more optimal ways to do this (for example,
03931       // powi(x,15) generates one more multiply than it should), but this has
03932       // the benefit of being both really simple and much better than a libcall.
03933       SDValue Res;  // Logically starts equal to 1.0
03934       SDValue CurSquare = LHS;
03935       while (Val) {
03936         if (Val & 1) {
03937           if (Res.getNode())
03938             Res = DAG.getNode(ISD::FMUL, DL,Res.getValueType(), Res, CurSquare);
03939           else
03940             Res = CurSquare;  // 1.0*CurSquare.
03941         }
03942 
03943         CurSquare = DAG.getNode(ISD::FMUL, DL, CurSquare.getValueType(),
03944                                 CurSquare, CurSquare);
03945         Val >>= 1;
03946       }
03947 
03948       // If the original was negative, invert the result, producing 1/(x*x*x).
03949       if (RHSC->getSExtValue() < 0)
03950         Res = DAG.getNode(ISD::FDIV, DL, LHS.getValueType(),
03951                           DAG.getConstantFP(1.0, DL, LHS.getValueType()), Res);
03952       return Res;
03953     }
03954   }
03955 
03956   // Otherwise, expand to a libcall.
03957   return DAG.getNode(ISD::FPOWI, DL, LHS.getValueType(), LHS, RHS);
03958 }
03959 
03960 // getTruncatedArgReg - Find underlying register used for an truncated
03961 // argument.
03962 static unsigned getTruncatedArgReg(const SDValue &N) {
03963   if (N.getOpcode() != ISD::TRUNCATE)
03964     return 0;
03965 
03966   const SDValue &Ext = N.getOperand(0);
03967   if (Ext.getOpcode() == ISD::AssertZext ||
03968       Ext.getOpcode() == ISD::AssertSext) {
03969     const SDValue &CFR = Ext.getOperand(0);
03970     if (CFR.getOpcode() == ISD::CopyFromReg)
03971       return cast<RegisterSDNode>(CFR.getOperand(1))->getReg();
03972     if (CFR.getOpcode() == ISD::TRUNCATE)
03973       return getTruncatedArgReg(CFR);
03974   }
03975   return 0;
03976 }
03977 
03978 /// EmitFuncArgumentDbgValue - If the DbgValueInst is a dbg_value of a function
03979 /// argument, create the corresponding DBG_VALUE machine instruction for it now.
03980 /// At the end of instruction selection, they will be inserted to the entry BB.
03981 bool SelectionDAGBuilder::EmitFuncArgumentDbgValue(
03982     const Value *V, DILocalVariable *Variable, DIExpression *Expr,
03983     DILocation *DL, int64_t Offset, bool IsIndirect, const SDValue &N) {
03984   const Argument *Arg = dyn_cast<Argument>(V);
03985   if (!Arg)
03986     return false;
03987 
03988   MachineFunction &MF = DAG.getMachineFunction();
03989   const TargetInstrInfo *TII = DAG.getSubtarget().getInstrInfo();
03990 
03991   // Ignore inlined function arguments here.
03992   //
03993   // FIXME: Should we be checking DL->inlinedAt() to determine this?
03994   if (!Variable->getScope()->getSubprogram()->describes(MF.getFunction()))
03995     return false;
03996 
03997   Optional<MachineOperand> Op;
03998   // Some arguments' frame index is recorded during argument lowering.
03999   if (int FI = FuncInfo.getArgumentFrameIndex(Arg))
04000     Op = MachineOperand::CreateFI(FI);
04001 
04002   if (!Op && N.getNode()) {
04003     unsigned Reg;
04004     if (N.getOpcode() == ISD::CopyFromReg)
04005       Reg = cast<RegisterSDNode>(N.getOperand(1))->getReg();
04006     else
04007       Reg = getTruncatedArgReg(N);
04008     if (Reg && TargetRegisterInfo::isVirtualRegister(Reg)) {
04009       MachineRegisterInfo &RegInfo = MF.getRegInfo();
04010       unsigned PR = RegInfo.getLiveInPhysReg(Reg);
04011       if (PR)
04012         Reg = PR;
04013     }
04014     if (Reg)
04015       Op = MachineOperand::CreateReg(Reg, false);
04016   }
04017 
04018   if (!Op) {
04019     // Check if ValueMap has reg number.
04020     DenseMap<const Value *, unsigned>::iterator VMI = FuncInfo.ValueMap.find(V);
04021     if (VMI != FuncInfo.ValueMap.end())
04022       Op = MachineOperand::CreateReg(VMI->second, false);
04023   }
04024 
04025   if (!Op && N.getNode())
04026     // Check if frame index is available.
04027     if (LoadSDNode *LNode = dyn_cast<LoadSDNode>(N.getNode()))
04028       if (FrameIndexSDNode *FINode =
04029           dyn_cast<FrameIndexSDNode>(LNode->getBasePtr().getNode()))
04030         Op = MachineOperand::CreateFI(FINode->getIndex());
04031 
04032   if (!Op)
04033     return false;
04034 
04035   assert(Variable->isValidLocationForIntrinsic(DL) &&
04036          "Expected inlined-at fields to agree");
04037   if (Op->isReg())
04038     FuncInfo.ArgDbgValues.push_back(
04039         BuildMI(MF, DL, TII->get(TargetOpcode::DBG_VALUE), IsIndirect,
04040                 Op->getReg(), Offset, Variable, Expr));
04041   else
04042     FuncInfo.ArgDbgValues.push_back(
04043         BuildMI(MF, DL, TII->get(TargetOpcode::DBG_VALUE))
04044             .addOperand(*Op)
04045             .addImm(Offset)
04046             .addMetadata(Variable)
04047             .addMetadata(Expr));
04048 
04049   return true;
04050 }
04051 
04052 // VisualStudio defines setjmp as _setjmp
04053 #if defined(_MSC_VER) && defined(setjmp) && \
04054                          !defined(setjmp_undefined_for_msvc)
04055 #  pragma push_macro("setjmp")
04056 #  undef setjmp
04057 #  define setjmp_undefined_for_msvc
04058 #endif
04059 
04060 /// visitIntrinsicCall - Lower the call to the specified intrinsic function.  If
04061 /// we want to emit this as a call to a named external function, return the name
04062 /// otherwise lower it and return null.
04063 const char *
04064 SelectionDAGBuilder::visitIntrinsicCall(const CallInst &I, unsigned Intrinsic) {
04065   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
04066   SDLoc sdl = getCurSDLoc();
04067   DebugLoc dl = getCurDebugLoc();
04068   SDValue Res;
04069 
04070   switch (Intrinsic) {
04071   default:
04072     // By default, turn this into a target intrinsic node.
04073     visitTargetIntrinsic(I, Intrinsic);
04074     return nullptr;
04075   case Intrinsic::vastart:  visitVAStart(I); return nullptr;
04076   case Intrinsic::vaend:    visitVAEnd(I); return nullptr;
04077   case Intrinsic::vacopy:   visitVACopy(I); return nullptr;
04078   case Intrinsic::returnaddress:
04079     setValue(&I, DAG.getNode(ISD::RETURNADDR, sdl, TLI.getPointerTy(),
04080                              getValue(I.getArgOperand(0))));
04081     return nullptr;
04082   case Intrinsic::frameaddress:
04083     setValue(&I, DAG.getNode(ISD::FRAMEADDR, sdl, TLI.getPointerTy(),
04084                              getValue(I.getArgOperand(0))));
04085     return nullptr;
04086   case Intrinsic::read_register: {
04087     Value *Reg = I.getArgOperand(0);
04088     SDValue Chain = getRoot();
04089     SDValue RegName =
04090         DAG.getMDNode(cast<MDNode>(cast<MetadataAsValue>(Reg)->getMetadata()));
04091     EVT VT = TLI.getValueType(I.getType());
04092     Res = DAG.getNode(ISD::READ_REGISTER, sdl,
04093       DAG.getVTList(VT, MVT::Other), Chain, RegName);
04094     setValue(&I, Res);
04095     DAG.setRoot(Res.getValue(1));
04096     return nullptr;
04097   }
04098   case Intrinsic::write_register: {
04099     Value *Reg = I.getArgOperand(0);
04100     Value *RegValue = I.getArgOperand(1);
04101     SDValue Chain = getRoot();
04102     SDValue RegName =
04103         DAG.getMDNode(cast<MDNode>(cast<MetadataAsValue>(Reg)->getMetadata()));
04104     DAG.setRoot(DAG.getNode(ISD::WRITE_REGISTER, sdl, MVT::Other, Chain,
04105                             RegName, getValue(RegValue)));
04106     return nullptr;
04107   }
04108   case Intrinsic::setjmp:
04109     return &"_setjmp"[!TLI.usesUnderscoreSetJmp()];
04110   case Intrinsic::longjmp:
04111     return &"_longjmp"[!TLI.usesUnderscoreLongJmp()];
04112   case Intrinsic::memcpy: {
04113     // FIXME: this definition of "user defined address space" is x86-specific
04114     // Assert for address < 256 since we support only user defined address
04115     // spaces.
04116     assert(cast<PointerType>(I.getArgOperand(0)->getType())->getAddressSpace()
04117            < 256 &&
04118            cast<PointerType>(I.getArgOperand(1)->getType())->getAddressSpace()
04119            < 256 &&
04120            "Unknown address space");
04121     SDValue Op1 = getValue(I.getArgOperand(0));
04122     SDValue Op2 = getValue(I.getArgOperand(1));
04123     SDValue Op3 = getValue(I.getArgOperand(2));
04124     unsigned Align = cast<ConstantInt>(I.getArgOperand(3))->getZExtValue();
04125     if (!Align)
04126       Align = 1; // @llvm.memcpy defines 0 and 1 to both mean no alignment.
04127     bool isVol = cast<ConstantInt>(I.getArgOperand(4))->getZExtValue();
04128     bool isTC = I.isTailCall() && isInTailCallPosition(&I, DAG.getTarget());
04129     SDValue MC = DAG.getMemcpy(getRoot(), sdl, Op1, Op2, Op3, Align, isVol,
04130                                false, isTC,
04131                                MachinePointerInfo(I.getArgOperand(0)),
04132                                MachinePointerInfo(I.getArgOperand(1)));
04133     updateDAGForMaybeTailCall(MC);
04134     return nullptr;
04135   }
04136   case Intrinsic::memset: {
04137     // FIXME: this definition of "user defined address space" is x86-specific
04138     // Assert for address < 256 since we support only user defined address
04139     // spaces.
04140     assert(cast<PointerType>(I.getArgOperand(0)->getType())->getAddressSpace()
04141            < 256 &&
04142            "Unknown address space");
04143     SDValue Op1 = getValue(I.getArgOperand(0));
04144     SDValue Op2 = getValue(I.getArgOperand(1));
04145     SDValue Op3 = getValue(I.getArgOperand(2));
04146     unsigned Align = cast<ConstantInt>(I.getArgOperand(3))->getZExtValue();
04147     if (!Align)
04148       Align = 1; // @llvm.memset defines 0 and 1 to both mean no alignment.
04149     bool isVol = cast<ConstantInt>(I.getArgOperand(4))->getZExtValue();
04150     bool isTC = I.isTailCall() && isInTailCallPosition(&I, DAG.getTarget());
04151     SDValue MS = DAG.getMemset(getRoot(), sdl, Op1, Op2, Op3, Align, isVol,
04152                                isTC, MachinePointerInfo(I.getArgOperand(0)));
04153     updateDAGForMaybeTailCall(MS);
04154     return nullptr;
04155   }
04156   case Intrinsic::memmove: {
04157     // FIXME: this definition of "user defined address space" is x86-specific
04158     // Assert for address < 256 since we support only user defined address
04159     // spaces.
04160     assert(cast<PointerType>(I.getArgOperand(0)->getType())->getAddressSpace()
04161            < 256 &&
04162            cast<PointerType>(I.getArgOperand(1)->getType())->getAddressSpace()
04163            < 256 &&
04164            "Unknown address space");
04165     SDValue Op1 = getValue(I.getArgOperand(0));
04166     SDValue Op2 = getValue(I.getArgOperand(1));
04167     SDValue Op3 = getValue(I.getArgOperand(2));
04168     unsigned Align = cast<ConstantInt>(I.getArgOperand(3))->getZExtValue();
04169     if (!Align)
04170       Align = 1; // @llvm.memmove defines 0 and 1 to both mean no alignment.
04171     bool isVol = cast<ConstantInt>(I.getArgOperand(4))->getZExtValue();
04172     bool isTC = I.isTailCall() && isInTailCallPosition(&I, DAG.getTarget());
04173     SDValue MM = DAG.getMemmove(getRoot(), sdl, Op1, Op2, Op3, Align, isVol,
04174                                 isTC, MachinePointerInfo(I.getArgOperand(0)),
04175                                 MachinePointerInfo(I.getArgOperand(1)));
04176     updateDAGForMaybeTailCall(MM);
04177     return nullptr;
04178   }
04179   case Intrinsic::dbg_declare: {
04180     const DbgDeclareInst &DI = cast<DbgDeclareInst>(I);
04181     DILocalVariable *Variable = DI.getVariable();
04182     DIExpression *Expression = DI.getExpression();
04183     const Value *Address = DI.getAddress();
04184     assert(Variable && "Missing variable");
04185     if (!Address) {
04186       DEBUG(dbgs() << "Dropping debug info for " << DI << "\n");
04187       return nullptr;
04188     }
04189 
04190     // Check if address has undef value.
04191     if (isa<UndefValue>(Address) ||
04192         (Address->use_empty() && !isa<Argument>(Address))) {
04193       DEBUG(dbgs() << "Dropping debug info for " << DI << "\n");
04194       return nullptr;
04195     }
04196 
04197     SDValue &N = NodeMap[Address];
04198     if (!N.getNode() && isa<Argument>(Address))
04199       // Check unused arguments map.
04200       N = UnusedArgNodeMap[Address];
04201     SDDbgValue *SDV;
04202     if (N.getNode()) {
04203       if (const BitCastInst *BCI = dyn_cast<BitCastInst>(Address))
04204         Address = BCI->getOperand(0);
04205       // Parameters are handled specially.
04206       bool isParameter = Variable->getTag() == dwarf::DW_TAG_arg_variable ||
04207                          isa<Argument>(Address);
04208 
04209       const AllocaInst *AI = dyn_cast<AllocaInst>(Address);
04210 
04211       if (isParameter && !AI) {
04212         FrameIndexSDNode *FINode = dyn_cast<FrameIndexSDNode>(N.getNode());
04213         if (FINode)
04214           // Byval parameter.  We have a frame index at this point.
04215           SDV = DAG.getFrameIndexDbgValue(
04216               Variable, Expression, FINode->getIndex(), 0, dl, SDNodeOrder);
04217         else {
04218           // Address is an argument, so try to emit its dbg value using
04219           // virtual register info from the FuncInfo.ValueMap.
04220           EmitFuncArgumentDbgValue(Address, Variable, Expression, dl, 0, false,
04221                                    N);
04222           return nullptr;
04223         }
04224       } else if (AI)
04225         SDV = DAG.getDbgValue(Variable, Expression, N.getNode(), N.getResNo(),
04226                               true, 0, dl, SDNodeOrder);
04227       else {
04228         // Can't do anything with other non-AI cases yet.
04229         DEBUG(dbgs() << "Dropping debug info for " << DI << "\n");
04230         DEBUG(dbgs() << "non-AllocaInst issue for Address: \n\t");
04231         DEBUG(Address->dump());
04232         return nullptr;
04233       }
04234       DAG.AddDbgValue(SDV, N.getNode(), isParameter);
04235     } else {
04236       // If Address is an argument then try to emit its dbg value using
04237       // virtual register info from the FuncInfo.ValueMap.
04238       if (!EmitFuncArgumentDbgValue(Address, Variable, Expression, dl, 0, false,
04239                                     N)) {
04240         // If variable is pinned by a alloca in dominating bb then
04241         // use StaticAllocaMap.
04242         if (const AllocaInst *AI = dyn_cast<AllocaInst>(Address)) {
04243           if (AI->getParent() != DI.getParent()) {
04244             DenseMap<const AllocaInst*, int>::iterator SI =
04245               FuncInfo.StaticAllocaMap.find(AI);
04246             if (SI != FuncInfo.StaticAllocaMap.end()) {
04247               SDV = DAG.getFrameIndexDbgValue(Variable, Expression, SI->second,
04248                                               0, dl, SDNodeOrder);
04249               DAG.AddDbgValue(SDV, nullptr, false);
04250               return nullptr;
04251             }
04252           }
04253         }
04254         DEBUG(dbgs() << "Dropping debug info for " << DI << "\n");
04255       }
04256     }
04257     return nullptr;
04258   }
04259   case Intrinsic::dbg_value: {
04260     const DbgValueInst &DI = cast<DbgValueInst>(I);
04261     assert(DI.getVariable() && "Missing variable");
04262 
04263     DILocalVariable *Variable = DI.getVariable();
04264     DIExpression *Expression = DI.getExpression();
04265     uint64_t Offset = DI.getOffset();
04266     const Value *V = DI.getValue();
04267     if (!V)
04268       return nullptr;
04269 
04270     SDDbgValue *SDV;
04271     if (isa<ConstantInt>(V) || isa<ConstantFP>(V) || isa<UndefValue>(V)) {
04272       SDV = DAG.getConstantDbgValue(Variable, Expression, V, Offset, dl,
04273                                     SDNodeOrder);
04274       DAG.AddDbgValue(SDV, nullptr, false);
04275     } else {
04276       // Do not use getValue() in here; we don't want to generate code at
04277       // this point if it hasn't been done yet.
04278       SDValue N = NodeMap[V];
04279       if (!N.getNode() && isa<Argument>(V))
04280         // Check unused arguments map.
04281         N = UnusedArgNodeMap[V];
04282       if (N.getNode()) {
04283         // A dbg.value for an alloca is always indirect.
04284         bool IsIndirect = isa<AllocaInst>(V) || Offset != 0;
04285         if (!EmitFuncArgumentDbgValue(V, Variable, Expression, dl, Offset,
04286                                       IsIndirect, N)) {
04287           SDV = DAG.getDbgValue(Variable, Expression, N.getNode(), N.getResNo(),
04288                                 IsIndirect, Offset, dl, SDNodeOrder);
04289           DAG.AddDbgValue(SDV, N.getNode(), false);
04290         }
04291       } else if (!V->use_empty() ) {
04292         // Do not call getValue(V) yet, as we don't want to generate code.
04293         // Remember it for later.
04294         DanglingDebugInfo DDI(&DI, dl, SDNodeOrder);
04295         DanglingDebugInfoMap[V] = DDI;
04296       } else {
04297         // We may expand this to cover more cases.  One case where we have no
04298         // data available is an unreferenced parameter.
04299         DEBUG(dbgs() << "Dropping debug info for " << DI << "\n");
04300       }
04301     }
04302 
04303     // Build a debug info table entry.
04304     if (const BitCastInst *BCI = dyn_cast<BitCastInst>(V))
04305       V = BCI->getOperand(0);
04306     const AllocaInst *AI = dyn_cast<AllocaInst>(V);
04307     // Don't handle byval struct arguments or VLAs, for example.
04308     if (!AI) {
04309       DEBUG(dbgs() << "Dropping debug location info for:\n  " << DI << "\n");
04310       DEBUG(dbgs() << "  Last seen at:\n    " << *V << "\n");
04311       return nullptr;
04312     }
04313     DenseMap<const AllocaInst*, int>::iterator SI =
04314       FuncInfo.StaticAllocaMap.find(AI);
04315     if (SI == FuncInfo.StaticAllocaMap.end())
04316       return nullptr; // VLAs.
04317     return nullptr;
04318   }
04319 
04320   case Intrinsic::eh_typeid_for: {
04321     // Find the type id for the given typeinfo.
04322     GlobalValue *GV = ExtractTypeInfo(I.getArgOperand(0));
04323     unsigned TypeID = DAG.getMachineFunction().getMMI().getTypeIDFor(GV);
04324     Res = DAG.getConstant(TypeID, sdl, MVT::i32);
04325     setValue(&I, Res);
04326     return nullptr;
04327   }
04328 
04329   case Intrinsic::eh_return_i32:
04330   case Intrinsic::eh_return_i64:
04331     DAG.getMachineFunction().getMMI().setCallsEHReturn(true);
04332     DAG.setRoot(DAG.getNode(ISD::EH_RETURN, sdl,
04333                             MVT::Other,
04334                             getControlRoot(),
04335                             getValue(I.getArgOperand(0)),
04336                             getValue(I.getArgOperand(1))));
04337     return nullptr;
04338   case Intrinsic::eh_unwind_init:
04339     DAG.getMachineFunction().getMMI().setCallsUnwindInit(true);
04340     return nullptr;
04341   case Intrinsic::eh_dwarf_cfa: {
04342     SDValue CfaArg = DAG.getSExtOrTrunc(getValue(I.getArgOperand(0)), sdl,
04343                                         TLI.getPointerTy());
04344     SDValue Offset = DAG.getNode(ISD::ADD, sdl,
04345                                  CfaArg.getValueType(),
04346                                  DAG.getNode(ISD::FRAME_TO_ARGS_OFFSET, sdl,
04347                                              CfaArg.getValueType()),
04348                                  CfaArg);
04349     SDValue FA = DAG.getNode(ISD::FRAMEADDR, sdl, TLI.getPointerTy(),
04350                              DAG.getConstant(0, sdl, TLI.getPointerTy()));
04351     setValue(&I, DAG.getNode(ISD::ADD, sdl, FA.getValueType(),
04352                              FA, Offset));
04353     return nullptr;
04354   }
04355   case Intrinsic::eh_sjlj_callsite: {
04356     MachineModuleInfo &MMI = DAG.getMachineFunction().getMMI();
04357     ConstantInt *CI = dyn_cast<ConstantInt>(I.getArgOperand(0));
04358     assert(CI && "Non-constant call site value in eh.sjlj.callsite!");
04359     assert(MMI.getCurrentCallSite() == 0 && "Overlapping call sites!");
04360 
04361     MMI.setCurrentCallSite(CI->getZExtValue());
04362     return nullptr;
04363   }
04364   case Intrinsic::eh_sjlj_functioncontext: {
04365     // Get and store the index of the function context.
04366     MachineFrameInfo *MFI = DAG.getMachineFunction().getFrameInfo();
04367     AllocaInst *FnCtx =
04368       cast<AllocaInst>(I.getArgOperand(0)->stripPointerCasts());
04369     int FI = FuncInfo.StaticAllocaMap[FnCtx];
04370     MFI->setFunctionContextIndex(FI);
04371     return nullptr;
04372   }
04373   case Intrinsic::eh_sjlj_setjmp: {
04374     SDValue Ops[2];
04375     Ops[0] = getRoot();
04376     Ops[1] = getValue(I.getArgOperand(0));
04377     SDValue Op = DAG.getNode(ISD::EH_SJLJ_SETJMP, sdl,
04378                              DAG.getVTList(MVT::i32, MVT::Other), Ops);
04379     setValue(&I, Op.getValue(0));
04380     DAG.setRoot(Op.getValue(1));
04381     return nullptr;
04382   }
04383   case Intrinsic::eh_sjlj_longjmp: {
04384     DAG.setRoot(DAG.getNode(ISD::EH_SJLJ_LONGJMP, sdl, MVT::Other,
04385                             getRoot(), getValue(I.getArgOperand(0))));
04386     return nullptr;
04387   }
04388 
04389   case Intrinsic::masked_gather:
04390     visitMaskedGather(I);
04391     return nullptr;
04392   case Intrinsic::masked_load:
04393     visitMaskedLoad(I);
04394     return nullptr;
04395   case Intrinsic::masked_scatter:
04396     visitMaskedScatter(I);
04397     return nullptr;
04398   case Intrinsic::masked_store:
04399     visitMaskedStore(I);
04400     return nullptr;
04401   case Intrinsic::x86_mmx_pslli_w:
04402   case Intrinsic::x86_mmx_pslli_d:
04403   case Intrinsic::x86_mmx_pslli_q:
04404   case Intrinsic::x86_mmx_psrli_w:
04405   case Intrinsic::x86_mmx_psrli_d:
04406   case Intrinsic::x86_mmx_psrli_q:
04407   case Intrinsic::x86_mmx_psrai_w:
04408   case Intrinsic::x86_mmx_psrai_d: {
04409     SDValue ShAmt = getValue(I.getArgOperand(1));
04410     if (isa<ConstantSDNode>(ShAmt)) {
04411       visitTargetIntrinsic(I, Intrinsic);
04412       return nullptr;
04413     }
04414     unsigned NewIntrinsic = 0;
04415     EVT ShAmtVT = MVT::v2i32;
04416     switch (Intrinsic) {
04417     case Intrinsic::x86_mmx_pslli_w:
04418       NewIntrinsic = Intrinsic::x86_mmx_psll_w;
04419       break;
04420     case Intrinsic::x86_mmx_pslli_d:
04421       NewIntrinsic = Intrinsic::x86_mmx_psll_d;
04422       break;
04423     case Intrinsic::x86_mmx_pslli_q:
04424       NewIntrinsic = Intrinsic::x86_mmx_psll_q;
04425       break;
04426     case Intrinsic::x86_mmx_psrli_w:
04427       NewIntrinsic = Intrinsic::x86_mmx_psrl_w;
04428       break;
04429     case Intrinsic::x86_mmx_psrli_d:
04430       NewIntrinsic = Intrinsic::x86_mmx_psrl_d;
04431       break;
04432     case Intrinsic::x86_mmx_psrli_q:
04433       NewIntrinsic = Intrinsic::x86_mmx_psrl_q;
04434       break;
04435     case Intrinsic::x86_mmx_psrai_w:
04436       NewIntrinsic = Intrinsic::x86_mmx_psra_w;
04437       break;
04438     case Intrinsic::x86_mmx_psrai_d:
04439       NewIntrinsic = Intrinsic::x86_mmx_psra_d;
04440       break;
04441     default: llvm_unreachable("Impossible intrinsic");  // Can't reach here.
04442     }
04443 
04444     // The vector shift intrinsics with scalars uses 32b shift amounts but
04445     // the sse2/mmx shift instructions reads 64 bits. Set the upper 32 bits
04446     // to be zero.
04447     // We must do this early because v2i32 is not a legal type.
04448     SDValue ShOps[2];
04449     ShOps[0] = ShAmt;
04450     ShOps[1] = DAG.getConstant(0, sdl, MVT::i32);
04451     ShAmt =  DAG.getNode(ISD::BUILD_VECTOR, sdl, ShAmtVT, ShOps);
04452     EVT DestVT = TLI.getValueType(I.getType());
04453     ShAmt = DAG.getNode(ISD::BITCAST, sdl, DestVT, ShAmt);
04454     Res = DAG.getNode(ISD::INTRINSIC_WO_CHAIN, sdl, DestVT,
04455                        DAG.getConstant(NewIntrinsic, sdl, MVT::i32),
04456                        getValue(I.getArgOperand(0)), ShAmt);
04457     setValue(&I, Res);
04458     return nullptr;
04459   }
04460   case Intrinsic::convertff:
04461   case Intrinsic::convertfsi:
04462   case Intrinsic::convertfui:
04463   case Intrinsic::convertsif:
04464   case Intrinsic::convertuif:
04465   case Intrinsic::convertss:
04466   case Intrinsic::convertsu:
04467   case Intrinsic::convertus:
04468   case Intrinsic::convertuu: {
04469     ISD::CvtCode Code = ISD::CVT_INVALID;
04470     switch (Intrinsic) {
04471     default: llvm_unreachable("Impossible intrinsic");  // Can't reach here.
04472     case Intrinsic::convertff:  Code = ISD::CVT_FF; break;
04473     case Intrinsic::convertfsi: Code = ISD::CVT_FS; break;
04474     case Intrinsic::convertfui: Code = ISD::CVT_FU; break;
04475     case Intrinsic::convertsif: Code = ISD::CVT_SF; break;
04476     case Intrinsic::convertuif: Code = ISD::CVT_UF; break;
04477     case Intrinsic::convertss:  Code = ISD::CVT_SS; break;
04478     case Intrinsic::convertsu:  Code = ISD::CVT_SU; break;
04479     case Intrinsic::convertus:  Code = ISD::CVT_US; break;
04480     case Intrinsic::convertuu:  Code = ISD::CVT_UU; break;
04481     }
04482     EVT DestVT = TLI.getValueType(I.getType());
04483     const Value *Op1 = I.getArgOperand(0);
04484     Res = DAG.getConvertRndSat(DestVT, sdl, getValue(Op1),
04485                                DAG.getValueType(DestVT),
04486                                DAG.getValueType(getValue(Op1).getValueType()),
04487                                getValue(I.getArgOperand(1)),
04488                                getValue(I.getArgOperand(2)),
04489                                Code);
04490     setValue(&I, Res);
04491     return nullptr;
04492   }
04493   case Intrinsic::powi:
04494     setValue(&I, ExpandPowI(sdl, getValue(I.getArgOperand(0)),
04495                             getValue(I.getArgOperand(1)), DAG));
04496     return nullptr;
04497   case Intrinsic::log:
04498     setValue(&I, expandLog(sdl, getValue(I.getArgOperand(0)), DAG, TLI));
04499     return nullptr;
04500   case Intrinsic::log2:
04501     setValue(&I, expandLog2(sdl, getValue(I.getArgOperand(0)), DAG, TLI));
04502     return nullptr;
04503   case Intrinsic::log10:
04504     setValue(&I, expandLog10(sdl, getValue(I.getArgOperand(0)), DAG, TLI));
04505     return nullptr;
04506   case Intrinsic::exp:
04507     setValue(&I, expandExp(sdl, getValue(I.getArgOperand(0)), DAG, TLI));
04508     return nullptr;
04509   case Intrinsic::exp2:
04510     setValue(&I, expandExp2(sdl, getValue(I.getArgOperand(0)), DAG, TLI));
04511     return nullptr;
04512   case Intrinsic::pow:
04513     setValue(&I, expandPow(sdl, getValue(I.getArgOperand(0)),
04514                            getValue(I.getArgOperand(1)), DAG, TLI));
04515     return nullptr;
04516   case Intrinsic::sqrt:
04517   case Intrinsic::fabs:
04518   case Intrinsic::sin:
04519   case Intrinsic::cos:
04520   case Intrinsic::floor:
04521   case Intrinsic::ceil:
04522   case Intrinsic::trunc:
04523   case Intrinsic::rint:
04524   case Intrinsic::nearbyint:
04525   case Intrinsic::round: {
04526     unsigned Opcode;
04527     switch (Intrinsic) {
04528     default: llvm_unreachable("Impossible intrinsic");  // Can't reach here.
04529     case Intrinsic::sqrt:      Opcode = ISD::FSQRT;      break;
04530     case Intrinsic::fabs:      Opcode = ISD::FABS;       break;
04531     case Intrinsic::sin:       Opcode = ISD::FSIN;       break;
04532     case Intrinsic::cos:       Opcode = ISD::FCOS;       break;
04533     case Intrinsic::floor:     Opcode = ISD::FFLOOR;     break;
04534     case Intrinsic::ceil:      Opcode = ISD::FCEIL;      break;
04535     case Intrinsic::trunc:     Opcode = ISD::FTRUNC;     break;
04536     case Intrinsic::rint:      Opcode = ISD::FRINT;      break;
04537     case Intrinsic::nearbyint: Opcode = ISD::FNEARBYINT; break;
04538     case Intrinsic::round:     Opcode = ISD::FROUND;     break;
04539     }
04540 
04541     setValue(&I, DAG.getNode(Opcode, sdl,
04542                              getValue(I.getArgOperand(0)).getValueType(),
04543                              getValue(I.getArgOperand(0))));
04544     return nullptr;
04545   }
04546   case Intrinsic::minnum:
04547     setValue(&I, DAG.getNode(ISD::FMINNUM, sdl,
04548                              getValue(I.getArgOperand(0)).getValueType(),
04549                              getValue(I.getArgOperand(0)),
04550                              getValue(I.getArgOperand(1))));
04551     return nullptr;
04552   case Intrinsic::maxnum:
04553     setValue(&I, DAG.getNode(ISD::FMAXNUM, sdl,
04554                              getValue(I.getArgOperand(0)).getValueType(),
04555                              getValue(I.getArgOperand(0)),
04556                              getValue(I.getArgOperand(1))));
04557     return nullptr;
04558   case Intrinsic::copysign:
04559     setValue(&I, DAG.getNode(ISD::FCOPYSIGN, sdl,
04560                              getValue(I.getArgOperand(0)).getValueType(),
04561                              getValue(I.getArgOperand(0)),
04562                              getValue(I.getArgOperand(1))));
04563     return nullptr;
04564   case Intrinsic::fma:
04565     setValue(&I, DAG.getNode(ISD::FMA, sdl,
04566                              getValue(I.getArgOperand(0)).getValueType(),
04567                              getValue(I.getArgOperand(0)),
04568                              getValue(I.getArgOperand(1)),
04569                              getValue(I.getArgOperand(2))));
04570     return nullptr;
04571   case Intrinsic::fmuladd: {
04572     EVT VT = TLI.getValueType(I.getType());
04573     if (TM.Options.AllowFPOpFusion != FPOpFusion::Strict &&
04574         TLI.isFMAFasterThanFMulAndFAdd(VT)) {
04575       setValue(&I, DAG.getNode(ISD::FMA, sdl,
04576                                getValue(I.getArgOperand(0)).getValueType(),
04577                                getValue(I.getArgOperand(0)),
04578                                getValue(I.getArgOperand(1)),
04579                                getValue(I.getArgOperand(2))));
04580     } else {
04581       SDValue Mul = DAG.getNode(ISD::FMUL, sdl,
04582                                 getValue(I.getArgOperand(0)).getValueType(),
04583                                 getValue(I.getArgOperand(0)),
04584                                 getValue(I.getArgOperand(1)));
04585       SDValue Add = DAG.getNode(ISD::FADD, sdl,
04586                                 getValue(I.getArgOperand(0)).getValueType(),
04587                                 Mul,
04588                                 getValue(I.getArgOperand(2)));
04589       setValue(&I, Add);
04590     }
04591     return nullptr;
04592   }
04593   case Intrinsic::convert_to_fp16:
04594     setValue(&I, DAG.getNode(ISD::BITCAST, sdl, MVT::i16,
04595                              DAG.getNode(ISD::FP_ROUND, sdl, MVT::f16,
04596                                          getValue(I.getArgOperand(0)),
04597                                          DAG.getTargetConstant(0, sdl,
04598                                                                MVT::i32))));
04599     return nullptr;
04600   case Intrinsic::convert_from_fp16:
04601     setValue(&I,
04602              DAG.getNode(ISD::FP_EXTEND, sdl, TLI.getValueType(I.getType()),
04603                          DAG.getNode(ISD::BITCAST, sdl, MVT::f16,
04604                                      getValue(I.getArgOperand(0)))));
04605     return nullptr;
04606   case Intrinsic::pcmarker: {
04607     SDValue Tmp = getValue(I.getArgOperand(0));
04608     DAG.setRoot(DAG.getNode(ISD::PCMARKER, sdl, MVT::Other, getRoot(), Tmp));
04609     return nullptr;
04610   }
04611   case Intrinsic::readcyclecounter: {
04612     SDValue Op = getRoot();
04613     Res = DAG.getNode(ISD::READCYCLECOUNTER, sdl,
04614                       DAG.getVTList(MVT::i64, MVT::Other), Op);
04615     setValue(&I, Res);
04616     DAG.setRoot(Res.getValue(1));
04617     return nullptr;
04618   }
04619   case Intrinsic::bswap:
04620     setValue(&I, DAG.getNode(ISD::BSWAP, sdl,
04621                              getValue(I.getArgOperand(0)).getValueType(),
04622                              getValue(I.getArgOperand(0))));
04623     return nullptr;
04624   case Intrinsic::cttz: {
04625     SDValue Arg = getValue(I.getArgOperand(0));
04626     ConstantInt *CI = cast<ConstantInt>(I.getArgOperand(1));
04627     EVT Ty = Arg.getValueType();
04628     setValue(&I, DAG.getNode(CI->isZero() ? ISD::CTTZ : ISD::CTTZ_ZERO_UNDEF,
04629                              sdl, Ty, Arg));
04630     return nullptr;
04631   }
04632   case Intrinsic::ctlz: {
04633     SDValue Arg = getValue(I.getArgOperand(0));
04634     ConstantInt *CI = cast<ConstantInt>(I.getArgOperand(1));
04635     EVT Ty = Arg.getValueType();
04636     setValue(&I, DAG.getNode(CI->isZero() ? ISD::CTLZ : ISD::CTLZ_ZERO_UNDEF,
04637                              sdl, Ty, Arg));
04638     return nullptr;
04639   }
04640   case Intrinsic::ctpop: {
04641     SDValue Arg = getValue(I.getArgOperand(0));
04642     EVT Ty = Arg.getValueType();
04643     setValue(&I, DAG.getNode(ISD::CTPOP, sdl, Ty, Arg));
04644     return nullptr;
04645   }
04646   case Intrinsic::stacksave: {
04647     SDValue Op = getRoot();
04648     Res = DAG.getNode(ISD::STACKSAVE, sdl,
04649                       DAG.getVTList(TLI.getPointerTy(), MVT::Other), Op);
04650     setValue(&I, Res);
04651     DAG.setRoot(Res.getValue(1));
04652     return nullptr;
04653   }
04654   case Intrinsic::stackrestore: {
04655     Res = getValue(I.getArgOperand(0));
04656     DAG.setRoot(DAG.getNode(ISD::STACKRESTORE, sdl, MVT::Other, getRoot(), Res));
04657     return nullptr;
04658   }
04659   case Intrinsic::stackprotector: {
04660     // Emit code into the DAG to store the stack guard onto the stack.
04661     MachineFunction &MF = DAG.getMachineFunction();
04662     MachineFrameInfo *MFI = MF.getFrameInfo();
04663     EVT PtrTy = TLI.getPointerTy();
04664     SDValue Src, Chain = getRoot();
04665     const Value *Ptr = cast<LoadInst>(I.getArgOperand(0))->getPointerOperand();
04666     const GlobalVariable *GV = dyn_cast<GlobalVariable>(Ptr);
04667 
04668     // See if Ptr is a bitcast. If it is, look through it and see if we can get
04669     // global variable __stack_chk_guard.
04670     if (!GV)
04671       if (const Operator *BC = dyn_cast<Operator>(Ptr))
04672         if (BC->getOpcode() == Instruction::BitCast)
04673           GV = dyn_cast<GlobalVariable>(BC->getOperand(0));
04674 
04675     if (GV && TLI.useLoadStackGuardNode()) {
04676       // Emit a LOAD_STACK_GUARD node.
04677       MachineSDNode *Node = DAG.getMachineNode(TargetOpcode::LOAD_STACK_GUARD,
04678                                                sdl, PtrTy, Chain);
04679       MachinePointerInfo MPInfo(GV);
04680       MachineInstr::mmo_iterator MemRefs = MF.allocateMemRefsArray(1);
04681       unsigned Flags = MachineMemOperand::MOLoad |
04682                        MachineMemOperand::MOInvariant;
04683       *MemRefs = MF.getMachineMemOperand(MPInfo, Flags,
04684                                          PtrTy.getSizeInBits() / 8,
04685                                          DAG.getEVTAlignment(PtrTy));
04686       Node->setMemRefs(MemRefs, MemRefs + 1);
04687 
04688       // Copy the guard value to a virtual register so that it can be
04689       // retrieved in the epilogue.
04690       Src = SDValue(Node, 0);
04691       const TargetRegisterClass *RC =
04692           TLI.getRegClassFor(Src.getSimpleValueType());
04693       unsigned Reg = MF.getRegInfo().createVirtualRegister(RC);
04694 
04695       SPDescriptor.setGuardReg(Reg);
04696       Chain = DAG.getCopyToReg(Chain, sdl, Reg, Src);
04697     } else {
04698       Src = getValue(I.getArgOperand(0));   // The guard's value.
04699     }
04700 
04701     AllocaInst *Slot = cast<AllocaInst>(I.getArgOperand(1));
04702 
04703     int FI = FuncInfo.StaticAllocaMap[Slot];
04704     MFI->setStackProtectorIndex(FI);
04705 
04706     SDValue FIN = DAG.getFrameIndex(FI, PtrTy);
04707 
04708     // Store the stack protector onto the stack.
04709     Res = DAG.getStore(Chain, sdl, Src, FIN,
04710                        MachinePointerInfo::getFixedStack(FI),
04711                        true, false, 0);
04712     setValue(&I, Res);
04713     DAG.setRoot(Res);
04714     return nullptr;
04715   }
04716   case Intrinsic::objectsize: {
04717     // If we don't know by now, we're never going to know.
04718     ConstantInt *CI = dyn_cast<ConstantInt>(I.getArgOperand(1));
04719 
04720     assert(CI && "Non-constant type in __builtin_object_size?");
04721 
04722     SDValue Arg = getValue(I.getCalledValue());
04723     EVT Ty = Arg.getValueType();
04724 
04725     if (CI->isZero())
04726       Res = DAG.getConstant(-1ULL, sdl, Ty);
04727     else
04728       Res = DAG.getConstant(0, sdl, Ty);
04729 
04730     setValue(&I, Res);
04731     return nullptr;
04732   }
04733   case Intrinsic::annotation:
04734   case Intrinsic::ptr_annotation:
04735     // Drop the intrinsic, but forward the value
04736     setValue(&I, getValue(I.