LLVM  mainline
SelectionDAGBuilder.cpp
Go to the documentation of this file.
00001 //===-- SelectionDAGBuilder.cpp - Selection-DAG building ------------------===//
00002 //
00003 //                     The LLVM Compiler Infrastructure
00004 //
00005 // This file is distributed under the University of Illinois Open Source
00006 // License. See LICENSE.TXT for details.
00007 //
00008 //===----------------------------------------------------------------------===//
00009 //
00010 // This implements routines for translating from LLVM IR into SelectionDAG IR.
00011 //
00012 //===----------------------------------------------------------------------===//
00013 
00014 #include "SelectionDAGBuilder.h"
00015 #include "SDNodeDbgValue.h"
00016 #include "llvm/ADT/BitVector.h"
00017 #include "llvm/ADT/Optional.h"
00018 #include "llvm/ADT/SmallSet.h"
00019 #include "llvm/ADT/Statistic.h"
00020 #include "llvm/Analysis/AliasAnalysis.h"
00021 #include "llvm/Analysis/BranchProbabilityInfo.h"
00022 #include "llvm/Analysis/ConstantFolding.h"
00023 #include "llvm/Analysis/TargetLibraryInfo.h"
00024 #include "llvm/Analysis/ValueTracking.h"
00025 #include "llvm/CodeGen/Analysis.h"
00026 #include "llvm/CodeGen/FastISel.h"
00027 #include "llvm/CodeGen/FunctionLoweringInfo.h"
00028 #include "llvm/CodeGen/GCMetadata.h"
00029 #include "llvm/CodeGen/GCStrategy.h"
00030 #include "llvm/CodeGen/MachineFrameInfo.h"
00031 #include "llvm/CodeGen/MachineFunction.h"
00032 #include "llvm/CodeGen/MachineInstrBuilder.h"
00033 #include "llvm/CodeGen/MachineJumpTableInfo.h"
00034 #include "llvm/CodeGen/MachineModuleInfo.h"
00035 #include "llvm/CodeGen/MachineRegisterInfo.h"
00036 #include "llvm/CodeGen/SelectionDAG.h"
00037 #include "llvm/CodeGen/StackMaps.h"
00038 #include "llvm/IR/CallingConv.h"
00039 #include "llvm/IR/Constants.h"
00040 #include "llvm/IR/DataLayout.h"
00041 #include "llvm/IR/DebugInfo.h"
00042 #include "llvm/IR/DerivedTypes.h"
00043 #include "llvm/IR/Function.h"
00044 #include "llvm/IR/GlobalVariable.h"
00045 #include "llvm/IR/InlineAsm.h"
00046 #include "llvm/IR/Instructions.h"
00047 #include "llvm/IR/IntrinsicInst.h"
00048 #include "llvm/IR/Intrinsics.h"
00049 #include "llvm/IR/LLVMContext.h"
00050 #include "llvm/IR/Module.h"
00051 #include "llvm/IR/Statepoint.h"
00052 #include "llvm/MC/MCSymbol.h"
00053 #include "llvm/Support/CommandLine.h"
00054 #include "llvm/Support/Debug.h"
00055 #include "llvm/Support/ErrorHandling.h"
00056 #include "llvm/Support/MathExtras.h"
00057 #include "llvm/Support/raw_ostream.h"
00058 #include "llvm/Target/TargetFrameLowering.h"
00059 #include "llvm/Target/TargetInstrInfo.h"
00060 #include "llvm/Target/TargetIntrinsicInfo.h"
00061 #include "llvm/Target/TargetLowering.h"
00062 #include "llvm/Target/TargetOptions.h"
00063 #include "llvm/Target/TargetSelectionDAGInfo.h"
00064 #include "llvm/Target/TargetSubtargetInfo.h"
00065 #include <algorithm>
00066 using namespace llvm;
00067 
00068 #define DEBUG_TYPE "isel"
00069 
00070 /// LimitFloatPrecision - Generate low-precision inline sequences for
00071 /// some float libcalls (6, 8 or 12 bits).
00072 static unsigned LimitFloatPrecision;
00073 
00074 static cl::opt<unsigned, true>
00075 LimitFPPrecision("limit-float-precision",
00076                  cl::desc("Generate low-precision inline sequences "
00077                           "for some float libcalls"),
00078                  cl::location(LimitFloatPrecision),
00079                  cl::init(0));
00080 
00081 // Limit the width of DAG chains. This is important in general to prevent
00082 // prevent DAG-based analysis from blowing up. For example, alias analysis and
00083 // load clustering may not complete in reasonable time. It is difficult to
00084 // recognize and avoid this situation within each individual analysis, and
00085 // future analyses are likely to have the same behavior. Limiting DAG width is
00086 // the safe approach, and will be especially important with global DAGs.
00087 //
00088 // MaxParallelChains default is arbitrarily high to avoid affecting
00089 // optimization, but could be lowered to improve compile time. Any ld-ld-st-st
00090 // sequence over this should have been converted to llvm.memcpy by the
00091 // frontend. It easy to induce this behavior with .ll code such as:
00092 // %buffer = alloca [4096 x i8]
00093 // %data = load [4096 x i8]* %argPtr
00094 // store [4096 x i8] %data, [4096 x i8]* %buffer
00095 static const unsigned MaxParallelChains = 64;
00096 
00097 static SDValue getCopyFromPartsVector(SelectionDAG &DAG, SDLoc DL,
00098                                       const SDValue *Parts, unsigned NumParts,
00099                                       MVT PartVT, EVT ValueVT, const Value *V);
00100 
00101 /// getCopyFromParts - Create a value that contains the specified legal parts
00102 /// combined into the value they represent.  If the parts combine to a type
00103 /// larger then ValueVT then AssertOp can be used to specify whether the extra
00104 /// bits are known to be zero (ISD::AssertZext) or sign extended from ValueVT
00105 /// (ISD::AssertSext).
00106 static SDValue getCopyFromParts(SelectionDAG &DAG, SDLoc DL,
00107                                 const SDValue *Parts,
00108                                 unsigned NumParts, MVT PartVT, EVT ValueVT,
00109                                 const Value *V,
00110                                 ISD::NodeType AssertOp = ISD::DELETED_NODE) {
00111   if (ValueVT.isVector())
00112     return getCopyFromPartsVector(DAG, DL, Parts, NumParts,
00113                                   PartVT, ValueVT, V);
00114 
00115   assert(NumParts > 0 && "No parts to assemble!");
00116   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
00117   SDValue Val = Parts[0];
00118 
00119   if (NumParts > 1) {
00120     // Assemble the value from multiple parts.
00121     if (ValueVT.isInteger()) {
00122       unsigned PartBits = PartVT.getSizeInBits();
00123       unsigned ValueBits = ValueVT.getSizeInBits();
00124 
00125       // Assemble the power of 2 part.
00126       unsigned RoundParts = NumParts & (NumParts - 1) ?
00127         1 << Log2_32(NumParts) : NumParts;
00128       unsigned RoundBits = PartBits * RoundParts;
00129       EVT RoundVT = RoundBits == ValueBits ?
00130         ValueVT : EVT::getIntegerVT(*DAG.getContext(), RoundBits);
00131       SDValue Lo, Hi;
00132 
00133       EVT HalfVT = EVT::getIntegerVT(*DAG.getContext(), RoundBits/2);
00134 
00135       if (RoundParts > 2) {
00136         Lo = getCopyFromParts(DAG, DL, Parts, RoundParts / 2,
00137                               PartVT, HalfVT, V);
00138         Hi = getCopyFromParts(DAG, DL, Parts + RoundParts / 2,
00139                               RoundParts / 2, PartVT, HalfVT, V);
00140       } else {
00141         Lo = DAG.getNode(ISD::BITCAST, DL, HalfVT, Parts[0]);
00142         Hi = DAG.getNode(ISD::BITCAST, DL, HalfVT, Parts[1]);
00143       }
00144 
00145       if (TLI.isBigEndian())
00146         std::swap(Lo, Hi);
00147 
00148       Val = DAG.getNode(ISD::BUILD_PAIR, DL, RoundVT, Lo, Hi);
00149 
00150       if (RoundParts < NumParts) {
00151         // Assemble the trailing non-power-of-2 part.
00152         unsigned OddParts = NumParts - RoundParts;
00153         EVT OddVT = EVT::getIntegerVT(*DAG.getContext(), OddParts * PartBits);
00154         Hi = getCopyFromParts(DAG, DL,
00155                               Parts + RoundParts, OddParts, PartVT, OddVT, V);
00156 
00157         // Combine the round and odd parts.
00158         Lo = Val;
00159         if (TLI.isBigEndian())
00160           std::swap(Lo, Hi);
00161         EVT TotalVT = EVT::getIntegerVT(*DAG.getContext(), NumParts * PartBits);
00162         Hi = DAG.getNode(ISD::ANY_EXTEND, DL, TotalVT, Hi);
00163         Hi = DAG.getNode(ISD::SHL, DL, TotalVT, Hi,
00164                          DAG.getConstant(Lo.getValueType().getSizeInBits(),
00165                                          TLI.getPointerTy()));
00166         Lo = DAG.getNode(ISD::ZERO_EXTEND, DL, TotalVT, Lo);
00167         Val = DAG.getNode(ISD::OR, DL, TotalVT, Lo, Hi);
00168       }
00169     } else if (PartVT.isFloatingPoint()) {
00170       // FP split into multiple FP parts (for ppcf128)
00171       assert(ValueVT == EVT(MVT::ppcf128) && PartVT == MVT::f64 &&
00172              "Unexpected split");
00173       SDValue Lo, Hi;
00174       Lo = DAG.getNode(ISD::BITCAST, DL, EVT(MVT::f64), Parts[0]);
00175       Hi = DAG.getNode(ISD::BITCAST, DL, EVT(MVT::f64), Parts[1]);
00176       if (TLI.hasBigEndianPartOrdering(ValueVT))
00177         std::swap(Lo, Hi);
00178       Val = DAG.getNode(ISD::BUILD_PAIR, DL, ValueVT, Lo, Hi);
00179     } else {
00180       // FP split into integer parts (soft fp)
00181       assert(ValueVT.isFloatingPoint() && PartVT.isInteger() &&
00182              !PartVT.isVector() && "Unexpected split");
00183       EVT IntVT = EVT::getIntegerVT(*DAG.getContext(), ValueVT.getSizeInBits());
00184       Val = getCopyFromParts(DAG, DL, Parts, NumParts, PartVT, IntVT, V);
00185     }
00186   }
00187 
00188   // There is now one part, held in Val.  Correct it to match ValueVT.
00189   EVT PartEVT = Val.getValueType();
00190 
00191   if (PartEVT == ValueVT)
00192     return Val;
00193 
00194   if (PartEVT.isInteger() && ValueVT.isInteger()) {
00195     if (ValueVT.bitsLT(PartEVT)) {
00196       // For a truncate, see if we have any information to
00197       // indicate whether the truncated bits will always be
00198       // zero or sign-extension.
00199       if (AssertOp != ISD::DELETED_NODE)
00200         Val = DAG.getNode(AssertOp, DL, PartEVT, Val,
00201                           DAG.getValueType(ValueVT));
00202       return DAG.getNode(ISD::TRUNCATE, DL, ValueVT, Val);
00203     }
00204     return DAG.getNode(ISD::ANY_EXTEND, DL, ValueVT, Val);
00205   }
00206 
00207   if (PartEVT.isFloatingPoint() && ValueVT.isFloatingPoint()) {
00208     // FP_ROUND's are always exact here.
00209     if (ValueVT.bitsLT(Val.getValueType()))
00210       return DAG.getNode(ISD::FP_ROUND, DL, ValueVT, Val,
00211                          DAG.getTargetConstant(1, TLI.getPointerTy()));
00212 
00213     return DAG.getNode(ISD::FP_EXTEND, DL, ValueVT, Val);
00214   }
00215 
00216   if (PartEVT.getSizeInBits() == ValueVT.getSizeInBits())
00217     return DAG.getNode(ISD::BITCAST, DL, ValueVT, Val);
00218 
00219   llvm_unreachable("Unknown mismatch!");
00220 }
00221 
00222 static void diagnosePossiblyInvalidConstraint(LLVMContext &Ctx, const Value *V,
00223                                               const Twine &ErrMsg) {
00224   const Instruction *I = dyn_cast_or_null<Instruction>(V);
00225   if (!V)
00226     return Ctx.emitError(ErrMsg);
00227 
00228   const char *AsmError = ", possible invalid constraint for vector type";
00229   if (const CallInst *CI = dyn_cast<CallInst>(I))
00230     if (isa<InlineAsm>(CI->getCalledValue()))
00231       return Ctx.emitError(I, ErrMsg + AsmError);
00232 
00233   return Ctx.emitError(I, ErrMsg);
00234 }
00235 
00236 /// getCopyFromPartsVector - Create a value that contains the specified legal
00237 /// parts combined into the value they represent.  If the parts combine to a
00238 /// type larger then ValueVT then AssertOp can be used to specify whether the
00239 /// extra bits are known to be zero (ISD::AssertZext) or sign extended from
00240 /// ValueVT (ISD::AssertSext).
00241 static SDValue getCopyFromPartsVector(SelectionDAG &DAG, SDLoc DL,
00242                                       const SDValue *Parts, unsigned NumParts,
00243                                       MVT PartVT, EVT ValueVT, const Value *V) {
00244   assert(ValueVT.isVector() && "Not a vector value");
00245   assert(NumParts > 0 && "No parts to assemble!");
00246   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
00247   SDValue Val = Parts[0];
00248 
00249   // Handle a multi-element vector.
00250   if (NumParts > 1) {
00251     EVT IntermediateVT;
00252     MVT RegisterVT;
00253     unsigned NumIntermediates;
00254     unsigned NumRegs =
00255     TLI.getVectorTypeBreakdown(*DAG.getContext(), ValueVT, IntermediateVT,
00256                                NumIntermediates, RegisterVT);
00257     assert(NumRegs == NumParts && "Part count doesn't match vector breakdown!");
00258     NumParts = NumRegs; // Silence a compiler warning.
00259     assert(RegisterVT == PartVT && "Part type doesn't match vector breakdown!");
00260     assert(RegisterVT == Parts[0].getSimpleValueType() &&
00261            "Part type doesn't match part!");
00262 
00263     // Assemble the parts into intermediate operands.
00264     SmallVector<SDValue, 8> Ops(NumIntermediates);
00265     if (NumIntermediates == NumParts) {
00266       // If the register was not expanded, truncate or copy the value,
00267       // as appropriate.
00268       for (unsigned i = 0; i != NumParts; ++i)
00269         Ops[i] = getCopyFromParts(DAG, DL, &Parts[i], 1,
00270                                   PartVT, IntermediateVT, V);
00271     } else if (NumParts > 0) {
00272       // If the intermediate type was expanded, build the intermediate
00273       // operands from the parts.
00274       assert(NumParts % NumIntermediates == 0 &&
00275              "Must expand into a divisible number of parts!");
00276       unsigned Factor = NumParts / NumIntermediates;
00277       for (unsigned i = 0; i != NumIntermediates; ++i)
00278         Ops[i] = getCopyFromParts(DAG, DL, &Parts[i * Factor], Factor,
00279                                   PartVT, IntermediateVT, V);
00280     }
00281 
00282     // Build a vector with BUILD_VECTOR or CONCAT_VECTORS from the
00283     // intermediate operands.
00284     Val = DAG.getNode(IntermediateVT.isVector() ? ISD::CONCAT_VECTORS
00285                                                 : ISD::BUILD_VECTOR,
00286                       DL, ValueVT, Ops);
00287   }
00288 
00289   // There is now one part, held in Val.  Correct it to match ValueVT.
00290   EVT PartEVT = Val.getValueType();
00291 
00292   if (PartEVT == ValueVT)
00293     return Val;
00294 
00295   if (PartEVT.isVector()) {
00296     // If the element type of the source/dest vectors are the same, but the
00297     // parts vector has more elements than the value vector, then we have a
00298     // vector widening case (e.g. <2 x float> -> <4 x float>).  Extract the
00299     // elements we want.
00300     if (PartEVT.getVectorElementType() == ValueVT.getVectorElementType()) {
00301       assert(PartEVT.getVectorNumElements() > ValueVT.getVectorNumElements() &&
00302              "Cannot narrow, it would be a lossy transformation");
00303       return DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, ValueVT, Val,
00304                          DAG.getConstant(0, TLI.getVectorIdxTy()));
00305     }
00306 
00307     // Vector/Vector bitcast.
00308     if (ValueVT.getSizeInBits() == PartEVT.getSizeInBits())
00309       return DAG.getNode(ISD::BITCAST, DL, ValueVT, Val);
00310 
00311     assert(PartEVT.getVectorNumElements() == ValueVT.getVectorNumElements() &&
00312       "Cannot handle this kind of promotion");
00313     // Promoted vector extract
00314     bool Smaller = ValueVT.bitsLE(PartEVT);
00315     return DAG.getNode((Smaller ? ISD::TRUNCATE : ISD::ANY_EXTEND),
00316                        DL, ValueVT, Val);
00317 
00318   }
00319 
00320   // Trivial bitcast if the types are the same size and the destination
00321   // vector type is legal.
00322   if (PartEVT.getSizeInBits() == ValueVT.getSizeInBits() &&
00323       TLI.isTypeLegal(ValueVT))
00324     return DAG.getNode(ISD::BITCAST, DL, ValueVT, Val);
00325 
00326   // Handle cases such as i8 -> <1 x i1>
00327   if (ValueVT.getVectorNumElements() != 1) {
00328     diagnosePossiblyInvalidConstraint(*DAG.getContext(), V,
00329                                       "non-trivial scalar-to-vector conversion");
00330     return DAG.getUNDEF(ValueVT);
00331   }
00332 
00333   if (ValueVT.getVectorNumElements() == 1 &&
00334       ValueVT.getVectorElementType() != PartEVT) {
00335     bool Smaller = ValueVT.bitsLE(PartEVT);
00336     Val = DAG.getNode((Smaller ? ISD::TRUNCATE : ISD::ANY_EXTEND),
00337                        DL, ValueVT.getScalarType(), Val);
00338   }
00339 
00340   return DAG.getNode(ISD::BUILD_VECTOR, DL, ValueVT, Val);
00341 }
00342 
00343 static void getCopyToPartsVector(SelectionDAG &DAG, SDLoc dl,
00344                                  SDValue Val, SDValue *Parts, unsigned NumParts,
00345                                  MVT PartVT, const Value *V);
00346 
00347 /// getCopyToParts - Create a series of nodes that contain the specified value
00348 /// split into legal parts.  If the parts contain more bits than Val, then, for
00349 /// integers, ExtendKind can be used to specify how to generate the extra bits.
00350 static void getCopyToParts(SelectionDAG &DAG, SDLoc DL,
00351                            SDValue Val, SDValue *Parts, unsigned NumParts,
00352                            MVT PartVT, const Value *V,
00353                            ISD::NodeType ExtendKind = ISD::ANY_EXTEND) {
00354   EVT ValueVT = Val.getValueType();
00355 
00356   // Handle the vector case separately.
00357   if (ValueVT.isVector())
00358     return getCopyToPartsVector(DAG, DL, Val, Parts, NumParts, PartVT, V);
00359 
00360   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
00361   unsigned PartBits = PartVT.getSizeInBits();
00362   unsigned OrigNumParts = NumParts;
00363   assert(TLI.isTypeLegal(PartVT) && "Copying to an illegal type!");
00364 
00365   if (NumParts == 0)
00366     return;
00367 
00368   assert(!ValueVT.isVector() && "Vector case handled elsewhere");
00369   EVT PartEVT = PartVT;
00370   if (PartEVT == ValueVT) {
00371     assert(NumParts == 1 && "No-op copy with multiple parts!");
00372     Parts[0] = Val;
00373     return;
00374   }
00375 
00376   if (NumParts * PartBits > ValueVT.getSizeInBits()) {
00377     // If the parts cover more bits than the value has, promote the value.
00378     if (PartVT.isFloatingPoint() && ValueVT.isFloatingPoint()) {
00379       assert(NumParts == 1 && "Do not know what to promote to!");
00380       Val = DAG.getNode(ISD::FP_EXTEND, DL, PartVT, Val);
00381     } else {
00382       assert((PartVT.isInteger() || PartVT == MVT::x86mmx) &&
00383              ValueVT.isInteger() &&
00384              "Unknown mismatch!");
00385       ValueVT = EVT::getIntegerVT(*DAG.getContext(), NumParts * PartBits);
00386       Val = DAG.getNode(ExtendKind, DL, ValueVT, Val);
00387       if (PartVT == MVT::x86mmx)
00388         Val = DAG.getNode(ISD::BITCAST, DL, PartVT, Val);
00389     }
00390   } else if (PartBits == ValueVT.getSizeInBits()) {
00391     // Different types of the same size.
00392     assert(NumParts == 1 && PartEVT != ValueVT);
00393     Val = DAG.getNode(ISD::BITCAST, DL, PartVT, Val);
00394   } else if (NumParts * PartBits < ValueVT.getSizeInBits()) {
00395     // If the parts cover less bits than value has, truncate the value.
00396     assert((PartVT.isInteger() || PartVT == MVT::x86mmx) &&
00397            ValueVT.isInteger() &&
00398            "Unknown mismatch!");
00399     ValueVT = EVT::getIntegerVT(*DAG.getContext(), NumParts * PartBits);
00400     Val = DAG.getNode(ISD::TRUNCATE, DL, ValueVT, Val);
00401     if (PartVT == MVT::x86mmx)
00402       Val = DAG.getNode(ISD::BITCAST, DL, PartVT, Val);
00403   }
00404 
00405   // The value may have changed - recompute ValueVT.
00406   ValueVT = Val.getValueType();
00407   assert(NumParts * PartBits == ValueVT.getSizeInBits() &&
00408          "Failed to tile the value with PartVT!");
00409 
00410   if (NumParts == 1) {
00411     if (PartEVT != ValueVT)
00412       diagnosePossiblyInvalidConstraint(*DAG.getContext(), V,
00413                                         "scalar-to-vector conversion failed");
00414 
00415     Parts[0] = Val;
00416     return;
00417   }
00418 
00419   // Expand the value into multiple parts.
00420   if (NumParts & (NumParts - 1)) {
00421     // The number of parts is not a power of 2.  Split off and copy the tail.
00422     assert(PartVT.isInteger() && ValueVT.isInteger() &&
00423            "Do not know what to expand to!");
00424     unsigned RoundParts = 1 << Log2_32(NumParts);
00425     unsigned RoundBits = RoundParts * PartBits;
00426     unsigned OddParts = NumParts - RoundParts;
00427     SDValue OddVal = DAG.getNode(ISD::SRL, DL, ValueVT, Val,
00428                                  DAG.getIntPtrConstant(RoundBits));
00429     getCopyToParts(DAG, DL, OddVal, Parts + RoundParts, OddParts, PartVT, V);
00430 
00431     if (TLI.isBigEndian())
00432       // The odd parts were reversed by getCopyToParts - unreverse them.
00433       std::reverse(Parts + RoundParts, Parts + NumParts);
00434 
00435     NumParts = RoundParts;
00436     ValueVT = EVT::getIntegerVT(*DAG.getContext(), NumParts * PartBits);
00437     Val = DAG.getNode(ISD::TRUNCATE, DL, ValueVT, Val);
00438   }
00439 
00440   // The number of parts is a power of 2.  Repeatedly bisect the value using
00441   // EXTRACT_ELEMENT.
00442   Parts[0] = DAG.getNode(ISD::BITCAST, DL,
00443                          EVT::getIntegerVT(*DAG.getContext(),
00444                                            ValueVT.getSizeInBits()),
00445                          Val);
00446 
00447   for (unsigned StepSize = NumParts; StepSize > 1; StepSize /= 2) {
00448     for (unsigned i = 0; i < NumParts; i += StepSize) {
00449       unsigned ThisBits = StepSize * PartBits / 2;
00450       EVT ThisVT = EVT::getIntegerVT(*DAG.getContext(), ThisBits);
00451       SDValue &Part0 = Parts[i];
00452       SDValue &Part1 = Parts[i+StepSize/2];
00453 
00454       Part1 = DAG.getNode(ISD::EXTRACT_ELEMENT, DL,
00455                           ThisVT, Part0, DAG.getIntPtrConstant(1));
00456       Part0 = DAG.getNode(ISD::EXTRACT_ELEMENT, DL,
00457                           ThisVT, Part0, DAG.getIntPtrConstant(0));
00458 
00459       if (ThisBits == PartBits && ThisVT != PartVT) {
00460         Part0 = DAG.getNode(ISD::BITCAST, DL, PartVT, Part0);
00461         Part1 = DAG.getNode(ISD::BITCAST, DL, PartVT, Part1);
00462       }
00463     }
00464   }
00465 
00466   if (TLI.isBigEndian())
00467     std::reverse(Parts, Parts + OrigNumParts);
00468 }
00469 
00470 
00471 /// getCopyToPartsVector - Create a series of nodes that contain the specified
00472 /// value split into legal parts.
00473 static void getCopyToPartsVector(SelectionDAG &DAG, SDLoc DL,
00474                                  SDValue Val, SDValue *Parts, unsigned NumParts,
00475                                  MVT PartVT, const Value *V) {
00476   EVT ValueVT = Val.getValueType();
00477   assert(ValueVT.isVector() && "Not a vector");
00478   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
00479 
00480   if (NumParts == 1) {
00481     EVT PartEVT = PartVT;
00482     if (PartEVT == ValueVT) {
00483       // Nothing to do.
00484     } else if (PartVT.getSizeInBits() == ValueVT.getSizeInBits()) {
00485       // Bitconvert vector->vector case.
00486       Val = DAG.getNode(ISD::BITCAST, DL, PartVT, Val);
00487     } else if (PartVT.isVector() &&
00488                PartEVT.getVectorElementType() == ValueVT.getVectorElementType() &&
00489                PartEVT.getVectorNumElements() > ValueVT.getVectorNumElements()) {
00490       EVT ElementVT = PartVT.getVectorElementType();
00491       // Vector widening case, e.g. <2 x float> -> <4 x float>.  Shuffle in
00492       // undef elements.
00493       SmallVector<SDValue, 16> Ops;
00494       for (unsigned i = 0, e = ValueVT.getVectorNumElements(); i != e; ++i)
00495         Ops.push_back(DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL,
00496                                   ElementVT, Val, DAG.getConstant(i,
00497                                                   TLI.getVectorIdxTy())));
00498 
00499       for (unsigned i = ValueVT.getVectorNumElements(),
00500            e = PartVT.getVectorNumElements(); i != e; ++i)
00501         Ops.push_back(DAG.getUNDEF(ElementVT));
00502 
00503       Val = DAG.getNode(ISD::BUILD_VECTOR, DL, PartVT, Ops);
00504 
00505       // FIXME: Use CONCAT for 2x -> 4x.
00506 
00507       //SDValue UndefElts = DAG.getUNDEF(VectorTy);
00508       //Val = DAG.getNode(ISD::CONCAT_VECTORS, DL, PartVT, Val, UndefElts);
00509     } else if (PartVT.isVector() &&
00510                PartEVT.getVectorElementType().bitsGE(
00511                  ValueVT.getVectorElementType()) &&
00512                PartEVT.getVectorNumElements() == ValueVT.getVectorNumElements()) {
00513 
00514       // Promoted vector extract
00515       bool Smaller = PartEVT.bitsLE(ValueVT);
00516       Val = DAG.getNode((Smaller ? ISD::TRUNCATE : ISD::ANY_EXTEND),
00517                         DL, PartVT, Val);
00518     } else{
00519       // Vector -> scalar conversion.
00520       assert(ValueVT.getVectorNumElements() == 1 &&
00521              "Only trivial vector-to-scalar conversions should get here!");
00522       Val = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL,
00523                         PartVT, Val, DAG.getConstant(0, TLI.getVectorIdxTy()));
00524 
00525       bool Smaller = ValueVT.bitsLE(PartVT);
00526       Val = DAG.getNode((Smaller ? ISD::TRUNCATE : ISD::ANY_EXTEND),
00527                          DL, PartVT, Val);
00528     }
00529 
00530     Parts[0] = Val;
00531     return;
00532   }
00533 
00534   // Handle a multi-element vector.
00535   EVT IntermediateVT;
00536   MVT RegisterVT;
00537   unsigned NumIntermediates;
00538   unsigned NumRegs = TLI.getVectorTypeBreakdown(*DAG.getContext(), ValueVT,
00539                                                 IntermediateVT,
00540                                                 NumIntermediates, RegisterVT);
00541   unsigned NumElements = ValueVT.getVectorNumElements();
00542 
00543   assert(NumRegs == NumParts && "Part count doesn't match vector breakdown!");
00544   NumParts = NumRegs; // Silence a compiler warning.
00545   assert(RegisterVT == PartVT && "Part type doesn't match vector breakdown!");
00546 
00547   // Split the vector into intermediate operands.
00548   SmallVector<SDValue, 8> Ops(NumIntermediates);
00549   for (unsigned i = 0; i != NumIntermediates; ++i) {
00550     if (IntermediateVT.isVector())
00551       Ops[i] = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL,
00552                            IntermediateVT, Val,
00553                    DAG.getConstant(i * (NumElements / NumIntermediates),
00554                                    TLI.getVectorIdxTy()));
00555     else
00556       Ops[i] = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL,
00557                            IntermediateVT, Val,
00558                            DAG.getConstant(i, TLI.getVectorIdxTy()));
00559   }
00560 
00561   // Split the intermediate operands into legal parts.
00562   if (NumParts == NumIntermediates) {
00563     // If the register was not expanded, promote or copy the value,
00564     // as appropriate.
00565     for (unsigned i = 0; i != NumParts; ++i)
00566       getCopyToParts(DAG, DL, Ops[i], &Parts[i], 1, PartVT, V);
00567   } else if (NumParts > 0) {
00568     // If the intermediate type was expanded, split each the value into
00569     // legal parts.
00570     assert(NumIntermediates != 0 && "division by zero");
00571     assert(NumParts % NumIntermediates == 0 &&
00572            "Must expand into a divisible number of parts!");
00573     unsigned Factor = NumParts / NumIntermediates;
00574     for (unsigned i = 0; i != NumIntermediates; ++i)
00575       getCopyToParts(DAG, DL, Ops[i], &Parts[i*Factor], Factor, PartVT, V);
00576   }
00577 }
00578 
00579 namespace {
00580   /// RegsForValue - This struct represents the registers (physical or virtual)
00581   /// that a particular set of values is assigned, and the type information
00582   /// about the value. The most common situation is to represent one value at a
00583   /// time, but struct or array values are handled element-wise as multiple
00584   /// values.  The splitting of aggregates is performed recursively, so that we
00585   /// never have aggregate-typed registers. The values at this point do not
00586   /// necessarily have legal types, so each value may require one or more
00587   /// registers of some legal type.
00588   ///
00589   struct RegsForValue {
00590     /// ValueVTs - The value types of the values, which may not be legal, and
00591     /// may need be promoted or synthesized from one or more registers.
00592     ///
00593     SmallVector<EVT, 4> ValueVTs;
00594 
00595     /// RegVTs - The value types of the registers. This is the same size as
00596     /// ValueVTs and it records, for each value, what the type of the assigned
00597     /// register or registers are. (Individual values are never synthesized
00598     /// from more than one type of register.)
00599     ///
00600     /// With virtual registers, the contents of RegVTs is redundant with TLI's
00601     /// getRegisterType member function, however when with physical registers
00602     /// it is necessary to have a separate record of the types.
00603     ///
00604     SmallVector<MVT, 4> RegVTs;
00605 
00606     /// Regs - This list holds the registers assigned to the values.
00607     /// Each legal or promoted value requires one register, and each
00608     /// expanded value requires multiple registers.
00609     ///
00610     SmallVector<unsigned, 4> Regs;
00611 
00612     RegsForValue() {}
00613 
00614     RegsForValue(const SmallVector<unsigned, 4> &regs,
00615                  MVT regvt, EVT valuevt)
00616       : ValueVTs(1, valuevt), RegVTs(1, regvt), Regs(regs) {}
00617 
00618     RegsForValue(LLVMContext &Context, const TargetLowering &tli,
00619                  unsigned Reg, Type *Ty) {
00620       ComputeValueVTs(tli, Ty, ValueVTs);
00621 
00622       for (unsigned Value = 0, e = ValueVTs.size(); Value != e; ++Value) {
00623         EVT ValueVT = ValueVTs[Value];
00624         unsigned NumRegs = tli.getNumRegisters(Context, ValueVT);
00625         MVT RegisterVT = tli.getRegisterType(Context, ValueVT);
00626         for (unsigned i = 0; i != NumRegs; ++i)
00627           Regs.push_back(Reg + i);
00628         RegVTs.push_back(RegisterVT);
00629         Reg += NumRegs;
00630       }
00631     }
00632 
00633     /// append - Add the specified values to this one.
00634     void append(const RegsForValue &RHS) {
00635       ValueVTs.append(RHS.ValueVTs.begin(), RHS.ValueVTs.end());
00636       RegVTs.append(RHS.RegVTs.begin(), RHS.RegVTs.end());
00637       Regs.append(RHS.Regs.begin(), RHS.Regs.end());
00638     }
00639 
00640     /// getCopyFromRegs - Emit a series of CopyFromReg nodes that copies from
00641     /// this value and returns the result as a ValueVTs value.  This uses
00642     /// Chain/Flag as the input and updates them for the output Chain/Flag.
00643     /// If the Flag pointer is NULL, no flag is used.
00644     SDValue getCopyFromRegs(SelectionDAG &DAG, FunctionLoweringInfo &FuncInfo,
00645                             SDLoc dl,
00646                             SDValue &Chain, SDValue *Flag,
00647                             const Value *V = nullptr) const;
00648 
00649     /// getCopyToRegs - Emit a series of CopyToReg nodes that copies the
00650     /// specified value into the registers specified by this object.  This uses
00651     /// Chain/Flag as the input and updates them for the output Chain/Flag.
00652     /// If the Flag pointer is NULL, no flag is used.
00653     void
00654     getCopyToRegs(SDValue Val, SelectionDAG &DAG, SDLoc dl, SDValue &Chain,
00655                   SDValue *Flag, const Value *V,
00656                   ISD::NodeType PreferredExtendType = ISD::ANY_EXTEND) const;
00657 
00658     /// AddInlineAsmOperands - Add this value to the specified inlineasm node
00659     /// operand list.  This adds the code marker, matching input operand index
00660     /// (if applicable), and includes the number of values added into it.
00661     void AddInlineAsmOperands(unsigned Kind,
00662                               bool HasMatching, unsigned MatchingIdx,
00663                               SelectionDAG &DAG,
00664                               std::vector<SDValue> &Ops) const;
00665   };
00666 }
00667 
00668 /// getCopyFromRegs - Emit a series of CopyFromReg nodes that copies from
00669 /// this value and returns the result as a ValueVT value.  This uses
00670 /// Chain/Flag as the input and updates them for the output Chain/Flag.
00671 /// If the Flag pointer is NULL, no flag is used.
00672 SDValue RegsForValue::getCopyFromRegs(SelectionDAG &DAG,
00673                                       FunctionLoweringInfo &FuncInfo,
00674                                       SDLoc dl,
00675                                       SDValue &Chain, SDValue *Flag,
00676                                       const Value *V) const {
00677   // A Value with type {} or [0 x %t] needs no registers.
00678   if (ValueVTs.empty())
00679     return SDValue();
00680 
00681   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
00682 
00683   // Assemble the legal parts into the final values.
00684   SmallVector<SDValue, 4> Values(ValueVTs.size());
00685   SmallVector<SDValue, 8> Parts;
00686   for (unsigned Value = 0, Part = 0, e = ValueVTs.size(); Value != e; ++Value) {
00687     // Copy the legal parts from the registers.
00688     EVT ValueVT = ValueVTs[Value];
00689     unsigned NumRegs = TLI.getNumRegisters(*DAG.getContext(), ValueVT);
00690     MVT RegisterVT = RegVTs[Value];
00691 
00692     Parts.resize(NumRegs);
00693     for (unsigned i = 0; i != NumRegs; ++i) {
00694       SDValue P;
00695       if (!Flag) {
00696         P = DAG.getCopyFromReg(Chain, dl, Regs[Part+i], RegisterVT);
00697       } else {
00698         P = DAG.getCopyFromReg(Chain, dl, Regs[Part+i], RegisterVT, *Flag);
00699         *Flag = P.getValue(2);
00700       }
00701 
00702       Chain = P.getValue(1);
00703       Parts[i] = P;
00704 
00705       // If the source register was virtual and if we know something about it,
00706       // add an assert node.
00707       if (!TargetRegisterInfo::isVirtualRegister(Regs[Part+i]) ||
00708           !RegisterVT.isInteger() || RegisterVT.isVector())
00709         continue;
00710 
00711       const FunctionLoweringInfo::LiveOutInfo *LOI =
00712         FuncInfo.GetLiveOutRegInfo(Regs[Part+i]);
00713       if (!LOI)
00714         continue;
00715 
00716       unsigned RegSize = RegisterVT.getSizeInBits();
00717       unsigned NumSignBits = LOI->NumSignBits;
00718       unsigned NumZeroBits = LOI->KnownZero.countLeadingOnes();
00719 
00720       if (NumZeroBits == RegSize) {
00721         // The current value is a zero.
00722         // Explicitly express that as it would be easier for
00723         // optimizations to kick in.
00724         Parts[i] = DAG.getConstant(0, RegisterVT);
00725         continue;
00726       }
00727 
00728       // FIXME: We capture more information than the dag can represent.  For
00729       // now, just use the tightest assertzext/assertsext possible.
00730       bool isSExt = true;
00731       EVT FromVT(MVT::Other);
00732       if (NumSignBits == RegSize)
00733         isSExt = true, FromVT = MVT::i1;   // ASSERT SEXT 1
00734       else if (NumZeroBits >= RegSize-1)
00735         isSExt = false, FromVT = MVT::i1;  // ASSERT ZEXT 1
00736       else if (NumSignBits > RegSize-8)
00737         isSExt = true, FromVT = MVT::i8;   // ASSERT SEXT 8
00738       else if (NumZeroBits >= RegSize-8)
00739         isSExt = false, FromVT = MVT::i8;  // ASSERT ZEXT 8
00740       else if (NumSignBits > RegSize-16)
00741         isSExt = true, FromVT = MVT::i16;  // ASSERT SEXT 16
00742       else if (NumZeroBits >= RegSize-16)
00743         isSExt = false, FromVT = MVT::i16; // ASSERT ZEXT 16
00744       else if (NumSignBits > RegSize-32)
00745         isSExt = true, FromVT = MVT::i32;  // ASSERT SEXT 32
00746       else if (NumZeroBits >= RegSize-32)
00747         isSExt = false, FromVT = MVT::i32; // ASSERT ZEXT 32
00748       else
00749         continue;
00750 
00751       // Add an assertion node.
00752       assert(FromVT != MVT::Other);
00753       Parts[i] = DAG.getNode(isSExt ? ISD::AssertSext : ISD::AssertZext, dl,
00754                              RegisterVT, P, DAG.getValueType(FromVT));
00755     }
00756 
00757     Values[Value] = getCopyFromParts(DAG, dl, Parts.begin(),
00758                                      NumRegs, RegisterVT, ValueVT, V);
00759     Part += NumRegs;
00760     Parts.clear();
00761   }
00762 
00763   return DAG.getNode(ISD::MERGE_VALUES, dl, DAG.getVTList(ValueVTs), Values);
00764 }
00765 
00766 /// getCopyToRegs - Emit a series of CopyToReg nodes that copies the
00767 /// specified value into the registers specified by this object.  This uses
00768 /// Chain/Flag as the input and updates them for the output Chain/Flag.
00769 /// If the Flag pointer is NULL, no flag is used.
00770 void RegsForValue::getCopyToRegs(SDValue Val, SelectionDAG &DAG, SDLoc dl,
00771                                  SDValue &Chain, SDValue *Flag, const Value *V,
00772                                  ISD::NodeType PreferredExtendType) const {
00773   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
00774   ISD::NodeType ExtendKind = PreferredExtendType;
00775 
00776   // Get the list of the values's legal parts.
00777   unsigned NumRegs = Regs.size();
00778   SmallVector<SDValue, 8> Parts(NumRegs);
00779   for (unsigned Value = 0, Part = 0, e = ValueVTs.size(); Value != e; ++Value) {
00780     EVT ValueVT = ValueVTs[Value];
00781     unsigned NumParts = TLI.getNumRegisters(*DAG.getContext(), ValueVT);
00782     MVT RegisterVT = RegVTs[Value];
00783 
00784     if (ExtendKind == ISD::ANY_EXTEND && TLI.isZExtFree(Val, RegisterVT))
00785       ExtendKind = ISD::ZERO_EXTEND;
00786 
00787     getCopyToParts(DAG, dl, Val.getValue(Val.getResNo() + Value),
00788                    &Parts[Part], NumParts, RegisterVT, V, ExtendKind);
00789     Part += NumParts;
00790   }
00791 
00792   // Copy the parts into the registers.
00793   SmallVector<SDValue, 8> Chains(NumRegs);
00794   for (unsigned i = 0; i != NumRegs; ++i) {
00795     SDValue Part;
00796     if (!Flag) {
00797       Part = DAG.getCopyToReg(Chain, dl, Regs[i], Parts[i]);
00798     } else {
00799       Part = DAG.getCopyToReg(Chain, dl, Regs[i], Parts[i], *Flag);
00800       *Flag = Part.getValue(1);
00801     }
00802 
00803     Chains[i] = Part.getValue(0);
00804   }
00805 
00806   if (NumRegs == 1 || Flag)
00807     // If NumRegs > 1 && Flag is used then the use of the last CopyToReg is
00808     // flagged to it. That is the CopyToReg nodes and the user are considered
00809     // a single scheduling unit. If we create a TokenFactor and return it as
00810     // chain, then the TokenFactor is both a predecessor (operand) of the
00811     // user as well as a successor (the TF operands are flagged to the user).
00812     // c1, f1 = CopyToReg
00813     // c2, f2 = CopyToReg
00814     // c3     = TokenFactor c1, c2
00815     // ...
00816     //        = op c3, ..., f2
00817     Chain = Chains[NumRegs-1];
00818   else
00819     Chain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Chains);
00820 }
00821 
00822 /// AddInlineAsmOperands - Add this value to the specified inlineasm node
00823 /// operand list.  This adds the code marker and includes the number of
00824 /// values added into it.
00825 void RegsForValue::AddInlineAsmOperands(unsigned Code, bool HasMatching,
00826                                         unsigned MatchingIdx,
00827                                         SelectionDAG &DAG,
00828                                         std::vector<SDValue> &Ops) const {
00829   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
00830 
00831   unsigned Flag = InlineAsm::getFlagWord(Code, Regs.size());
00832   if (HasMatching)
00833     Flag = InlineAsm::getFlagWordForMatchingOp(Flag, MatchingIdx);
00834   else if (!Regs.empty() &&
00835            TargetRegisterInfo::isVirtualRegister(Regs.front())) {
00836     // Put the register class of the virtual registers in the flag word.  That
00837     // way, later passes can recompute register class constraints for inline
00838     // assembly as well as normal instructions.
00839     // Don't do this for tied operands that can use the regclass information
00840     // from the def.
00841     const MachineRegisterInfo &MRI = DAG.getMachineFunction().getRegInfo();
00842     const TargetRegisterClass *RC = MRI.getRegClass(Regs.front());
00843     Flag = InlineAsm::getFlagWordForRegClass(Flag, RC->getID());
00844   }
00845 
00846   SDValue Res = DAG.getTargetConstant(Flag, MVT::i32);
00847   Ops.push_back(Res);
00848 
00849   unsigned SP = TLI.getStackPointerRegisterToSaveRestore();
00850   for (unsigned Value = 0, Reg = 0, e = ValueVTs.size(); Value != e; ++Value) {
00851     unsigned NumRegs = TLI.getNumRegisters(*DAG.getContext(), ValueVTs[Value]);
00852     MVT RegisterVT = RegVTs[Value];
00853     for (unsigned i = 0; i != NumRegs; ++i) {
00854       assert(Reg < Regs.size() && "Mismatch in # registers expected");
00855       unsigned TheReg = Regs[Reg++];
00856       Ops.push_back(DAG.getRegister(TheReg, RegisterVT));
00857 
00858       if (TheReg == SP && Code == InlineAsm::Kind_Clobber) {
00859         // If we clobbered the stack pointer, MFI should know about it.
00860         assert(DAG.getMachineFunction().getFrameInfo()->
00861             hasInlineAsmWithSPAdjust());
00862       }
00863     }
00864   }
00865 }
00866 
00867 void SelectionDAGBuilder::init(GCFunctionInfo *gfi, AliasAnalysis &aa,
00868                                const TargetLibraryInfo *li) {
00869   AA = &aa;
00870   GFI = gfi;
00871   LibInfo = li;
00872   DL = DAG.getTarget().getDataLayout();
00873   Context = DAG.getContext();
00874   LPadToCallSiteMap.clear();
00875 }
00876 
00877 /// clear - Clear out the current SelectionDAG and the associated
00878 /// state and prepare this SelectionDAGBuilder object to be used
00879 /// for a new block. This doesn't clear out information about
00880 /// additional blocks that are needed to complete switch lowering
00881 /// or PHI node updating; that information is cleared out as it is
00882 /// consumed.
00883 void SelectionDAGBuilder::clear() {
00884   NodeMap.clear();
00885   UnusedArgNodeMap.clear();
00886   PendingLoads.clear();
00887   PendingExports.clear();
00888   CurInst = nullptr;
00889   HasTailCall = false;
00890   SDNodeOrder = LowestSDNodeOrder;
00891   StatepointLowering.clear();
00892 }
00893 
00894 /// clearDanglingDebugInfo - Clear the dangling debug information
00895 /// map. This function is separated from the clear so that debug
00896 /// information that is dangling in a basic block can be properly
00897 /// resolved in a different basic block. This allows the
00898 /// SelectionDAG to resolve dangling debug information attached
00899 /// to PHI nodes.
00900 void SelectionDAGBuilder::clearDanglingDebugInfo() {
00901   DanglingDebugInfoMap.clear();
00902 }
00903 
00904 /// getRoot - Return the current virtual root of the Selection DAG,
00905 /// flushing any PendingLoad items. This must be done before emitting
00906 /// a store or any other node that may need to be ordered after any
00907 /// prior load instructions.
00908 ///
00909 SDValue SelectionDAGBuilder::getRoot() {
00910   if (PendingLoads.empty())
00911     return DAG.getRoot();
00912 
00913   if (PendingLoads.size() == 1) {
00914     SDValue Root = PendingLoads[0];
00915     DAG.setRoot(Root);
00916     PendingLoads.clear();
00917     return Root;
00918   }
00919 
00920   // Otherwise, we have to make a token factor node.
00921   SDValue Root = DAG.getNode(ISD::TokenFactor, getCurSDLoc(), MVT::Other,
00922                              PendingLoads);
00923   PendingLoads.clear();
00924   DAG.setRoot(Root);
00925   return Root;
00926 }
00927 
00928 /// getControlRoot - Similar to getRoot, but instead of flushing all the
00929 /// PendingLoad items, flush all the PendingExports items. It is necessary
00930 /// to do this before emitting a terminator instruction.
00931 ///
00932 SDValue SelectionDAGBuilder::getControlRoot() {
00933   SDValue Root = DAG.getRoot();
00934 
00935   if (PendingExports.empty())
00936     return Root;
00937 
00938   // Turn all of the CopyToReg chains into one factored node.
00939   if (Root.getOpcode() != ISD::EntryToken) {
00940     unsigned i = 0, e = PendingExports.size();
00941     for (; i != e; ++i) {
00942       assert(PendingExports[i].getNode()->getNumOperands() > 1);
00943       if (PendingExports[i].getNode()->getOperand(0) == Root)
00944         break;  // Don't add the root if we already indirectly depend on it.
00945     }
00946 
00947     if (i == e)
00948       PendingExports.push_back(Root);
00949   }
00950 
00951   Root = DAG.getNode(ISD::TokenFactor, getCurSDLoc(), MVT::Other,
00952                      PendingExports);
00953   PendingExports.clear();
00954   DAG.setRoot(Root);
00955   return Root;
00956 }
00957 
00958 void SelectionDAGBuilder::visit(const Instruction &I) {
00959   // Set up outgoing PHI node register values before emitting the terminator.
00960   if (isa<TerminatorInst>(&I))
00961     HandlePHINodesInSuccessorBlocks(I.getParent());
00962 
00963   ++SDNodeOrder;
00964 
00965   CurInst = &I;
00966 
00967   visit(I.getOpcode(), I);
00968 
00969   if (!isa<TerminatorInst>(&I) && !HasTailCall)
00970     CopyToExportRegsIfNeeded(&I);
00971 
00972   CurInst = nullptr;
00973 }
00974 
00975 void SelectionDAGBuilder::visitPHI(const PHINode &) {
00976   llvm_unreachable("SelectionDAGBuilder shouldn't visit PHI nodes!");
00977 }
00978 
00979 void SelectionDAGBuilder::visit(unsigned Opcode, const User &I) {
00980   // Note: this doesn't use InstVisitor, because it has to work with
00981   // ConstantExpr's in addition to instructions.
00982   switch (Opcode) {
00983   default: llvm_unreachable("Unknown instruction type encountered!");
00984     // Build the switch statement using the Instruction.def file.
00985 #define HANDLE_INST(NUM, OPCODE, CLASS) \
00986     case Instruction::OPCODE: visit##OPCODE((const CLASS&)I); break;
00987 #include "llvm/IR/Instruction.def"
00988   }
00989 }
00990 
00991 // resolveDanglingDebugInfo - if we saw an earlier dbg_value referring to V,
00992 // generate the debug data structures now that we've seen its definition.
00993 void SelectionDAGBuilder::resolveDanglingDebugInfo(const Value *V,
00994                                                    SDValue Val) {
00995   DanglingDebugInfo &DDI = DanglingDebugInfoMap[V];
00996   if (DDI.getDI()) {
00997     const DbgValueInst *DI = DDI.getDI();
00998     DebugLoc dl = DDI.getdl();
00999     unsigned DbgSDNodeOrder = DDI.getSDNodeOrder();
01000     MDNode *Variable = DI->getVariable();
01001     MDNode *Expr = DI->getExpression();
01002     uint64_t Offset = DI->getOffset();
01003     // A dbg.value for an alloca is always indirect.
01004     bool IsIndirect = isa<AllocaInst>(V) || Offset != 0;
01005     SDDbgValue *SDV;
01006     if (Val.getNode()) {
01007       if (!EmitFuncArgumentDbgValue(V, Variable, Expr, Offset, IsIndirect,
01008                                     Val)) {
01009         SDV = DAG.getDbgValue(Variable, Expr, Val.getNode(), Val.getResNo(),
01010                               IsIndirect, Offset, dl, DbgSDNodeOrder);
01011         DAG.AddDbgValue(SDV, Val.getNode(), false);
01012       }
01013     } else
01014       DEBUG(dbgs() << "Dropping debug info for " << *DI << "\n");
01015     DanglingDebugInfoMap[V] = DanglingDebugInfo();
01016   }
01017 }
01018 
01019 /// getCopyFromRegs - If there was virtual register allocated for the value V
01020 /// emit CopyFromReg of the specified type Ty. Return empty SDValue() otherwise.
01021 SDValue SelectionDAGBuilder::getCopyFromRegs(const Value *V, Type *Ty) {
01022   DenseMap<const Value *, unsigned>::iterator It = FuncInfo.ValueMap.find(V);
01023   SDValue res;
01024 
01025   if (It != FuncInfo.ValueMap.end()) {
01026     unsigned InReg = It->second;
01027     RegsForValue RFV(*DAG.getContext(), DAG.getTargetLoweringInfo(), InReg,
01028                      Ty);
01029     SDValue Chain = DAG.getEntryNode();
01030     res = RFV.getCopyFromRegs(DAG, FuncInfo, getCurSDLoc(), Chain, nullptr, V);
01031     resolveDanglingDebugInfo(V, res);
01032   }
01033 
01034   return res;
01035 }
01036 
01037 /// getValue - Return an SDValue for the given Value.
01038 SDValue SelectionDAGBuilder::getValue(const Value *V) {
01039   // If we already have an SDValue for this value, use it. It's important
01040   // to do this first, so that we don't create a CopyFromReg if we already
01041   // have a regular SDValue.
01042   SDValue &N = NodeMap[V];
01043   if (N.getNode()) return N;
01044 
01045   // If there's a virtual register allocated and initialized for this
01046   // value, use it.
01047   SDValue copyFromReg = getCopyFromRegs(V, V->getType());
01048   if (copyFromReg.getNode()) {
01049     return copyFromReg;
01050   }
01051 
01052   // Otherwise create a new SDValue and remember it.
01053   SDValue Val = getValueImpl(V);
01054   NodeMap[V] = Val;
01055   resolveDanglingDebugInfo(V, Val);
01056   return Val;
01057 }
01058 
01059 /// getNonRegisterValue - Return an SDValue for the given Value, but
01060 /// don't look in FuncInfo.ValueMap for a virtual register.
01061 SDValue SelectionDAGBuilder::getNonRegisterValue(const Value *V) {
01062   // If we already have an SDValue for this value, use it.
01063   SDValue &N = NodeMap[V];
01064   if (N.getNode()) return N;
01065 
01066   // Otherwise create a new SDValue and remember it.
01067   SDValue Val = getValueImpl(V);
01068   NodeMap[V] = Val;
01069   resolveDanglingDebugInfo(V, Val);
01070   return Val;
01071 }
01072 
01073 /// getValueImpl - Helper function for getValue and getNonRegisterValue.
01074 /// Create an SDValue for the given value.
01075 SDValue SelectionDAGBuilder::getValueImpl(const Value *V) {
01076   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
01077 
01078   if (const Constant *C = dyn_cast<Constant>(V)) {
01079     EVT VT = TLI.getValueType(V->getType(), true);
01080 
01081     if (const ConstantInt *CI = dyn_cast<ConstantInt>(C))
01082       return DAG.getConstant(*CI, VT);
01083 
01084     if (const GlobalValue *GV = dyn_cast<GlobalValue>(C))
01085       return DAG.getGlobalAddress(GV, getCurSDLoc(), VT);
01086 
01087     if (isa<ConstantPointerNull>(C)) {
01088       unsigned AS = V->getType()->getPointerAddressSpace();
01089       return DAG.getConstant(0, TLI.getPointerTy(AS));
01090     }
01091 
01092     if (const ConstantFP *CFP = dyn_cast<ConstantFP>(C))
01093       return DAG.getConstantFP(*CFP, VT);
01094 
01095     if (isa<UndefValue>(C) && !V->getType()->isAggregateType())
01096       return DAG.getUNDEF(VT);
01097 
01098     if (const ConstantExpr *CE = dyn_cast<ConstantExpr>(C)) {
01099       visit(CE->getOpcode(), *CE);
01100       SDValue N1 = NodeMap[V];
01101       assert(N1.getNode() && "visit didn't populate the NodeMap!");
01102       return N1;
01103     }
01104 
01105     if (isa<ConstantStruct>(C) || isa<ConstantArray>(C)) {
01106       SmallVector<SDValue, 4> Constants;
01107       for (User::const_op_iterator OI = C->op_begin(), OE = C->op_end();
01108            OI != OE; ++OI) {
01109         SDNode *Val = getValue(*OI).getNode();
01110         // If the operand is an empty aggregate, there are no values.
01111         if (!Val) continue;
01112         // Add each leaf value from the operand to the Constants list
01113         // to form a flattened list of all the values.
01114         for (unsigned i = 0, e = Val->getNumValues(); i != e; ++i)
01115           Constants.push_back(SDValue(Val, i));
01116       }
01117 
01118       return DAG.getMergeValues(Constants, getCurSDLoc());
01119     }
01120 
01121     if (const ConstantDataSequential *CDS =
01122           dyn_cast<ConstantDataSequential>(C)) {
01123       SmallVector<SDValue, 4> Ops;
01124       for (unsigned i = 0, e = CDS->getNumElements(); i != e; ++i) {
01125         SDNode *Val = getValue(CDS->getElementAsConstant(i)).getNode();
01126         // Add each leaf value from the operand to the Constants list
01127         // to form a flattened list of all the values.
01128         for (unsigned i = 0, e = Val->getNumValues(); i != e; ++i)
01129           Ops.push_back(SDValue(Val, i));
01130       }
01131 
01132       if (isa<ArrayType>(CDS->getType()))
01133         return DAG.getMergeValues(Ops, getCurSDLoc());
01134       return NodeMap[V] = DAG.getNode(ISD::BUILD_VECTOR, getCurSDLoc(),
01135                                       VT, Ops);
01136     }
01137 
01138     if (C->getType()->isStructTy() || C->getType()->isArrayTy()) {
01139       assert((isa<ConstantAggregateZero>(C) || isa<UndefValue>(C)) &&
01140              "Unknown struct or array constant!");
01141 
01142       SmallVector<EVT, 4> ValueVTs;
01143       ComputeValueVTs(TLI, C->getType(), ValueVTs);
01144       unsigned NumElts = ValueVTs.size();
01145       if (NumElts == 0)
01146         return SDValue(); // empty struct
01147       SmallVector<SDValue, 4> Constants(NumElts);
01148       for (unsigned i = 0; i != NumElts; ++i) {
01149         EVT EltVT = ValueVTs[i];
01150         if (isa<UndefValue>(C))
01151           Constants[i] = DAG.getUNDEF(EltVT);
01152         else if (EltVT.isFloatingPoint())
01153           Constants[i] = DAG.getConstantFP(0, EltVT);
01154         else
01155           Constants[i] = DAG.getConstant(0, EltVT);
01156       }
01157 
01158       return DAG.getMergeValues(Constants, getCurSDLoc());
01159     }
01160 
01161     if (const BlockAddress *BA = dyn_cast<BlockAddress>(C))
01162       return DAG.getBlockAddress(BA, VT);
01163 
01164     VectorType *VecTy = cast<VectorType>(V->getType());
01165     unsigned NumElements = VecTy->getNumElements();
01166 
01167     // Now that we know the number and type of the elements, get that number of
01168     // elements into the Ops array based on what kind of constant it is.
01169     SmallVector<SDValue, 16> Ops;
01170     if (const ConstantVector *CV = dyn_cast<ConstantVector>(C)) {
01171       for (unsigned i = 0; i != NumElements; ++i)
01172         Ops.push_back(getValue(CV->getOperand(i)));
01173     } else {
01174       assert(isa<ConstantAggregateZero>(C) && "Unknown vector constant!");
01175       EVT EltVT = TLI.getValueType(VecTy->getElementType());
01176 
01177       SDValue Op;
01178       if (EltVT.isFloatingPoint())
01179         Op = DAG.getConstantFP(0, EltVT);
01180       else
01181         Op = DAG.getConstant(0, EltVT);
01182       Ops.assign(NumElements, Op);
01183     }
01184 
01185     // Create a BUILD_VECTOR node.
01186     return NodeMap[V] = DAG.getNode(ISD::BUILD_VECTOR, getCurSDLoc(), VT, Ops);
01187   }
01188 
01189   // If this is a static alloca, generate it as the frameindex instead of
01190   // computation.
01191   if (const AllocaInst *AI = dyn_cast<AllocaInst>(V)) {
01192     DenseMap<const AllocaInst*, int>::iterator SI =
01193       FuncInfo.StaticAllocaMap.find(AI);
01194     if (SI != FuncInfo.StaticAllocaMap.end())
01195       return DAG.getFrameIndex(SI->second, TLI.getPointerTy());
01196   }
01197 
01198   // If this is an instruction which fast-isel has deferred, select it now.
01199   if (const Instruction *Inst = dyn_cast<Instruction>(V)) {
01200     unsigned InReg = FuncInfo.InitializeRegForValue(Inst);
01201     RegsForValue RFV(*DAG.getContext(), TLI, InReg, Inst->getType());
01202     SDValue Chain = DAG.getEntryNode();
01203     return RFV.getCopyFromRegs(DAG, FuncInfo, getCurSDLoc(), Chain, nullptr, V);
01204   }
01205 
01206   llvm_unreachable("Can't get register for value!");
01207 }
01208 
01209 void SelectionDAGBuilder::visitRet(const ReturnInst &I) {
01210   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
01211   SDValue Chain = getControlRoot();
01212   SmallVector<ISD::OutputArg, 8> Outs;
01213   SmallVector<SDValue, 8> OutVals;
01214 
01215   if (!FuncInfo.CanLowerReturn) {
01216     unsigned DemoteReg = FuncInfo.DemoteRegister;
01217     const Function *F = I.getParent()->getParent();
01218 
01219     // Emit a store of the return value through the virtual register.
01220     // Leave Outs empty so that LowerReturn won't try to load return
01221     // registers the usual way.
01222     SmallVector<EVT, 1> PtrValueVTs;
01223     ComputeValueVTs(TLI, PointerType::getUnqual(F->getReturnType()),
01224                     PtrValueVTs);
01225 
01226     SDValue RetPtr = DAG.getRegister(DemoteReg, PtrValueVTs[0]);
01227     SDValue RetOp = getValue(I.getOperand(0));
01228 
01229     SmallVector<EVT, 4> ValueVTs;
01230     SmallVector<uint64_t, 4> Offsets;
01231     ComputeValueVTs(TLI, I.getOperand(0)->getType(), ValueVTs, &Offsets);
01232     unsigned NumValues = ValueVTs.size();
01233 
01234     SmallVector<SDValue, 4> Chains(NumValues);
01235     for (unsigned i = 0; i != NumValues; ++i) {
01236       SDValue Add = DAG.getNode(ISD::ADD, getCurSDLoc(),
01237                                 RetPtr.getValueType(), RetPtr,
01238                                 DAG.getIntPtrConstant(Offsets[i]));
01239       Chains[i] =
01240         DAG.getStore(Chain, getCurSDLoc(),
01241                      SDValue(RetOp.getNode(), RetOp.getResNo() + i),
01242                      // FIXME: better loc info would be nice.
01243                      Add, MachinePointerInfo(), false, false, 0);
01244     }
01245 
01246     Chain = DAG.getNode(ISD::TokenFactor, getCurSDLoc(),
01247                         MVT::Other, Chains);
01248   } else if (I.getNumOperands() != 0) {
01249     SmallVector<EVT, 4> ValueVTs;
01250     ComputeValueVTs(TLI, I.getOperand(0)->getType(), ValueVTs);
01251     unsigned NumValues = ValueVTs.size();
01252     if (NumValues) {
01253       SDValue RetOp = getValue(I.getOperand(0));
01254 
01255       const Function *F = I.getParent()->getParent();
01256 
01257       ISD::NodeType ExtendKind = ISD::ANY_EXTEND;
01258       if (F->getAttributes().hasAttribute(AttributeSet::ReturnIndex,
01259                                           Attribute::SExt))
01260         ExtendKind = ISD::SIGN_EXTEND;
01261       else if (F->getAttributes().hasAttribute(AttributeSet::ReturnIndex,
01262                                                Attribute::ZExt))
01263         ExtendKind = ISD::ZERO_EXTEND;
01264 
01265       LLVMContext &Context = F->getContext();
01266       bool RetInReg = F->getAttributes().hasAttribute(AttributeSet::ReturnIndex,
01267                                                       Attribute::InReg);
01268 
01269       for (unsigned j = 0; j != NumValues; ++j) {
01270         EVT VT = ValueVTs[j];
01271 
01272         if (ExtendKind != ISD::ANY_EXTEND && VT.isInteger())
01273           VT = TLI.getTypeForExtArgOrReturn(Context, VT, ExtendKind);
01274 
01275         unsigned NumParts = TLI.getNumRegisters(Context, VT);
01276         MVT PartVT = TLI.getRegisterType(Context, VT);
01277         SmallVector<SDValue, 4> Parts(NumParts);
01278         getCopyToParts(DAG, getCurSDLoc(),
01279                        SDValue(RetOp.getNode(), RetOp.getResNo() + j),
01280                        &Parts[0], NumParts, PartVT, &I, ExtendKind);
01281 
01282         // 'inreg' on function refers to return value
01283         ISD::ArgFlagsTy Flags = ISD::ArgFlagsTy();
01284         if (RetInReg)
01285           Flags.setInReg();
01286 
01287         // Propagate extension type if any
01288         if (ExtendKind == ISD::SIGN_EXTEND)
01289           Flags.setSExt();
01290         else if (ExtendKind == ISD::ZERO_EXTEND)
01291           Flags.setZExt();
01292 
01293         for (unsigned i = 0; i < NumParts; ++i) {
01294           Outs.push_back(ISD::OutputArg(Flags, Parts[i].getValueType(),
01295                                         VT, /*isfixed=*/true, 0, 0));
01296           OutVals.push_back(Parts[i]);
01297         }
01298       }
01299     }
01300   }
01301 
01302   bool isVarArg = DAG.getMachineFunction().getFunction()->isVarArg();
01303   CallingConv::ID CallConv =
01304     DAG.getMachineFunction().getFunction()->getCallingConv();
01305   Chain = DAG.getTargetLoweringInfo().LowerReturn(
01306       Chain, CallConv, isVarArg, Outs, OutVals, getCurSDLoc(), DAG);
01307 
01308   // Verify that the target's LowerReturn behaved as expected.
01309   assert(Chain.getNode() && Chain.getValueType() == MVT::Other &&
01310          "LowerReturn didn't return a valid chain!");
01311 
01312   // Update the DAG with the new chain value resulting from return lowering.
01313   DAG.setRoot(Chain);
01314 }
01315 
01316 /// CopyToExportRegsIfNeeded - If the given value has virtual registers
01317 /// created for it, emit nodes to copy the value into the virtual
01318 /// registers.
01319 void SelectionDAGBuilder::CopyToExportRegsIfNeeded(const Value *V) {
01320   // Skip empty types
01321   if (V->getType()->isEmptyTy())
01322     return;
01323 
01324   DenseMap<const Value *, unsigned>::iterator VMI = FuncInfo.ValueMap.find(V);
01325   if (VMI != FuncInfo.ValueMap.end()) {
01326     assert(!V->use_empty() && "Unused value assigned virtual registers!");
01327     CopyValueToVirtualRegister(V, VMI->second);
01328   }
01329 }
01330 
01331 /// ExportFromCurrentBlock - If this condition isn't known to be exported from
01332 /// the current basic block, add it to ValueMap now so that we'll get a
01333 /// CopyTo/FromReg.
01334 void SelectionDAGBuilder::ExportFromCurrentBlock(const Value *V) {
01335   // No need to export constants.
01336   if (!isa<Instruction>(V) && !isa<Argument>(V)) return;
01337 
01338   // Already exported?
01339   if (FuncInfo.isExportedInst(V)) return;
01340 
01341   unsigned Reg = FuncInfo.InitializeRegForValue(V);
01342   CopyValueToVirtualRegister(V, Reg);
01343 }
01344 
01345 bool SelectionDAGBuilder::isExportableFromCurrentBlock(const Value *V,
01346                                                      const BasicBlock *FromBB) {
01347   // The operands of the setcc have to be in this block.  We don't know
01348   // how to export them from some other block.
01349   if (const Instruction *VI = dyn_cast<Instruction>(V)) {
01350     // Can export from current BB.
01351     if (VI->getParent() == FromBB)
01352       return true;
01353 
01354     // Is already exported, noop.
01355     return FuncInfo.isExportedInst(V);
01356   }
01357 
01358   // If this is an argument, we can export it if the BB is the entry block or
01359   // if it is already exported.
01360   if (isa<Argument>(V)) {
01361     if (FromBB == &FromBB->getParent()->getEntryBlock())
01362       return true;
01363 
01364     // Otherwise, can only export this if it is already exported.
01365     return FuncInfo.isExportedInst(V);
01366   }
01367 
01368   // Otherwise, constants can always be exported.
01369   return true;
01370 }
01371 
01372 /// Return branch probability calculated by BranchProbabilityInfo for IR blocks.
01373 uint32_t SelectionDAGBuilder::getEdgeWeight(const MachineBasicBlock *Src,
01374                                             const MachineBasicBlock *Dst) const {
01375   BranchProbabilityInfo *BPI = FuncInfo.BPI;
01376   if (!BPI)
01377     return 0;
01378   const BasicBlock *SrcBB = Src->getBasicBlock();
01379   const BasicBlock *DstBB = Dst->getBasicBlock();
01380   return BPI->getEdgeWeight(SrcBB, DstBB);
01381 }
01382 
01383 void SelectionDAGBuilder::
01384 addSuccessorWithWeight(MachineBasicBlock *Src, MachineBasicBlock *Dst,
01385                        uint32_t Weight /* = 0 */) {
01386   if (!Weight)
01387     Weight = getEdgeWeight(Src, Dst);
01388   Src->addSuccessor(Dst, Weight);
01389 }
01390 
01391 
01392 static bool InBlock(const Value *V, const BasicBlock *BB) {
01393   if (const Instruction *I = dyn_cast<Instruction>(V))
01394     return I->getParent() == BB;
01395   return true;
01396 }
01397 
01398 /// EmitBranchForMergedCondition - Helper method for FindMergedConditions.
01399 /// This function emits a branch and is used at the leaves of an OR or an
01400 /// AND operator tree.
01401 ///
01402 void
01403 SelectionDAGBuilder::EmitBranchForMergedCondition(const Value *Cond,
01404                                                   MachineBasicBlock *TBB,
01405                                                   MachineBasicBlock *FBB,
01406                                                   MachineBasicBlock *CurBB,
01407                                                   MachineBasicBlock *SwitchBB,
01408                                                   uint32_t TWeight,
01409                                                   uint32_t FWeight) {
01410   const BasicBlock *BB = CurBB->getBasicBlock();
01411 
01412   // If the leaf of the tree is a comparison, merge the condition into
01413   // the caseblock.
01414   if (const CmpInst *BOp = dyn_cast<CmpInst>(Cond)) {
01415     // The operands of the cmp have to be in this block.  We don't know
01416     // how to export them from some other block.  If this is the first block
01417     // of the sequence, no exporting is needed.
01418     if (CurBB == SwitchBB ||
01419         (isExportableFromCurrentBlock(BOp->getOperand(0), BB) &&
01420          isExportableFromCurrentBlock(BOp->getOperand(1), BB))) {
01421       ISD::CondCode Condition;
01422       if (const ICmpInst *IC = dyn_cast<ICmpInst>(Cond)) {
01423         Condition = getICmpCondCode(IC->getPredicate());
01424       } else if (const FCmpInst *FC = dyn_cast<FCmpInst>(Cond)) {
01425         Condition = getFCmpCondCode(FC->getPredicate());
01426         if (TM.Options.NoNaNsFPMath)
01427           Condition = getFCmpCodeWithoutNaN(Condition);
01428       } else {
01429         (void)Condition; // silence warning.
01430         llvm_unreachable("Unknown compare instruction");
01431       }
01432 
01433       CaseBlock CB(Condition, BOp->getOperand(0), BOp->getOperand(1), nullptr,
01434                    TBB, FBB, CurBB, TWeight, FWeight);
01435       SwitchCases.push_back(CB);
01436       return;
01437     }
01438   }
01439 
01440   // Create a CaseBlock record representing this branch.
01441   CaseBlock CB(ISD::SETEQ, Cond, ConstantInt::getTrue(*DAG.getContext()),
01442                nullptr, TBB, FBB, CurBB, TWeight, FWeight);
01443   SwitchCases.push_back(CB);
01444 }
01445 
01446 /// Scale down both weights to fit into uint32_t.
01447 static void ScaleWeights(uint64_t &NewTrue, uint64_t &NewFalse) {
01448   uint64_t NewMax = (NewTrue > NewFalse) ? NewTrue : NewFalse;
01449   uint32_t Scale = (NewMax / UINT32_MAX) + 1;
01450   NewTrue = NewTrue / Scale;
01451   NewFalse = NewFalse / Scale;
01452 }
01453 
01454 /// FindMergedConditions - If Cond is an expression like
01455 void SelectionDAGBuilder::FindMergedConditions(const Value *Cond,
01456                                                MachineBasicBlock *TBB,
01457                                                MachineBasicBlock *FBB,
01458                                                MachineBasicBlock *CurBB,
01459                                                MachineBasicBlock *SwitchBB,
01460                                                unsigned Opc, uint32_t TWeight,
01461                                                uint32_t FWeight) {
01462   // If this node is not part of the or/and tree, emit it as a branch.
01463   const Instruction *BOp = dyn_cast<Instruction>(Cond);
01464   if (!BOp || !(isa<BinaryOperator>(BOp) || isa<CmpInst>(BOp)) ||
01465       (unsigned)BOp->getOpcode() != Opc || !BOp->hasOneUse() ||
01466       BOp->getParent() != CurBB->getBasicBlock() ||
01467       !InBlock(BOp->getOperand(0), CurBB->getBasicBlock()) ||
01468       !InBlock(BOp->getOperand(1), CurBB->getBasicBlock())) {
01469     EmitBranchForMergedCondition(Cond, TBB, FBB, CurBB, SwitchBB,
01470                                  TWeight, FWeight);
01471     return;
01472   }
01473 
01474   //  Create TmpBB after CurBB.
01475   MachineFunction::iterator BBI = CurBB;
01476   MachineFunction &MF = DAG.getMachineFunction();
01477   MachineBasicBlock *TmpBB = MF.CreateMachineBasicBlock(CurBB->getBasicBlock());
01478   CurBB->getParent()->insert(++BBI, TmpBB);
01479 
01480   if (Opc == Instruction::Or) {
01481     // Codegen X | Y as:
01482     // BB1:
01483     //   jmp_if_X TBB
01484     //   jmp TmpBB
01485     // TmpBB:
01486     //   jmp_if_Y TBB
01487     //   jmp FBB
01488     //
01489 
01490     // We have flexibility in setting Prob for BB1 and Prob for TmpBB.
01491     // The requirement is that
01492     //   TrueProb for BB1 + (FalseProb for BB1 * TrueProb for TmpBB)
01493     //     = TrueProb for orignal BB.
01494     // Assuming the orignal weights are A and B, one choice is to set BB1's
01495     // weights to A and A+2B, and set TmpBB's weights to A and 2B. This choice
01496     // assumes that
01497     //   TrueProb for BB1 == FalseProb for BB1 * TrueProb for TmpBB.
01498     // Another choice is to assume TrueProb for BB1 equals to TrueProb for
01499     // TmpBB, but the math is more complicated.
01500 
01501     uint64_t NewTrueWeight = TWeight;
01502     uint64_t NewFalseWeight = (uint64_t)TWeight + 2 * (uint64_t)FWeight;
01503     ScaleWeights(NewTrueWeight, NewFalseWeight);
01504     // Emit the LHS condition.
01505     FindMergedConditions(BOp->getOperand(0), TBB, TmpBB, CurBB, SwitchBB, Opc,
01506                          NewTrueWeight, NewFalseWeight);
01507 
01508     NewTrueWeight = TWeight;
01509     NewFalseWeight = 2 * (uint64_t)FWeight;
01510     ScaleWeights(NewTrueWeight, NewFalseWeight);
01511     // Emit the RHS condition into TmpBB.
01512     FindMergedConditions(BOp->getOperand(1), TBB, FBB, TmpBB, SwitchBB, Opc,
01513                          NewTrueWeight, NewFalseWeight);
01514   } else {
01515     assert(Opc == Instruction::And && "Unknown merge op!");
01516     // Codegen X & Y as:
01517     // BB1:
01518     //   jmp_if_X TmpBB
01519     //   jmp FBB
01520     // TmpBB:
01521     //   jmp_if_Y TBB
01522     //   jmp FBB
01523     //
01524     //  This requires creation of TmpBB after CurBB.
01525 
01526     // We have flexibility in setting Prob for BB1 and Prob for TmpBB.
01527     // The requirement is that
01528     //   FalseProb for BB1 + (TrueProb for BB1 * FalseProb for TmpBB)
01529     //     = FalseProb for orignal BB.
01530     // Assuming the orignal weights are A and B, one choice is to set BB1's
01531     // weights to 2A+B and B, and set TmpBB's weights to 2A and B. This choice
01532     // assumes that
01533     //   FalseProb for BB1 == TrueProb for BB1 * FalseProb for TmpBB.
01534 
01535     uint64_t NewTrueWeight = 2 * (uint64_t)TWeight + (uint64_t)FWeight;
01536     uint64_t NewFalseWeight = FWeight;
01537     ScaleWeights(NewTrueWeight, NewFalseWeight);
01538     // Emit the LHS condition.
01539     FindMergedConditions(BOp->getOperand(0), TmpBB, FBB, CurBB, SwitchBB, Opc,
01540                          NewTrueWeight, NewFalseWeight);
01541 
01542     NewTrueWeight = 2 * (uint64_t)TWeight;
01543     NewFalseWeight = FWeight;
01544     ScaleWeights(NewTrueWeight, NewFalseWeight);
01545     // Emit the RHS condition into TmpBB.
01546     FindMergedConditions(BOp->getOperand(1), TBB, FBB, TmpBB, SwitchBB, Opc,
01547                          NewTrueWeight, NewFalseWeight);
01548   }
01549 }
01550 
01551 /// If the set of cases should be emitted as a series of branches, return true.
01552 /// If we should emit this as a bunch of and/or'd together conditions, return
01553 /// false.
01554 bool
01555 SelectionDAGBuilder::ShouldEmitAsBranches(const std::vector<CaseBlock> &Cases) {
01556   if (Cases.size() != 2) return true;
01557 
01558   // If this is two comparisons of the same values or'd or and'd together, they
01559   // will get folded into a single comparison, so don't emit two blocks.
01560   if ((Cases[0].CmpLHS == Cases[1].CmpLHS &&
01561        Cases[0].CmpRHS == Cases[1].CmpRHS) ||
01562       (Cases[0].CmpRHS == Cases[1].CmpLHS &&
01563        Cases[0].CmpLHS == Cases[1].CmpRHS)) {
01564     return false;
01565   }
01566 
01567   // Handle: (X != null) | (Y != null) --> (X|Y) != 0
01568   // Handle: (X == null) & (Y == null) --> (X|Y) == 0
01569   if (Cases[0].CmpRHS == Cases[1].CmpRHS &&
01570       Cases[0].CC == Cases[1].CC &&
01571       isa<Constant>(Cases[0].CmpRHS) &&
01572       cast<Constant>(Cases[0].CmpRHS)->isNullValue()) {
01573     if (Cases[0].CC == ISD::SETEQ && Cases[0].TrueBB == Cases[1].ThisBB)
01574       return false;
01575     if (Cases[0].CC == ISD::SETNE && Cases[0].FalseBB == Cases[1].ThisBB)
01576       return false;
01577   }
01578 
01579   return true;
01580 }
01581 
01582 void SelectionDAGBuilder::visitBr(const BranchInst &I) {
01583   MachineBasicBlock *BrMBB = FuncInfo.MBB;
01584 
01585   // Update machine-CFG edges.
01586   MachineBasicBlock *Succ0MBB = FuncInfo.MBBMap[I.getSuccessor(0)];
01587 
01588   if (I.isUnconditional()) {
01589     // Update machine-CFG edges.
01590     BrMBB->addSuccessor(Succ0MBB);
01591 
01592     // If this is not a fall-through branch or optimizations are switched off,
01593     // emit the branch.
01594     if (Succ0MBB != NextBlock(BrMBB) || TM.getOptLevel() == CodeGenOpt::None)
01595       DAG.setRoot(DAG.getNode(ISD::BR, getCurSDLoc(),
01596                               MVT::Other, getControlRoot(),
01597                               DAG.getBasicBlock(Succ0MBB)));
01598 
01599     return;
01600   }
01601 
01602   // If this condition is one of the special cases we handle, do special stuff
01603   // now.
01604   const Value *CondVal = I.getCondition();
01605   MachineBasicBlock *Succ1MBB = FuncInfo.MBBMap[I.getSuccessor(1)];
01606 
01607   // If this is a series of conditions that are or'd or and'd together, emit
01608   // this as a sequence of branches instead of setcc's with and/or operations.
01609   // As long as jumps are not expensive, this should improve performance.
01610   // For example, instead of something like:
01611   //     cmp A, B
01612   //     C = seteq
01613   //     cmp D, E
01614   //     F = setle
01615   //     or C, F
01616   //     jnz foo
01617   // Emit:
01618   //     cmp A, B
01619   //     je foo
01620   //     cmp D, E
01621   //     jle foo
01622   //
01623   if (const BinaryOperator *BOp = dyn_cast<BinaryOperator>(CondVal)) {
01624     if (!DAG.getTargetLoweringInfo().isJumpExpensive() &&
01625         BOp->hasOneUse() && (BOp->getOpcode() == Instruction::And ||
01626                              BOp->getOpcode() == Instruction::Or)) {
01627       FindMergedConditions(BOp, Succ0MBB, Succ1MBB, BrMBB, BrMBB,
01628                            BOp->getOpcode(), getEdgeWeight(BrMBB, Succ0MBB),
01629                            getEdgeWeight(BrMBB, Succ1MBB));
01630       // If the compares in later blocks need to use values not currently
01631       // exported from this block, export them now.  This block should always
01632       // be the first entry.
01633       assert(SwitchCases[0].ThisBB == BrMBB && "Unexpected lowering!");
01634 
01635       // Allow some cases to be rejected.
01636       if (ShouldEmitAsBranches(SwitchCases)) {
01637         for (unsigned i = 1, e = SwitchCases.size(); i != e; ++i) {
01638           ExportFromCurrentBlock(SwitchCases[i].CmpLHS);
01639           ExportFromCurrentBlock(SwitchCases[i].CmpRHS);
01640         }
01641 
01642         // Emit the branch for this block.
01643         visitSwitchCase(SwitchCases[0], BrMBB);
01644         SwitchCases.erase(SwitchCases.begin());
01645         return;
01646       }
01647 
01648       // Okay, we decided not to do this, remove any inserted MBB's and clear
01649       // SwitchCases.
01650       for (unsigned i = 1, e = SwitchCases.size(); i != e; ++i)
01651         FuncInfo.MF->erase(SwitchCases[i].ThisBB);
01652 
01653       SwitchCases.clear();
01654     }
01655   }
01656 
01657   // Create a CaseBlock record representing this branch.
01658   CaseBlock CB(ISD::SETEQ, CondVal, ConstantInt::getTrue(*DAG.getContext()),
01659                nullptr, Succ0MBB, Succ1MBB, BrMBB);
01660 
01661   // Use visitSwitchCase to actually insert the fast branch sequence for this
01662   // cond branch.
01663   visitSwitchCase(CB, BrMBB);
01664 }
01665 
01666 /// visitSwitchCase - Emits the necessary code to represent a single node in
01667 /// the binary search tree resulting from lowering a switch instruction.
01668 void SelectionDAGBuilder::visitSwitchCase(CaseBlock &CB,
01669                                           MachineBasicBlock *SwitchBB) {
01670   SDValue Cond;
01671   SDValue CondLHS = getValue(CB.CmpLHS);
01672   SDLoc dl = getCurSDLoc();
01673 
01674   // Build the setcc now.
01675   if (!CB.CmpMHS) {
01676     // Fold "(X == true)" to X and "(X == false)" to !X to
01677     // handle common cases produced by branch lowering.
01678     if (CB.CmpRHS == ConstantInt::getTrue(*DAG.getContext()) &&
01679         CB.CC == ISD::SETEQ)
01680       Cond = CondLHS;
01681     else if (CB.CmpRHS == ConstantInt::getFalse(*DAG.getContext()) &&
01682              CB.CC == ISD::SETEQ) {
01683       SDValue True = DAG.getConstant(1, CondLHS.getValueType());
01684       Cond = DAG.getNode(ISD::XOR, dl, CondLHS.getValueType(), CondLHS, True);
01685     } else
01686       Cond = DAG.getSetCC(dl, MVT::i1, CondLHS, getValue(CB.CmpRHS), CB.CC);
01687   } else {
01688     assert(CB.CC == ISD::SETLE && "Can handle only LE ranges now");
01689 
01690     const APInt& Low = cast<ConstantInt>(CB.CmpLHS)->getValue();
01691     const APInt& High = cast<ConstantInt>(CB.CmpRHS)->getValue();
01692 
01693     SDValue CmpOp = getValue(CB.CmpMHS);
01694     EVT VT = CmpOp.getValueType();
01695 
01696     if (cast<ConstantInt>(CB.CmpLHS)->isMinValue(true)) {
01697       Cond = DAG.getSetCC(dl, MVT::i1, CmpOp, DAG.getConstant(High, VT),
01698                           ISD::SETLE);
01699     } else {
01700       SDValue SUB = DAG.getNode(ISD::SUB, dl,
01701                                 VT, CmpOp, DAG.getConstant(Low, VT));
01702       Cond = DAG.getSetCC(dl, MVT::i1, SUB,
01703                           DAG.getConstant(High-Low, VT), ISD::SETULE);
01704     }
01705   }
01706 
01707   // Update successor info
01708   addSuccessorWithWeight(SwitchBB, CB.TrueBB, CB.TrueWeight);
01709   // TrueBB and FalseBB are always different unless the incoming IR is
01710   // degenerate. This only happens when running llc on weird IR.
01711   if (CB.TrueBB != CB.FalseBB)
01712     addSuccessorWithWeight(SwitchBB, CB.FalseBB, CB.FalseWeight);
01713 
01714   // If the lhs block is the next block, invert the condition so that we can
01715   // fall through to the lhs instead of the rhs block.
01716   if (CB.TrueBB == NextBlock(SwitchBB)) {
01717     std::swap(CB.TrueBB, CB.FalseBB);
01718     SDValue True = DAG.getConstant(1, Cond.getValueType());
01719     Cond = DAG.getNode(ISD::XOR, dl, Cond.getValueType(), Cond, True);
01720   }
01721 
01722   SDValue BrCond = DAG.getNode(ISD::BRCOND, dl,
01723                                MVT::Other, getControlRoot(), Cond,
01724                                DAG.getBasicBlock(CB.TrueBB));
01725 
01726   // Insert the false branch. Do this even if it's a fall through branch,
01727   // this makes it easier to do DAG optimizations which require inverting
01728   // the branch condition.
01729   BrCond = DAG.getNode(ISD::BR, dl, MVT::Other, BrCond,
01730                        DAG.getBasicBlock(CB.FalseBB));
01731 
01732   DAG.setRoot(BrCond);
01733 }
01734 
01735 /// visitJumpTable - Emit JumpTable node in the current MBB
01736 void SelectionDAGBuilder::visitJumpTable(JumpTable &JT) {
01737   // Emit the code for the jump table
01738   assert(JT.Reg != -1U && "Should lower JT Header first!");
01739   EVT PTy = DAG.getTargetLoweringInfo().getPointerTy();
01740   SDValue Index = DAG.getCopyFromReg(getControlRoot(), getCurSDLoc(),
01741                                      JT.Reg, PTy);
01742   SDValue Table = DAG.getJumpTable(JT.JTI, PTy);
01743   SDValue BrJumpTable = DAG.getNode(ISD::BR_JT, getCurSDLoc(),
01744                                     MVT::Other, Index.getValue(1),
01745                                     Table, Index);
01746   DAG.setRoot(BrJumpTable);
01747 }
01748 
01749 /// visitJumpTableHeader - This function emits necessary code to produce index
01750 /// in the JumpTable from switch case.
01751 void SelectionDAGBuilder::visitJumpTableHeader(JumpTable &JT,
01752                                                JumpTableHeader &JTH,
01753                                                MachineBasicBlock *SwitchBB) {
01754   // Subtract the lowest switch case value from the value being switched on and
01755   // conditional branch to default mbb if the result is greater than the
01756   // difference between smallest and largest cases.
01757   SDValue SwitchOp = getValue(JTH.SValue);
01758   EVT VT = SwitchOp.getValueType();
01759   SDValue Sub = DAG.getNode(ISD::SUB, getCurSDLoc(), VT, SwitchOp,
01760                             DAG.getConstant(JTH.First, VT));
01761 
01762   // The SDNode we just created, which holds the value being switched on minus
01763   // the smallest case value, needs to be copied to a virtual register so it
01764   // can be used as an index into the jump table in a subsequent basic block.
01765   // This value may be smaller or larger than the target's pointer type, and
01766   // therefore require extension or truncating.
01767   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
01768   SwitchOp = DAG.getZExtOrTrunc(Sub, getCurSDLoc(), TLI.getPointerTy());
01769 
01770   unsigned JumpTableReg = FuncInfo.CreateReg(TLI.getPointerTy());
01771   SDValue CopyTo = DAG.getCopyToReg(getControlRoot(), getCurSDLoc(),
01772                                     JumpTableReg, SwitchOp);
01773   JT.Reg = JumpTableReg;
01774 
01775   // Emit the range check for the jump table, and branch to the default block
01776   // for the switch statement if the value being switched on exceeds the largest
01777   // case in the switch.
01778   SDValue CMP =
01779       DAG.getSetCC(getCurSDLoc(), TLI.getSetCCResultType(*DAG.getContext(),
01780                                                          Sub.getValueType()),
01781                    Sub, DAG.getConstant(JTH.Last - JTH.First, VT), ISD::SETUGT);
01782 
01783   SDValue BrCond = DAG.getNode(ISD::BRCOND, getCurSDLoc(),
01784                                MVT::Other, CopyTo, CMP,
01785                                DAG.getBasicBlock(JT.Default));
01786 
01787   // Avoid emitting unnecessary branches to the next block.
01788   if (JT.MBB != NextBlock(SwitchBB))
01789     BrCond = DAG.getNode(ISD::BR, getCurSDLoc(), MVT::Other, BrCond,
01790                          DAG.getBasicBlock(JT.MBB));
01791 
01792   DAG.setRoot(BrCond);
01793 }
01794 
01795 /// Codegen a new tail for a stack protector check ParentMBB which has had its
01796 /// tail spliced into a stack protector check success bb.
01797 ///
01798 /// For a high level explanation of how this fits into the stack protector
01799 /// generation see the comment on the declaration of class
01800 /// StackProtectorDescriptor.
01801 void SelectionDAGBuilder::visitSPDescriptorParent(StackProtectorDescriptor &SPD,
01802                                                   MachineBasicBlock *ParentBB) {
01803 
01804   // First create the loads to the guard/stack slot for the comparison.
01805   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
01806   EVT PtrTy = TLI.getPointerTy();
01807 
01808   MachineFrameInfo *MFI = ParentBB->getParent()->getFrameInfo();
01809   int FI = MFI->getStackProtectorIndex();
01810 
01811   const Value *IRGuard = SPD.getGuard();
01812   SDValue GuardPtr = getValue(IRGuard);
01813   SDValue StackSlotPtr = DAG.getFrameIndex(FI, PtrTy);
01814 
01815   unsigned Align =
01816     TLI.getDataLayout()->getPrefTypeAlignment(IRGuard->getType());
01817 
01818   SDValue Guard;
01819 
01820   // If GuardReg is set and useLoadStackGuardNode returns true, retrieve the
01821   // guard value from the virtual register holding the value. Otherwise, emit a
01822   // volatile load to retrieve the stack guard value.
01823   unsigned GuardReg = SPD.getGuardReg();
01824 
01825   if (GuardReg && TLI.useLoadStackGuardNode())
01826     Guard = DAG.getCopyFromReg(DAG.getEntryNode(), getCurSDLoc(), GuardReg,
01827                                PtrTy);
01828   else
01829     Guard = DAG.getLoad(PtrTy, getCurSDLoc(), DAG.getEntryNode(),
01830                         GuardPtr, MachinePointerInfo(IRGuard, 0),
01831                         true, false, false, Align);
01832 
01833   SDValue StackSlot = DAG.getLoad(PtrTy, getCurSDLoc(), DAG.getEntryNode(),
01834                                   StackSlotPtr,
01835                                   MachinePointerInfo::getFixedStack(FI),
01836                                   true, false, false, Align);
01837 
01838   // Perform the comparison via a subtract/getsetcc.
01839   EVT VT = Guard.getValueType();
01840   SDValue Sub = DAG.getNode(ISD::SUB, getCurSDLoc(), VT, Guard, StackSlot);
01841 
01842   SDValue Cmp =
01843       DAG.getSetCC(getCurSDLoc(), TLI.getSetCCResultType(*DAG.getContext(),
01844                                                          Sub.getValueType()),
01845                    Sub, DAG.getConstant(0, VT), ISD::SETNE);
01846 
01847   // If the sub is not 0, then we know the guard/stackslot do not equal, so
01848   // branch to failure MBB.
01849   SDValue BrCond = DAG.getNode(ISD::BRCOND, getCurSDLoc(),
01850                                MVT::Other, StackSlot.getOperand(0),
01851                                Cmp, DAG.getBasicBlock(SPD.getFailureMBB()));
01852   // Otherwise branch to success MBB.
01853   SDValue Br = DAG.getNode(ISD::BR, getCurSDLoc(),
01854                            MVT::Other, BrCond,
01855                            DAG.getBasicBlock(SPD.getSuccessMBB()));
01856 
01857   DAG.setRoot(Br);
01858 }
01859 
01860 /// Codegen the failure basic block for a stack protector check.
01861 ///
01862 /// A failure stack protector machine basic block consists simply of a call to
01863 /// __stack_chk_fail().
01864 ///
01865 /// For a high level explanation of how this fits into the stack protector
01866 /// generation see the comment on the declaration of class
01867 /// StackProtectorDescriptor.
01868 void
01869 SelectionDAGBuilder::visitSPDescriptorFailure(StackProtectorDescriptor &SPD) {
01870   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
01871   SDValue Chain =
01872       TLI.makeLibCall(DAG, RTLIB::STACKPROTECTOR_CHECK_FAIL, MVT::isVoid,
01873                       nullptr, 0, false, getCurSDLoc(), false, false).second;
01874   DAG.setRoot(Chain);
01875 }
01876 
01877 /// visitBitTestHeader - This function emits necessary code to produce value
01878 /// suitable for "bit tests"
01879 void SelectionDAGBuilder::visitBitTestHeader(BitTestBlock &B,
01880                                              MachineBasicBlock *SwitchBB) {
01881   // Subtract the minimum value
01882   SDValue SwitchOp = getValue(B.SValue);
01883   EVT VT = SwitchOp.getValueType();
01884   SDValue Sub = DAG.getNode(ISD::SUB, getCurSDLoc(), VT, SwitchOp,
01885                             DAG.getConstant(B.First, VT));
01886 
01887   // Check range
01888   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
01889   SDValue RangeCmp =
01890       DAG.getSetCC(getCurSDLoc(), TLI.getSetCCResultType(*DAG.getContext(),
01891                                                          Sub.getValueType()),
01892                    Sub, DAG.getConstant(B.Range, VT), ISD::SETUGT);
01893 
01894   // Determine the type of the test operands.
01895   bool UsePtrType = false;
01896   if (!TLI.isTypeLegal(VT))
01897     UsePtrType = true;
01898   else {
01899     for (unsigned i = 0, e = B.Cases.size(); i != e; ++i)
01900       if (!isUIntN(VT.getSizeInBits(), B.Cases[i].Mask)) {
01901         // Switch table case range are encoded into series of masks.
01902         // Just use pointer type, it's guaranteed to fit.
01903         UsePtrType = true;
01904         break;
01905       }
01906   }
01907   if (UsePtrType) {
01908     VT = TLI.getPointerTy();
01909     Sub = DAG.getZExtOrTrunc(Sub, getCurSDLoc(), VT);
01910   }
01911 
01912   B.RegVT = VT.getSimpleVT();
01913   B.Reg = FuncInfo.CreateReg(B.RegVT);
01914   SDValue CopyTo = DAG.getCopyToReg(getControlRoot(), getCurSDLoc(),
01915                                     B.Reg, Sub);
01916 
01917   MachineBasicBlock* MBB = B.Cases[0].ThisBB;
01918 
01919   addSuccessorWithWeight(SwitchBB, B.Default);
01920   addSuccessorWithWeight(SwitchBB, MBB);
01921 
01922   SDValue BrRange = DAG.getNode(ISD::BRCOND, getCurSDLoc(),
01923                                 MVT::Other, CopyTo, RangeCmp,
01924                                 DAG.getBasicBlock(B.Default));
01925 
01926   // Avoid emitting unnecessary branches to the next block.
01927   if (MBB != NextBlock(SwitchBB))
01928     BrRange = DAG.getNode(ISD::BR, getCurSDLoc(), MVT::Other, CopyTo,
01929                           DAG.getBasicBlock(MBB));
01930 
01931   DAG.setRoot(BrRange);
01932 }
01933 
01934 /// visitBitTestCase - this function produces one "bit test"
01935 void SelectionDAGBuilder::visitBitTestCase(BitTestBlock &BB,
01936                                            MachineBasicBlock* NextMBB,
01937                                            uint32_t BranchWeightToNext,
01938                                            unsigned Reg,
01939                                            BitTestCase &B,
01940                                            MachineBasicBlock *SwitchBB) {
01941   MVT VT = BB.RegVT;
01942   SDValue ShiftOp = DAG.getCopyFromReg(getControlRoot(), getCurSDLoc(),
01943                                        Reg, VT);
01944   SDValue Cmp;
01945   unsigned PopCount = countPopulation(B.Mask);
01946   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
01947   if (PopCount == 1) {
01948     // Testing for a single bit; just compare the shift count with what it
01949     // would need to be to shift a 1 bit in that position.
01950     Cmp = DAG.getSetCC(
01951         getCurSDLoc(), TLI.getSetCCResultType(*DAG.getContext(), VT), ShiftOp,
01952         DAG.getConstant(countTrailingZeros(B.Mask), VT), ISD::SETEQ);
01953   } else if (PopCount == BB.Range) {
01954     // There is only one zero bit in the range, test for it directly.
01955     Cmp = DAG.getSetCC(
01956         getCurSDLoc(), TLI.getSetCCResultType(*DAG.getContext(), VT), ShiftOp,
01957         DAG.getConstant(countTrailingOnes(B.Mask), VT), ISD::SETNE);
01958   } else {
01959     // Make desired shift
01960     SDValue SwitchVal = DAG.getNode(ISD::SHL, getCurSDLoc(), VT,
01961                                     DAG.getConstant(1, VT), ShiftOp);
01962 
01963     // Emit bit tests and jumps
01964     SDValue AndOp = DAG.getNode(ISD::AND, getCurSDLoc(),
01965                                 VT, SwitchVal, DAG.getConstant(B.Mask, VT));
01966     Cmp = DAG.getSetCC(getCurSDLoc(),
01967                        TLI.getSetCCResultType(*DAG.getContext(), VT), AndOp,
01968                        DAG.getConstant(0, VT), ISD::SETNE);
01969   }
01970 
01971   // The branch weight from SwitchBB to B.TargetBB is B.ExtraWeight.
01972   addSuccessorWithWeight(SwitchBB, B.TargetBB, B.ExtraWeight);
01973   // The branch weight from SwitchBB to NextMBB is BranchWeightToNext.
01974   addSuccessorWithWeight(SwitchBB, NextMBB, BranchWeightToNext);
01975 
01976   SDValue BrAnd = DAG.getNode(ISD::BRCOND, getCurSDLoc(),
01977                               MVT::Other, getControlRoot(),
01978                               Cmp, DAG.getBasicBlock(B.TargetBB));
01979 
01980   // Avoid emitting unnecessary branches to the next block.
01981   if (NextMBB != NextBlock(SwitchBB))
01982     BrAnd = DAG.getNode(ISD::BR, getCurSDLoc(), MVT::Other, BrAnd,
01983                         DAG.getBasicBlock(NextMBB));
01984 
01985   DAG.setRoot(BrAnd);
01986 }
01987 
01988 void SelectionDAGBuilder::visitInvoke(const InvokeInst &I) {
01989   MachineBasicBlock *InvokeMBB = FuncInfo.MBB;
01990 
01991   // Retrieve successors.
01992   MachineBasicBlock *Return = FuncInfo.MBBMap[I.getSuccessor(0)];
01993   MachineBasicBlock *LandingPad = FuncInfo.MBBMap[I.getSuccessor(1)];
01994 
01995   const Value *Callee(I.getCalledValue());
01996   const Function *Fn = dyn_cast<Function>(Callee);
01997   if (isa<InlineAsm>(Callee))
01998     visitInlineAsm(&I);
01999   else if (Fn && Fn->isIntrinsic()) {
02000     switch (Fn->getIntrinsicID()) {
02001     default:
02002       llvm_unreachable("Cannot invoke this intrinsic");
02003     case Intrinsic::donothing:
02004       // Ignore invokes to @llvm.donothing: jump directly to the next BB.
02005       break;
02006     case Intrinsic::experimental_patchpoint_void:
02007     case Intrinsic::experimental_patchpoint_i64:
02008       visitPatchpoint(&I, LandingPad);
02009       break;
02010     case Intrinsic::experimental_gc_statepoint:
02011       LowerStatepoint(ImmutableStatepoint(&I), LandingPad);
02012       break;
02013     }
02014   } else
02015     LowerCallTo(&I, getValue(Callee), false, LandingPad);
02016 
02017   // If the value of the invoke is used outside of its defining block, make it
02018   // available as a virtual register.
02019   // We already took care of the exported value for the statepoint instruction
02020   // during call to the LowerStatepoint.
02021   if (!isStatepoint(I)) {
02022     CopyToExportRegsIfNeeded(&I);
02023   }
02024 
02025   // Update successor info
02026   addSuccessorWithWeight(InvokeMBB, Return);
02027   addSuccessorWithWeight(InvokeMBB, LandingPad);
02028 
02029   // Drop into normal successor.
02030   DAG.setRoot(DAG.getNode(ISD::BR, getCurSDLoc(),
02031                           MVT::Other, getControlRoot(),
02032                           DAG.getBasicBlock(Return)));
02033 }
02034 
02035 void SelectionDAGBuilder::visitResume(const ResumeInst &RI) {
02036   llvm_unreachable("SelectionDAGBuilder shouldn't visit resume instructions!");
02037 }
02038 
02039 void SelectionDAGBuilder::visitLandingPad(const LandingPadInst &LP) {
02040   assert(FuncInfo.MBB->isLandingPad() &&
02041          "Call to landingpad not in landing pad!");
02042 
02043   MachineBasicBlock *MBB = FuncInfo.MBB;
02044   MachineModuleInfo &MMI = DAG.getMachineFunction().getMMI();
02045   AddLandingPadInfo(LP, MMI, MBB);
02046 
02047   // If there aren't registers to copy the values into (e.g., during SjLj
02048   // exceptions), then don't bother to create these DAG nodes.
02049   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
02050   if (TLI.getExceptionPointerRegister() == 0 &&
02051       TLI.getExceptionSelectorRegister() == 0)
02052     return;
02053 
02054   SmallVector<EVT, 2> ValueVTs;
02055   ComputeValueVTs(TLI, LP.getType(), ValueVTs);
02056   assert(ValueVTs.size() == 2 && "Only two-valued landingpads are supported");
02057 
02058   // Get the two live-in registers as SDValues. The physregs have already been
02059   // copied into virtual registers.
02060   SDValue Ops[2];
02061   if (FuncInfo.ExceptionPointerVirtReg) {
02062     Ops[0] = DAG.getZExtOrTrunc(
02063         DAG.getCopyFromReg(DAG.getEntryNode(), getCurSDLoc(),
02064                            FuncInfo.ExceptionPointerVirtReg, TLI.getPointerTy()),
02065         getCurSDLoc(), ValueVTs[0]);
02066   } else {
02067     Ops[0] = DAG.getConstant(0, TLI.getPointerTy());
02068   }
02069   Ops[1] = DAG.getZExtOrTrunc(
02070       DAG.getCopyFromReg(DAG.getEntryNode(), getCurSDLoc(),
02071                          FuncInfo.ExceptionSelectorVirtReg, TLI.getPointerTy()),
02072       getCurSDLoc(), ValueVTs[1]);
02073 
02074   // Merge into one.
02075   SDValue Res = DAG.getNode(ISD::MERGE_VALUES, getCurSDLoc(),
02076                             DAG.getVTList(ValueVTs), Ops);
02077   setValue(&LP, Res);
02078 }
02079 
02080 unsigned
02081 SelectionDAGBuilder::visitLandingPadClauseBB(GlobalValue *ClauseGV,
02082                                              MachineBasicBlock *LPadBB) {
02083   SDValue Chain = getControlRoot();
02084 
02085   // Get the typeid that we will dispatch on later.
02086   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
02087   const TargetRegisterClass *RC = TLI.getRegClassFor(TLI.getPointerTy());
02088   unsigned VReg = FuncInfo.MF->getRegInfo().createVirtualRegister(RC);
02089   unsigned TypeID = DAG.getMachineFunction().getMMI().getTypeIDFor(ClauseGV);
02090   SDValue Sel = DAG.getConstant(TypeID, TLI.getPointerTy());
02091   Chain = DAG.getCopyToReg(Chain, getCurSDLoc(), VReg, Sel);
02092 
02093   // Branch to the main landing pad block.
02094   MachineBasicBlock *ClauseMBB = FuncInfo.MBB;
02095   ClauseMBB->addSuccessor(LPadBB);
02096   DAG.setRoot(DAG.getNode(ISD::BR, getCurSDLoc(), MVT::Other, Chain,
02097                           DAG.getBasicBlock(LPadBB)));
02098   return VReg;
02099 }
02100 
02101 /// handleSmallSwitchCaseRange - Emit a series of specific tests (suitable for
02102 /// small case ranges).
02103 bool SelectionDAGBuilder::handleSmallSwitchRange(CaseRec& CR,
02104                                                  CaseRecVector& WorkList,
02105                                                  const Value* SV,
02106                                                  MachineBasicBlock *Default,
02107                                                  MachineBasicBlock *SwitchBB) {
02108   // Size is the number of Cases represented by this range.
02109   size_t Size = CR.Range.second - CR.Range.first;
02110   if (Size > 3)
02111     return false;
02112 
02113   // Get the MachineFunction which holds the current MBB.  This is used when
02114   // inserting any additional MBBs necessary to represent the switch.
02115   MachineFunction *CurMF = FuncInfo.MF;
02116 
02117   // Figure out which block is immediately after the current one.
02118   MachineBasicBlock *NextMBB = nullptr;
02119   MachineFunction::iterator BBI = CR.CaseBB;
02120   if (++BBI != FuncInfo.MF->end())
02121     NextMBB = BBI;
02122 
02123   BranchProbabilityInfo *BPI = FuncInfo.BPI;
02124   // If any two of the cases has the same destination, and if one value
02125   // is the same as the other, but has one bit unset that the other has set,
02126   // use bit manipulation to do two compares at once.  For example:
02127   // "if (X == 6 || X == 4)" -> "if ((X|2) == 6)"
02128   // TODO: This could be extended to merge any 2 cases in switches with 3 cases.
02129   // TODO: Handle cases where CR.CaseBB != SwitchBB.
02130   if (Size == 2 && CR.CaseBB == SwitchBB) {
02131     Case &Small = *CR.Range.first;
02132     Case &Big = *(CR.Range.second-1);
02133 
02134     if (Small.Low == Small.High && Big.Low == Big.High && Small.BB == Big.BB) {
02135       const APInt& SmallValue = Small.Low->getValue();
02136       const APInt& BigValue = Big.Low->getValue();
02137 
02138       // Check that there is only one bit different.
02139       if (BigValue.countPopulation() == SmallValue.countPopulation() + 1 &&
02140           (SmallValue | BigValue) == BigValue) {
02141         // Isolate the common bit.
02142         APInt CommonBit = BigValue & ~SmallValue;
02143         assert((SmallValue | CommonBit) == BigValue &&
02144                CommonBit.countPopulation() == 1 && "Not a common bit?");
02145 
02146         SDValue CondLHS = getValue(SV);
02147         EVT VT = CondLHS.getValueType();
02148         SDLoc DL = getCurSDLoc();
02149 
02150         SDValue Or = DAG.getNode(ISD::OR, DL, VT, CondLHS,
02151                                  DAG.getConstant(CommonBit, VT));
02152         SDValue Cond = DAG.getSetCC(DL, MVT::i1,
02153                                     Or, DAG.getConstant(BigValue, VT),
02154                                     ISD::SETEQ);
02155 
02156         // Update successor info.
02157         // Both Small and Big will jump to Small.BB, so we sum up the weights.
02158         addSuccessorWithWeight(SwitchBB, Small.BB,
02159                                Small.ExtraWeight + Big.ExtraWeight);
02160         addSuccessorWithWeight(SwitchBB, Default,
02161           // The default destination is the first successor in IR.
02162           BPI ? BPI->getEdgeWeight(SwitchBB->getBasicBlock(), (unsigned)0) : 0);
02163 
02164         // Insert the true branch.
02165         SDValue BrCond = DAG.getNode(ISD::BRCOND, DL, MVT::Other,
02166                                      getControlRoot(), Cond,
02167                                      DAG.getBasicBlock(Small.BB));
02168 
02169         // Insert the false branch.
02170         BrCond = DAG.getNode(ISD::BR, DL, MVT::Other, BrCond,
02171                              DAG.getBasicBlock(Default));
02172 
02173         DAG.setRoot(BrCond);
02174         return true;
02175       }
02176     }
02177   }
02178 
02179   // Order cases by weight so the most likely case will be checked first.
02180   uint32_t UnhandledWeights = 0;
02181   if (BPI) {
02182     for (CaseItr I = CR.Range.first, IE = CR.Range.second; I != IE; ++I) {
02183       uint32_t IWeight = I->ExtraWeight;
02184       UnhandledWeights += IWeight;
02185       for (CaseItr J = CR.Range.first; J < I; ++J) {
02186         uint32_t JWeight = J->ExtraWeight;
02187         if (IWeight > JWeight)
02188           std::swap(*I, *J);
02189       }
02190     }
02191   }
02192   // Rearrange the case blocks so that the last one falls through if possible.
02193   Case &BackCase = *(CR.Range.second-1);
02194   if (Size > 1 && NextMBB && Default != NextMBB && BackCase.BB != NextMBB) {
02195     // The last case block won't fall through into 'NextMBB' if we emit the
02196     // branches in this order.  See if rearranging a case value would help.
02197     // We start at the bottom as it's the case with the least weight.
02198     for (Case *I = &*(CR.Range.second-2), *E = &*CR.Range.first-1; I != E; --I)
02199       if (I->BB == NextMBB) {
02200         std::swap(*I, BackCase);
02201         break;
02202       }
02203   }
02204 
02205   // Create a CaseBlock record representing a conditional branch to
02206   // the Case's target mbb if the value being switched on SV is equal
02207   // to C.
02208   MachineBasicBlock *CurBlock = CR.CaseBB;
02209   for (CaseItr I = CR.Range.first, E = CR.Range.second; I != E; ++I) {
02210     MachineBasicBlock *FallThrough;
02211     if (I != E-1) {
02212       FallThrough = CurMF->CreateMachineBasicBlock(CurBlock->getBasicBlock());
02213       CurMF->insert(BBI, FallThrough);
02214 
02215       // Put SV in a virtual register to make it available from the new blocks.
02216       ExportFromCurrentBlock(SV);
02217     } else {
02218       // If the last case doesn't match, go to the default block.
02219       FallThrough = Default;
02220     }
02221 
02222     const Value *RHS, *LHS, *MHS;
02223     ISD::CondCode CC;
02224     if (I->High == I->Low) {
02225       // This is just small small case range :) containing exactly 1 case
02226       CC = ISD::SETEQ;
02227       LHS = SV; RHS = I->High; MHS = nullptr;
02228     } else {
02229       CC = ISD::SETLE;
02230       LHS = I->Low; MHS = SV; RHS = I->High;
02231     }
02232 
02233     // The false weight should be sum of all un-handled cases.
02234     UnhandledWeights -= I->ExtraWeight;
02235     CaseBlock CB(CC, LHS, RHS, MHS, /* truebb */ I->BB, /* falsebb */ FallThrough,
02236                  /* me */ CurBlock,
02237                  /* trueweight */ I->ExtraWeight,
02238                  /* falseweight */ UnhandledWeights);
02239 
02240     // If emitting the first comparison, just call visitSwitchCase to emit the
02241     // code into the current block.  Otherwise, push the CaseBlock onto the
02242     // vector to be later processed by SDISel, and insert the node's MBB
02243     // before the next MBB.
02244     if (CurBlock == SwitchBB)
02245       visitSwitchCase(CB, SwitchBB);
02246     else
02247       SwitchCases.push_back(CB);
02248 
02249     CurBlock = FallThrough;
02250   }
02251 
02252   return true;
02253 }
02254 
02255 static inline bool areJTsAllowed(const TargetLowering &TLI) {
02256   return TLI.isOperationLegalOrCustom(ISD::BR_JT, MVT::Other) ||
02257          TLI.isOperationLegalOrCustom(ISD::BRIND, MVT::Other);
02258 }
02259 
02260 static APInt ComputeRange(const APInt &First, const APInt &Last) {
02261   uint32_t BitWidth = std::max(Last.getBitWidth(), First.getBitWidth()) + 1;
02262   APInt LastExt = Last.sext(BitWidth), FirstExt = First.sext(BitWidth);
02263   return (LastExt - FirstExt + 1ULL);
02264 }
02265 
02266 /// handleJTSwitchCase - Emit jumptable for current switch case range
02267 bool SelectionDAGBuilder::handleJTSwitchCase(CaseRec &CR,
02268                                              CaseRecVector &WorkList,
02269                                              const Value *SV,
02270                                              MachineBasicBlock *Default,
02271                                              MachineBasicBlock *SwitchBB) {
02272   Case& FrontCase = *CR.Range.first;
02273   Case& BackCase  = *(CR.Range.second-1);
02274 
02275   const APInt &First = FrontCase.Low->getValue();
02276   const APInt &Last  = BackCase.High->getValue();
02277 
02278   APInt TSize(First.getBitWidth(), 0);
02279   for (CaseItr I = CR.Range.first, E = CR.Range.second; I != E; ++I)
02280     TSize += I->size();
02281 
02282   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
02283   if (!areJTsAllowed(TLI) || TSize.ult(TLI.getMinimumJumpTableEntries()))
02284     return false;
02285 
02286   APInt Range = ComputeRange(First, Last);
02287   // The density is TSize / Range. Require at least 40%.
02288   // It should not be possible for IntTSize to saturate for sane code, but make
02289   // sure we handle Range saturation correctly.
02290   uint64_t IntRange = Range.getLimitedValue(UINT64_MAX/10);
02291   uint64_t IntTSize = TSize.getLimitedValue(UINT64_MAX/10);
02292   if (IntTSize * 10 < IntRange * 4)
02293     return false;
02294 
02295   DEBUG(dbgs() << "Lowering jump table\n"
02296                << "First entry: " << First << ". Last entry: " << Last << '\n'
02297                << "Range: " << Range << ". Size: " << TSize << ".\n\n");
02298 
02299   // Get the MachineFunction which holds the current MBB.  This is used when
02300   // inserting any additional MBBs necessary to represent the switch.
02301   MachineFunction *CurMF = FuncInfo.MF;
02302 
02303   // Figure out which block is immediately after the current one.
02304   MachineFunction::iterator BBI = CR.CaseBB;
02305   ++BBI;
02306 
02307   const BasicBlock *LLVMBB = CR.CaseBB->getBasicBlock();
02308 
02309   // Create a new basic block to hold the code for loading the address
02310   // of the jump table, and jumping to it.  Update successor information;
02311   // we will either branch to the default case for the switch, or the jump
02312   // table.
02313   MachineBasicBlock *JumpTableBB = CurMF->CreateMachineBasicBlock(LLVMBB);
02314   CurMF->insert(BBI, JumpTableBB);
02315 
02316   addSuccessorWithWeight(CR.CaseBB, Default);
02317   addSuccessorWithWeight(CR.CaseBB, JumpTableBB);
02318 
02319   // Build a vector of destination BBs, corresponding to each target
02320   // of the jump table. If the value of the jump table slot corresponds to
02321   // a case statement, push the case's BB onto the vector, otherwise, push
02322   // the default BB.
02323   std::vector<MachineBasicBlock*> DestBBs;
02324   APInt TEI = First;
02325   for (CaseItr I = CR.Range.first, E = CR.Range.second; I != E; ++TEI) {
02326     const APInt &Low = I->Low->getValue();
02327     const APInt &High = I->High->getValue();
02328 
02329     if (Low.sle(TEI) && TEI.sle(High)) {
02330       DestBBs.push_back(I->BB);
02331       if (TEI==High)
02332         ++I;
02333     } else {
02334       DestBBs.push_back(Default);
02335     }
02336   }
02337 
02338   // Calculate weight for each unique destination in CR.
02339   DenseMap<MachineBasicBlock*, uint32_t> DestWeights;
02340   if (FuncInfo.BPI) {
02341     for (CaseItr I = CR.Range.first, E = CR.Range.second; I != E; ++I)
02342       DestWeights[I->BB] += I->ExtraWeight;
02343   }
02344 
02345   // Update successor info. Add one edge to each unique successor.
02346   BitVector SuccsHandled(CR.CaseBB->getParent()->getNumBlockIDs());
02347   for (MachineBasicBlock *DestBB : DestBBs) {
02348     if (!SuccsHandled[DestBB->getNumber()]) {
02349       SuccsHandled[DestBB->getNumber()] = true;
02350       auto I = DestWeights.find(DestBB);
02351       addSuccessorWithWeight(JumpTableBB, DestBB,
02352                              I != DestWeights.end() ? I->second : 0);
02353     }
02354   }
02355 
02356   // Create a jump table index for this jump table.
02357   unsigned JTEncoding = TLI.getJumpTableEncoding();
02358   unsigned JTI = CurMF->getOrCreateJumpTableInfo(JTEncoding)
02359                        ->createJumpTableIndex(DestBBs);
02360 
02361   // Set the jump table information so that we can codegen it as a second
02362   // MachineBasicBlock
02363   JumpTable JT(-1U, JTI, JumpTableBB, Default);
02364   JumpTableHeader JTH(First, Last, SV, CR.CaseBB, (CR.CaseBB == SwitchBB));
02365   if (CR.CaseBB == SwitchBB)
02366     visitJumpTableHeader(JT, JTH, SwitchBB);
02367 
02368   JTCases.push_back(JumpTableBlock(JTH, JT));
02369   return true;
02370 }
02371 
02372 /// handleBTSplitSwitchCase - emit comparison and split binary search tree into
02373 /// 2 subtrees.
02374 bool SelectionDAGBuilder::handleBTSplitSwitchCase(CaseRec& CR,
02375                                                   CaseRecVector& WorkList,
02376                                                   const Value* SV,
02377                                                   MachineBasicBlock* SwitchBB) {
02378   Case& FrontCase = *CR.Range.first;
02379   Case& BackCase  = *(CR.Range.second-1);
02380 
02381   // Size is the number of Cases represented by this range.
02382   unsigned Size = CR.Range.second - CR.Range.first;
02383 
02384   const APInt &First = FrontCase.Low->getValue();
02385   const APInt &Last  = BackCase.High->getValue();
02386   double FMetric = 0;
02387   CaseItr Pivot = CR.Range.first + Size/2;
02388 
02389   // Select optimal pivot, maximizing sum density of LHS and RHS. This will
02390   // (heuristically) allow us to emit JumpTable's later.
02391   APInt TSize(First.getBitWidth(), 0);
02392   for (CaseItr I = CR.Range.first, E = CR.Range.second;
02393        I!=E; ++I)
02394     TSize += I->size();
02395 
02396   APInt LSize = FrontCase.size();
02397   APInt RSize = TSize-LSize;
02398   DEBUG(dbgs() << "Selecting best pivot: \n"
02399                << "First: " << First << ", Last: " << Last <<'\n'
02400                << "LSize: " << LSize << ", RSize: " << RSize << '\n');
02401   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
02402   for (CaseItr I = CR.Range.first, J=I+1, E = CR.Range.second;
02403        J!=E; ++I, ++J) {
02404     const APInt &LEnd = I->High->getValue();
02405     const APInt &RBegin = J->Low->getValue();
02406     APInt Range = ComputeRange(LEnd, RBegin);
02407     assert((Range - 2ULL).isNonNegative() &&
02408            "Invalid case distance");
02409     // Use volatile double here to avoid excess precision issues on some hosts,
02410     // e.g. that use 80-bit X87 registers.
02411     // Only consider the density of sub-ranges that actually have sufficient
02412     // entries to be lowered as a jump table.
02413     volatile double LDensity =
02414         LSize.ult(TLI.getMinimumJumpTableEntries())
02415             ? 0.0
02416             : LSize.roundToDouble() / (LEnd - First + 1ULL).roundToDouble();
02417     volatile double RDensity =
02418         RSize.ult(TLI.getMinimumJumpTableEntries())
02419             ? 0.0
02420             : RSize.roundToDouble() / (Last - RBegin + 1ULL).roundToDouble();
02421     volatile double Metric = Range.logBase2() * (LDensity + RDensity);
02422     // Should always split in some non-trivial place
02423     DEBUG(dbgs() <<"=>Step\n"
02424                  << "LEnd: " << LEnd << ", RBegin: " << RBegin << '\n'
02425                  << "LDensity: " << LDensity
02426                  << ", RDensity: " << RDensity << '\n'
02427                  << "Metric: " << Metric << '\n');
02428     if (FMetric < Metric) {
02429       Pivot = J;
02430       FMetric = Metric;
02431       DEBUG(dbgs() << "Current metric set to: " << FMetric << '\n');
02432     }
02433 
02434     LSize += J->size();
02435     RSize -= J->size();
02436   }
02437 
02438   if (FMetric == 0 || !areJTsAllowed(TLI))
02439     Pivot = CR.Range.first + Size/2;
02440   splitSwitchCase(CR, Pivot, WorkList, SV, SwitchBB);
02441   return true;
02442 }
02443 
02444 void SelectionDAGBuilder::splitSwitchCase(CaseRec &CR, CaseItr Pivot,
02445                                           CaseRecVector &WorkList,
02446                                           const Value *SV,
02447                                           MachineBasicBlock *SwitchBB) {
02448   // Get the MachineFunction which holds the current MBB.  This is used when
02449   // inserting any additional MBBs necessary to represent the switch.
02450   MachineFunction *CurMF = FuncInfo.MF;
02451 
02452   // Figure out which block is immediately after the current one.
02453   MachineFunction::iterator BBI = CR.CaseBB;
02454   ++BBI;
02455 
02456   const BasicBlock *LLVMBB = CR.CaseBB->getBasicBlock();
02457 
02458   CaseRange LHSR(CR.Range.first, Pivot);
02459   CaseRange RHSR(Pivot, CR.Range.second);
02460   const ConstantInt *C = Pivot->Low;
02461   MachineBasicBlock *FalseBB = nullptr, *TrueBB = nullptr;
02462 
02463   // We know that we branch to the LHS if the Value being switched on is
02464   // less than the Pivot value, C.  We use this to optimize our binary
02465   // tree a bit, by recognizing that if SV is greater than or equal to the
02466   // LHS's Case Value, and that Case Value is exactly one less than the
02467   // Pivot's Value, then we can branch directly to the LHS's Target,
02468   // rather than creating a leaf node for it.
02469   if ((LHSR.second - LHSR.first) == 1 && LHSR.first->High == CR.GE &&
02470       C->getValue() == (CR.GE->getValue() + 1LL)) {
02471     TrueBB = LHSR.first->BB;
02472   } else {
02473     TrueBB = CurMF->CreateMachineBasicBlock(LLVMBB);
02474     CurMF->insert(BBI, TrueBB);
02475     WorkList.push_back(CaseRec(TrueBB, C, CR.GE, LHSR));
02476 
02477     // Put SV in a virtual register to make it available from the new blocks.
02478     ExportFromCurrentBlock(SV);
02479   }
02480 
02481   // Similar to the optimization above, if the Value being switched on is
02482   // known to be less than the Constant CR.LT, and the current Case Value
02483   // is CR.LT - 1, then we can branch directly to the target block for
02484   // the current Case Value, rather than emitting a RHS leaf node for it.
02485   if ((RHSR.second - RHSR.first) == 1 && CR.LT &&
02486       RHSR.first->Low->getValue() == (CR.LT->getValue() - 1LL)) {
02487     FalseBB = RHSR.first->BB;
02488   } else {
02489     FalseBB = CurMF->CreateMachineBasicBlock(LLVMBB);
02490     CurMF->insert(BBI, FalseBB);
02491     WorkList.push_back(CaseRec(FalseBB, CR.LT, C, RHSR));
02492 
02493     // Put SV in a virtual register to make it available from the new blocks.
02494     ExportFromCurrentBlock(SV);
02495   }
02496 
02497   // Create a CaseBlock record representing a conditional branch to
02498   // the LHS node if the value being switched on SV is less than C.
02499   // Otherwise, branch to LHS.
02500   CaseBlock CB(ISD::SETLT, SV, C, nullptr, TrueBB, FalseBB, CR.CaseBB);
02501 
02502   if (CR.CaseBB == SwitchBB)
02503     visitSwitchCase(CB, SwitchBB);
02504   else
02505     SwitchCases.push_back(CB);
02506 }
02507 
02508 /// handleBitTestsSwitchCase - if current case range has few destination and
02509 /// range span less, than machine word bitwidth, encode case range into series
02510 /// of masks and emit bit tests with these masks.
02511 bool SelectionDAGBuilder::handleBitTestsSwitchCase(CaseRec& CR,
02512                                                    CaseRecVector& WorkList,
02513                                                    const Value* SV,
02514                                                    MachineBasicBlock* Default,
02515                                                    MachineBasicBlock* SwitchBB) {
02516   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
02517   EVT PTy = TLI.getPointerTy();
02518   unsigned IntPtrBits = PTy.getSizeInBits();
02519 
02520   Case& FrontCase = *CR.Range.first;
02521   Case& BackCase  = *(CR.Range.second-1);
02522 
02523   // Get the MachineFunction which holds the current MBB.  This is used when
02524   // inserting any additional MBBs necessary to represent the switch.
02525   MachineFunction *CurMF = FuncInfo.MF;
02526 
02527   // If target does not have legal shift left, do not emit bit tests at all.
02528   if (!TLI.isOperationLegal(ISD::SHL, PTy))
02529     return false;
02530 
02531   size_t numCmps = 0;
02532   for (CaseItr I = CR.Range.first, E = CR.Range.second; I != E; ++I) {
02533     // Single case counts one, case range - two.
02534     numCmps += (I->Low == I->High ? 1 : 2);
02535   }
02536 
02537   // Count unique destinations
02538   SmallSet<MachineBasicBlock*, 4> Dests;
02539   for (CaseItr I = CR.Range.first, E = CR.Range.second; I != E; ++I) {
02540     Dests.insert(I->BB);
02541     if (Dests.size() > 3)
02542       // Don't bother the code below, if there are too much unique destinations
02543       return false;
02544   }
02545   DEBUG(dbgs() << "Total number of unique destinations: "
02546         << Dests.size() << '\n'
02547         << "Total number of comparisons: " << numCmps << '\n');
02548 
02549   // Compute span of values.
02550   const APInt& minValue = FrontCase.Low->getValue();
02551   const APInt& maxValue = BackCase.High->getValue();
02552   APInt cmpRange = maxValue - minValue;
02553 
02554   DEBUG(dbgs() << "Compare range: " << cmpRange << '\n'
02555                << "Low bound: " << minValue << '\n'
02556                << "High bound: " << maxValue << '\n');
02557 
02558   if (cmpRange.uge(IntPtrBits) ||
02559       (!(Dests.size() == 1 && numCmps >= 3) &&
02560        !(Dests.size() == 2 && numCmps >= 5) &&
02561        !(Dests.size() >= 3 && numCmps >= 6)))
02562     return false;
02563 
02564   DEBUG(dbgs() << "Emitting bit tests\n");
02565   APInt lowBound = APInt::getNullValue(cmpRange.getBitWidth());
02566 
02567   // Optimize the case where all the case values fit in a
02568   // word without having to subtract minValue. In this case,
02569   // we can optimize away the subtraction.
02570   if (minValue.isNonNegative() && maxValue.slt(IntPtrBits)) {
02571     cmpRange = maxValue;
02572   } else {
02573     lowBound = minValue;
02574   }
02575 
02576   CaseBitsVector CasesBits;
02577   unsigned i, count = 0;
02578 
02579   for (CaseItr I = CR.Range.first, E = CR.Range.second; I!=E; ++I) {
02580     MachineBasicBlock* Dest = I->BB;
02581     for (i = 0; i < count; ++i)
02582       if (Dest == CasesBits[i].BB)
02583         break;
02584 
02585     if (i == count) {
02586       assert((count < 3) && "Too much destinations to test!");
02587       CasesBits.push_back(CaseBits(0, Dest, 0, 0/*Weight*/));
02588       count++;
02589     }
02590 
02591     const APInt& lowValue = I->Low->getValue();
02592     const APInt& highValue = I->High->getValue();
02593 
02594     uint64_t lo = (lowValue - lowBound).getZExtValue();
02595     uint64_t hi = (highValue - lowBound).getZExtValue();
02596     CasesBits[i].ExtraWeight += I->ExtraWeight;
02597 
02598     for (uint64_t j = lo; j <= hi; j++) {
02599       CasesBits[i].Mask |=  1ULL << j;
02600       CasesBits[i].Bits++;
02601     }
02602 
02603   }
02604   std::sort(CasesBits.begin(), CasesBits.end(), CaseBitsCmp());
02605 
02606   BitTestInfo BTC;
02607 
02608   // Figure out which block is immediately after the current one.
02609   MachineFunction::iterator BBI = CR.CaseBB;
02610   ++BBI;
02611 
02612   const BasicBlock *LLVMBB = CR.CaseBB->getBasicBlock();
02613 
02614   DEBUG(dbgs() << "Cases:\n");
02615   for (unsigned i = 0, e = CasesBits.size(); i!=e; ++i) {
02616     DEBUG(dbgs() << "Mask: " << CasesBits[i].Mask
02617                  << ", Bits: " << CasesBits[i].Bits
02618                  << ", BB: " << CasesBits[i].BB << '\n');
02619 
02620     MachineBasicBlock *CaseBB = CurMF->CreateMachineBasicBlock(LLVMBB);
02621     CurMF->insert(BBI, CaseBB);
02622     BTC.push_back(BitTestCase(CasesBits[i].Mask,
02623                               CaseBB,
02624                               CasesBits[i].BB, CasesBits[i].ExtraWeight));
02625 
02626     // Put SV in a virtual register to make it available from the new blocks.
02627     ExportFromCurrentBlock(SV);
02628   }
02629 
02630   BitTestBlock BTB(lowBound, cmpRange, SV,
02631                    -1U, MVT::Other, (CR.CaseBB == SwitchBB),
02632                    CR.CaseBB, Default, std::move(BTC));
02633 
02634   if (CR.CaseBB == SwitchBB)
02635     visitBitTestHeader(BTB, SwitchBB);
02636 
02637   BitTestCases.push_back(std::move(BTB));
02638 
02639   return true;
02640 }
02641 
02642 void SelectionDAGBuilder::Clusterify(CaseVector &Cases, const SwitchInst *SI) {
02643   BranchProbabilityInfo *BPI = FuncInfo.BPI;
02644 
02645   // Extract cases from the switch and sort them.
02646   typedef std::pair<const ConstantInt*, unsigned> CasePair;
02647   std::vector<CasePair> Sorted;
02648   Sorted.reserve(SI->getNumCases());
02649   for (auto I : SI->cases())
02650     Sorted.push_back(std::make_pair(I.getCaseValue(), I.getSuccessorIndex()));
02651   std::sort(Sorted.begin(), Sorted.end(), [](CasePair a, CasePair b) {
02652     return a.first->getValue().slt(b.first->getValue());
02653   });
02654 
02655   // Merge adjacent cases with the same destination, build Cases vector.
02656   assert(Cases.empty() && "Cases should be empty before Clusterify;");
02657   Cases.reserve(SI->getNumCases());
02658   MachineBasicBlock *PreviousSucc = nullptr;
02659   for (CasePair &CP : Sorted) {
02660     const ConstantInt *CaseVal = CP.first;
02661     unsigned SuccIndex = CP.second;
02662     MachineBasicBlock *Succ = FuncInfo.MBBMap[SI->getSuccessor(SuccIndex)];
02663     uint32_t Weight = BPI ? BPI->getEdgeWeight(SI->getParent(), SuccIndex) : 0;
02664 
02665     if (PreviousSucc == Succ &&
02666         (CaseVal->getValue() - Cases.back().High->getValue()) == 1) {
02667       // If this case has the same successor and is a neighbour, merge it into
02668       // the previous cluster.
02669       Cases.back().High = CaseVal;
02670       Cases.back().ExtraWeight += Weight;
02671     } else {
02672       Cases.push_back(Case(CaseVal, CaseVal, Succ, Weight));
02673     }
02674 
02675     PreviousSucc = Succ;
02676   }
02677 
02678   DEBUG({
02679       size_t numCmps = 0;
02680       for (auto &I : Cases)
02681         // A range counts double, since it requires two compares.
02682         numCmps += I.Low != I.High ? 2 : 1;
02683 
02684       dbgs() << "Clusterify finished. Total clusters: " << Cases.size()
02685              << ". Total compares: " << numCmps << '\n';
02686     });
02687 }
02688 
02689 void SelectionDAGBuilder::UpdateSplitBlock(MachineBasicBlock *First,
02690                                            MachineBasicBlock *Last) {
02691   // Update JTCases.
02692   for (unsigned i = 0, e = JTCases.size(); i != e; ++i)
02693     if (JTCases[i].first.HeaderBB == First)
02694       JTCases[i].first.HeaderBB = Last;
02695 
02696   // Update BitTestCases.
02697   for (unsigned i = 0, e = BitTestCases.size(); i != e; ++i)
02698     if (BitTestCases[i].Parent == First)
02699       BitTestCases[i].Parent = Last;
02700 }
02701 
02702 void SelectionDAGBuilder::visitSwitch(const SwitchInst &SI) {
02703   MachineBasicBlock *SwitchMBB = FuncInfo.MBB;
02704 
02705   // Create a vector of Cases, sorted so that we can efficiently create a binary
02706   // search tree from them.
02707   CaseVector Cases;
02708   Clusterify(Cases, &SI);
02709 
02710   // Get the default destination MBB.
02711   MachineBasicBlock *Default = FuncInfo.MBBMap[SI.getDefaultDest()];
02712 
02713   if (isa<UnreachableInst>(SI.getDefaultDest()->getFirstNonPHIOrDbg()) &&
02714       !Cases.empty()) {
02715     // Replace an unreachable default destination with the most popular case
02716     // destination.
02717     DenseMap<const BasicBlock *, unsigned> Popularity;
02718     unsigned MaxPop = 0;
02719     const BasicBlock *MaxBB = nullptr;
02720     for (auto I : SI.cases()) {
02721       const BasicBlock *BB = I.getCaseSuccessor();
02722       if (++Popularity[BB] > MaxPop) {
02723         MaxPop = Popularity[BB];
02724         MaxBB = BB;
02725       }
02726     }
02727 
02728     // Set new default.
02729     assert(MaxPop > 0);
02730     assert(MaxBB);
02731     Default = FuncInfo.MBBMap[MaxBB];
02732 
02733     // Remove cases that were pointing to the destination that is now the default.
02734     Cases.erase(std::remove_if(Cases.begin(), Cases.end(),
02735                                [&](const Case &C) { return C.BB == Default; }),
02736                 Cases.end());
02737   }
02738 
02739   // If there is only the default destination, go there directly.
02740   if (Cases.empty()) {
02741     // Update machine-CFG edges.
02742     SwitchMBB->addSuccessor(Default);
02743 
02744     // If this is not a fall-through branch, emit the branch.
02745     if (Default != NextBlock(SwitchMBB)) {
02746       DAG.setRoot(DAG.getNode(ISD::BR, getCurSDLoc(), MVT::Other,
02747                               getControlRoot(), DAG.getBasicBlock(Default)));
02748     }
02749     return;
02750   }
02751 
02752   // Get the Value to be switched on.
02753   const Value *SV = SI.getCondition();
02754 
02755   // Push the initial CaseRec onto the worklist
02756   CaseRecVector WorkList;
02757   WorkList.push_back(CaseRec(SwitchMBB,nullptr,nullptr,
02758                              CaseRange(Cases.begin(),Cases.end())));
02759 
02760   while (!WorkList.empty()) {
02761     // Grab a record representing a case range to process off the worklist
02762     CaseRec CR = WorkList.back();
02763     WorkList.pop_back();
02764 
02765     if (handleBitTestsSwitchCase(CR, WorkList, SV, Default, SwitchMBB))
02766       continue;
02767 
02768     // If the range has few cases (two or less) emit a series of specific
02769     // tests.
02770     if (handleSmallSwitchRange(CR, WorkList, SV, Default, SwitchMBB))
02771       continue;
02772 
02773     // If the switch has more than N blocks, and is at least 40% dense, and the
02774     // target supports indirect branches, then emit a jump table rather than
02775     // lowering the switch to a binary tree of conditional branches.
02776     // N defaults to 4 and is controlled via TLS.getMinimumJumpTableEntries().
02777     if (handleJTSwitchCase(CR, WorkList, SV, Default, SwitchMBB))
02778       continue;
02779 
02780     // Emit binary tree. We need to pick a pivot, and push left and right ranges
02781     // onto the worklist. Leafs are handled via handleSmallSwitchRange() call.
02782     handleBTSplitSwitchCase(CR, WorkList, SV, SwitchMBB);
02783   }
02784 }
02785 
02786 void SelectionDAGBuilder::visitIndirectBr(const IndirectBrInst &I) {
02787   MachineBasicBlock *IndirectBrMBB = FuncInfo.MBB;
02788 
02789   // Update machine-CFG edges with unique successors.
02790   SmallSet<BasicBlock*, 32> Done;
02791   for (unsigned i = 0, e = I.getNumSuccessors(); i != e; ++i) {
02792     BasicBlock *BB = I.getSuccessor(i);
02793     bool Inserted = Done.insert(BB).second;
02794     if (!Inserted)
02795         continue;
02796 
02797     MachineBasicBlock *Succ = FuncInfo.MBBMap[BB];
02798     addSuccessorWithWeight(IndirectBrMBB, Succ);
02799   }
02800 
02801   DAG.setRoot(DAG.getNode(ISD::BRIND, getCurSDLoc(),
02802                           MVT::Other, getControlRoot(),
02803                           getValue(I.getAddress())));
02804 }
02805 
02806 void SelectionDAGBuilder::visitUnreachable(const UnreachableInst &I) {
02807   if (DAG.getTarget().Options.TrapUnreachable)
02808     DAG.setRoot(DAG.getNode(ISD::TRAP, getCurSDLoc(), MVT::Other, DAG.getRoot()));
02809 }
02810 
02811 void SelectionDAGBuilder::visitFSub(const User &I) {
02812   // -0.0 - X --> fneg
02813   Type *Ty = I.getType();
02814   if (isa<Constant>(I.getOperand(0)) &&
02815       I.getOperand(0) == ConstantFP::getZeroValueForNegation(Ty)) {
02816     SDValue Op2 = getValue(I.getOperand(1));
02817     setValue(&I, DAG.getNode(ISD::FNEG, getCurSDLoc(),
02818                              Op2.getValueType(), Op2));
02819     return;
02820   }
02821 
02822   visitBinary(I, ISD::FSUB);
02823 }
02824 
02825 void SelectionDAGBuilder::visitBinary(const User &I, unsigned OpCode) {
02826   SDValue Op1 = getValue(I.getOperand(0));
02827   SDValue Op2 = getValue(I.getOperand(1));
02828 
02829   bool nuw = false;
02830   bool nsw = false;
02831   bool exact = false;
02832   if (const OverflowingBinaryOperator *OFBinOp =
02833           dyn_cast<const OverflowingBinaryOperator>(&I)) {
02834     nuw = OFBinOp->hasNoUnsignedWrap();
02835     nsw = OFBinOp->hasNoSignedWrap();
02836   }
02837   if (const PossiblyExactOperator *ExactOp =
02838           dyn_cast<const PossiblyExactOperator>(&I))
02839     exact = ExactOp->isExact();
02840 
02841   SDValue BinNodeValue = DAG.getNode(OpCode, getCurSDLoc(), Op1.getValueType(),
02842                                      Op1, Op2, nuw, nsw, exact);
02843   setValue(&I, BinNodeValue);
02844 }
02845 
02846 void SelectionDAGBuilder::visitShift(const User &I, unsigned Opcode) {
02847   SDValue Op1 = getValue(I.getOperand(0));
02848   SDValue Op2 = getValue(I.getOperand(1));
02849 
02850   EVT ShiftTy =
02851       DAG.getTargetLoweringInfo().getShiftAmountTy(Op2.getValueType());
02852 
02853   // Coerce the shift amount to the right type if we can.
02854   if (!I.getType()->isVectorTy() && Op2.getValueType() != ShiftTy) {
02855     unsigned ShiftSize = ShiftTy.getSizeInBits();
02856     unsigned Op2Size = Op2.getValueType().getSizeInBits();
02857     SDLoc DL = getCurSDLoc();
02858 
02859     // If the operand is smaller than the shift count type, promote it.
02860     if (ShiftSize > Op2Size)
02861       Op2 = DAG.getNode(ISD::ZERO_EXTEND, DL, ShiftTy, Op2);
02862 
02863     // If the operand is larger than the shift count type but the shift
02864     // count type has enough bits to represent any shift value, truncate
02865     // it now. This is a common case and it exposes the truncate to
02866     // optimization early.
02867     else if (ShiftSize >= Log2_32_Ceil(Op2.getValueType().getSizeInBits()))
02868       Op2 = DAG.getNode(ISD::TRUNCATE, DL, ShiftTy, Op2);
02869     // Otherwise we'll need to temporarily settle for some other convenient
02870     // type.  Type legalization will make adjustments once the shiftee is split.
02871     else
02872       Op2 = DAG.getZExtOrTrunc(Op2, DL, MVT::i32);
02873   }
02874 
02875   bool nuw = false;
02876   bool nsw = false;
02877   bool exact = false;
02878 
02879   if (Opcode == ISD::SRL || Opcode == ISD::SRA || Opcode == ISD::SHL) {
02880 
02881     if (const OverflowingBinaryOperator *OFBinOp =
02882             dyn_cast<const OverflowingBinaryOperator>(&I)) {
02883       nuw = OFBinOp->hasNoUnsignedWrap();
02884       nsw = OFBinOp->hasNoSignedWrap();
02885     }
02886     if (const PossiblyExactOperator *ExactOp =
02887             dyn_cast<const PossiblyExactOperator>(&I))
02888       exact = ExactOp->isExact();
02889   }
02890 
02891   SDValue Res = DAG.getNode(Opcode, getCurSDLoc(), Op1.getValueType(), Op1, Op2,
02892                             nuw, nsw, exact);
02893   setValue(&I, Res);
02894 }
02895 
02896 void SelectionDAGBuilder::visitSDiv(const User &I) {
02897   SDValue Op1 = getValue(I.getOperand(0));
02898   SDValue Op2 = getValue(I.getOperand(1));
02899 
02900   // Turn exact SDivs into multiplications.
02901   // FIXME: This should be in DAGCombiner, but it doesn't have access to the
02902   // exact bit.
02903   if (isa<BinaryOperator>(&I) && cast<BinaryOperator>(&I)->isExact() &&
02904       !isa<ConstantSDNode>(Op1) &&
02905       isa<ConstantSDNode>(Op2) && !cast<ConstantSDNode>(Op2)->isNullValue())
02906     setValue(&I, DAG.getTargetLoweringInfo()
02907                      .BuildExactSDIV(Op1, Op2, getCurSDLoc(), DAG));
02908   else
02909     setValue(&I, DAG.getNode(ISD::SDIV, getCurSDLoc(), Op1.getValueType(),
02910                              Op1, Op2));
02911 }
02912 
02913 void SelectionDAGBuilder::visitICmp(const User &I) {
02914   ICmpInst::Predicate predicate = ICmpInst::BAD_ICMP_PREDICATE;
02915   if (const ICmpInst *IC = dyn_cast<ICmpInst>(&I))
02916     predicate = IC->getPredicate();
02917   else if (const ConstantExpr *IC = dyn_cast<ConstantExpr>(&I))
02918     predicate = ICmpInst::Predicate(IC->getPredicate());
02919   SDValue Op1 = getValue(I.getOperand(0));
02920   SDValue Op2 = getValue(I.getOperand(1));
02921   ISD::CondCode Opcode = getICmpCondCode(predicate);
02922 
02923   EVT DestVT = DAG.getTargetLoweringInfo().getValueType(I.getType());
02924   setValue(&I, DAG.getSetCC(getCurSDLoc(), DestVT, Op1, Op2, Opcode));
02925 }
02926 
02927 void SelectionDAGBuilder::visitFCmp(const User &I) {
02928   FCmpInst::Predicate predicate = FCmpInst::BAD_FCMP_PREDICATE;
02929   if (const FCmpInst *FC = dyn_cast<FCmpInst>(&I))
02930     predicate = FC->getPredicate();
02931   else if (const ConstantExpr *FC = dyn_cast<ConstantExpr>(&I))
02932     predicate = FCmpInst::Predicate(FC->getPredicate());
02933   SDValue Op1 = getValue(I.getOperand(0));
02934   SDValue Op2 = getValue(I.getOperand(1));
02935   ISD::CondCode Condition = getFCmpCondCode(predicate);
02936   if (TM.Options.NoNaNsFPMath)
02937     Condition = getFCmpCodeWithoutNaN(Condition);
02938   EVT DestVT = DAG.getTargetLoweringInfo().getValueType(I.getType());
02939   setValue(&I, DAG.getSetCC(getCurSDLoc(), DestVT, Op1, Op2, Condition));
02940 }
02941 
02942 void SelectionDAGBuilder::visitSelect(const User &I) {
02943   SmallVector<EVT, 4> ValueVTs;
02944   ComputeValueVTs(DAG.getTargetLoweringInfo(), I.getType(), ValueVTs);
02945   unsigned NumValues = ValueVTs.size();
02946   if (NumValues == 0) return;
02947 
02948   SmallVector<SDValue, 4> Values(NumValues);
02949   SDValue Cond     = getValue(I.getOperand(0));
02950   SDValue TrueVal  = getValue(I.getOperand(1));
02951   SDValue FalseVal = getValue(I.getOperand(2));
02952   ISD::NodeType OpCode = Cond.getValueType().isVector() ?
02953     ISD::VSELECT : ISD::SELECT;
02954 
02955   for (unsigned i = 0; i != NumValues; ++i)
02956     Values[i] = DAG.getNode(OpCode, getCurSDLoc(),
02957                             TrueVal.getNode()->getValueType(TrueVal.getResNo()+i),
02958                             Cond,
02959                             SDValue(TrueVal.getNode(),
02960                                     TrueVal.getResNo() + i),
02961                             SDValue(FalseVal.getNode(),
02962                                     FalseVal.getResNo() + i));
02963 
02964   setValue(&I, DAG.getNode(ISD::MERGE_VALUES, getCurSDLoc(),
02965                            DAG.getVTList(ValueVTs), Values));
02966 }
02967 
02968 void SelectionDAGBuilder::visitTrunc(const User &I) {
02969   // TruncInst cannot be a no-op cast because sizeof(src) > sizeof(dest).
02970   SDValue N = getValue(I.getOperand(0));
02971   EVT DestVT = DAG.getTargetLoweringInfo().getValueType(I.getType());
02972   setValue(&I, DAG.getNode(ISD::TRUNCATE, getCurSDLoc(), DestVT, N));
02973 }
02974 
02975 void SelectionDAGBuilder::visitZExt(const User &I) {
02976   // ZExt cannot be a no-op cast because sizeof(src) < sizeof(dest).
02977   // ZExt also can't be a cast to bool for same reason. So, nothing much to do
02978   SDValue N = getValue(I.getOperand(0));
02979   EVT DestVT = DAG.getTargetLoweringInfo().getValueType(I.getType());
02980   setValue(&I, DAG.getNode(ISD::ZERO_EXTEND, getCurSDLoc(), DestVT, N));
02981 }
02982 
02983 void SelectionDAGBuilder::visitSExt(const User &I) {
02984   // SExt cannot be a no-op cast because sizeof(src) < sizeof(dest).
02985   // SExt also can't be a cast to bool for same reason. So, nothing much to do
02986   SDValue N = getValue(I.getOperand(0));
02987   EVT DestVT = DAG.getTargetLoweringInfo().getValueType(I.getType());
02988   setValue(&I, DAG.getNode(ISD::SIGN_EXTEND, getCurSDLoc(), DestVT, N));
02989 }
02990 
02991 void SelectionDAGBuilder::visitFPTrunc(const User &I) {
02992   // FPTrunc is never a no-op cast, no need to check
02993   SDValue N = getValue(I.getOperand(0));
02994   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
02995   EVT DestVT = TLI.getValueType(I.getType());
02996   setValue(&I, DAG.getNode(ISD::FP_ROUND, getCurSDLoc(), DestVT, N,
02997                            DAG.getTargetConstant(0, TLI.getPointerTy())));
02998 }
02999 
03000 void SelectionDAGBuilder::visitFPExt(const User &I) {
03001   // FPExt is never a no-op cast, no need to check
03002   SDValue N = getValue(I.getOperand(0));
03003   EVT DestVT = DAG.getTargetLoweringInfo().getValueType(I.getType());
03004   setValue(&I, DAG.getNode(ISD::FP_EXTEND, getCurSDLoc(), DestVT, N));
03005 }
03006 
03007 void SelectionDAGBuilder::visitFPToUI(const User &I) {
03008   // FPToUI is never a no-op cast, no need to check
03009   SDValue N = getValue(I.getOperand(0));
03010   EVT DestVT = DAG.getTargetLoweringInfo().getValueType(I.getType());
03011   setValue(&I, DAG.getNode(ISD::FP_TO_UINT, getCurSDLoc(), DestVT, N));
03012 }
03013 
03014 void SelectionDAGBuilder::visitFPToSI(const User &I) {
03015   // FPToSI is never a no-op cast, no need to check
03016   SDValue N = getValue(I.getOperand(0));
03017   EVT DestVT = DAG.getTargetLoweringInfo().getValueType(I.getType());
03018   setValue(&I, DAG.getNode(ISD::FP_TO_SINT, getCurSDLoc(), DestVT, N));
03019 }
03020 
03021 void SelectionDAGBuilder::visitUIToFP(const User &I) {
03022   // UIToFP is never a no-op cast, no need to check
03023   SDValue N = getValue(I.getOperand(0));
03024   EVT DestVT = DAG.getTargetLoweringInfo().getValueType(I.getType());
03025   setValue(&I, DAG.getNode(ISD::UINT_TO_FP, getCurSDLoc(), DestVT, N));
03026 }
03027 
03028 void SelectionDAGBuilder::visitSIToFP(const User &I) {
03029   // SIToFP is never a no-op cast, no need to check
03030   SDValue N = getValue(I.getOperand(0));
03031   EVT DestVT = DAG.getTargetLoweringInfo().getValueType(I.getType());
03032   setValue(&I, DAG.getNode(ISD::SINT_TO_FP, getCurSDLoc(), DestVT, N));
03033 }
03034 
03035 void SelectionDAGBuilder::visitPtrToInt(const User &I) {
03036   // What to do depends on the size of the integer and the size of the pointer.
03037   // We can either truncate, zero extend, or no-op, accordingly.
03038   SDValue N = getValue(I.getOperand(0));
03039   EVT DestVT = DAG.getTargetLoweringInfo().getValueType(I.getType());
03040   setValue(&I, DAG.getZExtOrTrunc(N, getCurSDLoc(), DestVT));
03041 }
03042 
03043 void SelectionDAGBuilder::visitIntToPtr(const User &I) {
03044   // What to do depends on the size of the integer and the size of the pointer.
03045   // We can either truncate, zero extend, or no-op, accordingly.
03046   SDValue N = getValue(I.getOperand(0));
03047   EVT DestVT = DAG.getTargetLoweringInfo().getValueType(I.getType());
03048   setValue(&I, DAG.getZExtOrTrunc(N, getCurSDLoc(), DestVT));
03049 }
03050 
03051 void SelectionDAGBuilder::visitBitCast(const User &I) {
03052   SDValue N = getValue(I.getOperand(0));
03053   EVT DestVT = DAG.getTargetLoweringInfo().getValueType(I.getType());
03054 
03055   // BitCast assures us that source and destination are the same size so this is
03056   // either a BITCAST or a no-op.
03057   if (DestVT != N.getValueType())
03058     setValue(&I, DAG.getNode(ISD::BITCAST, getCurSDLoc(),
03059                              DestVT, N)); // convert types.
03060   // Check if the original LLVM IR Operand was a ConstantInt, because getValue()
03061   // might fold any kind of constant expression to an integer constant and that
03062   // is not what we are looking for. Only regcognize a bitcast of a genuine
03063   // constant integer as an opaque constant.
03064   else if(ConstantInt *C = dyn_cast<ConstantInt>(I.getOperand(0)))
03065     setValue(&I, DAG.getConstant(C->getValue(), DestVT, /*isTarget=*/false,
03066                                  /*isOpaque*/true));
03067   else
03068     setValue(&I, N);            // noop cast.
03069 }
03070 
03071 void SelectionDAGBuilder::visitAddrSpaceCast(const User &I) {
03072   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
03073   const Value *SV = I.getOperand(0);
03074   SDValue N = getValue(SV);
03075   EVT DestVT = TLI.getValueType(I.getType());
03076 
03077   unsigned SrcAS = SV->getType()->getPointerAddressSpace();
03078   unsigned DestAS = I.getType()->getPointerAddressSpace();
03079 
03080   if (!TLI.isNoopAddrSpaceCast(SrcAS, DestAS))
03081     N = DAG.getAddrSpaceCast(getCurSDLoc(), DestVT, N, SrcAS, DestAS);
03082 
03083   setValue(&I, N);
03084 }
03085 
03086 void SelectionDAGBuilder::visitInsertElement(const User &I) {
03087   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
03088   SDValue InVec = getValue(I.getOperand(0));
03089   SDValue InVal = getValue(I.getOperand(1));
03090   SDValue InIdx = DAG.getSExtOrTrunc(getValue(I.getOperand(2)),
03091                                      getCurSDLoc(), TLI.getVectorIdxTy());
03092   setValue(&I, DAG.getNode(ISD::INSERT_VECTOR_ELT, getCurSDLoc(),
03093                            TLI.getValueType(I.getType()), InVec, InVal, InIdx));
03094 }
03095 
03096 void SelectionDAGBuilder::visitExtractElement(const User &I) {
03097   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
03098   SDValue InVec = getValue(I.getOperand(0));
03099   SDValue InIdx = DAG.getSExtOrTrunc(getValue(I.getOperand(1)),
03100                                      getCurSDLoc(), TLI.getVectorIdxTy());
03101   setValue(&I, DAG.getNode(ISD::EXTRACT_VECTOR_ELT, getCurSDLoc(),
03102                            TLI.getValueType(I.getType()), InVec, InIdx));
03103 }
03104 
03105 // Utility for visitShuffleVector - Return true if every element in Mask,
03106 // beginning from position Pos and ending in Pos+Size, falls within the
03107 // specified sequential range [L, L+Pos). or is undef.
03108 static bool isSequentialInRange(const SmallVectorImpl<int> &Mask,
03109                                 unsigned Pos, unsigned Size, int Low) {
03110   for (unsigned i = Pos, e = Pos+Size; i != e; ++i, ++Low)
03111     if (Mask[i] >= 0 && Mask[i] != Low)
03112       return false;
03113   return true;
03114 }
03115 
03116 void SelectionDAGBuilder::visitShuffleVector(const User &I) {
03117   SDValue Src1 = getValue(I.getOperand(0));
03118   SDValue Src2 = getValue(I.getOperand(1));
03119 
03120   SmallVector<int, 8> Mask;
03121   ShuffleVectorInst::getShuffleMask(cast<Constant>(I.getOperand(2)), Mask);
03122   unsigned MaskNumElts = Mask.size();
03123 
03124   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
03125   EVT VT = TLI.getValueType(I.getType());
03126   EVT SrcVT = Src1.getValueType();
03127   unsigned SrcNumElts = SrcVT.getVectorNumElements();
03128 
03129   if (SrcNumElts == MaskNumElts) {
03130     setValue(&I, DAG.getVectorShuffle(VT, getCurSDLoc(), Src1, Src2,
03131                                       &Mask[0]));
03132     return;
03133   }
03134 
03135   // Normalize the shuffle vector since mask and vector length don't match.
03136   if (SrcNumElts < MaskNumElts && MaskNumElts % SrcNumElts == 0) {
03137     // Mask is longer than the source vectors and is a multiple of the source
03138     // vectors.  We can use concatenate vector to make the mask and vectors
03139     // lengths match.
03140     if (SrcNumElts*2 == MaskNumElts) {
03141       // First check for Src1 in low and Src2 in high
03142       if (isSequentialInRange(Mask, 0, SrcNumElts, 0) &&
03143           isSequentialInRange(Mask, SrcNumElts, SrcNumElts, SrcNumElts)) {
03144         // The shuffle is concatenating two vectors together.
03145         setValue(&I, DAG.getNode(ISD::CONCAT_VECTORS, getCurSDLoc(),
03146                                  VT, Src1, Src2));
03147         return;
03148       }
03149       // Then check for Src2 in low and Src1 in high
03150       if (isSequentialInRange(Mask, 0, SrcNumElts, SrcNumElts) &&
03151           isSequentialInRange(Mask, SrcNumElts, SrcNumElts, 0)) {
03152         // The shuffle is concatenating two vectors together.
03153         setValue(&I, DAG.getNode(ISD::CONCAT_VECTORS, getCurSDLoc(),
03154                                  VT, Src2, Src1));
03155         return;
03156       }
03157     }
03158 
03159     // Pad both vectors with undefs to make them the same length as the mask.
03160     unsigned NumConcat = MaskNumElts / SrcNumElts;
03161     bool Src1U = Src1.getOpcode() == ISD::UNDEF;
03162     bool Src2U = Src2.getOpcode() == ISD::UNDEF;
03163     SDValue UndefVal = DAG.getUNDEF(SrcVT);
03164 
03165     SmallVector<SDValue, 8> MOps1(NumConcat, UndefVal);
03166     SmallVector<SDValue, 8> MOps2(NumConcat, UndefVal);
03167     MOps1[0] = Src1;
03168     MOps2[0] = Src2;
03169 
03170     Src1 = Src1U ? DAG.getUNDEF(VT) : DAG.getNode(ISD::CONCAT_VECTORS,
03171                                                   getCurSDLoc(), VT, MOps1);
03172     Src2 = Src2U ? DAG.getUNDEF(VT) : DAG.getNode(ISD::CONCAT_VECTORS,
03173                                                   getCurSDLoc(), VT, MOps2);
03174 
03175     // Readjust mask for new input vector length.
03176     SmallVector<int, 8> MappedOps;
03177     for (unsigned i = 0; i != MaskNumElts; ++i) {
03178       int Idx = Mask[i];
03179       if (Idx >= (int)SrcNumElts)
03180         Idx -= SrcNumElts - MaskNumElts;
03181       MappedOps.push_back(Idx);
03182     }
03183 
03184     setValue(&I, DAG.getVectorShuffle(VT, getCurSDLoc(), Src1, Src2,
03185                                       &MappedOps[0]));
03186     return;
03187   }
03188 
03189   if (SrcNumElts > MaskNumElts) {
03190     // Analyze the access pattern of the vector to see if we can extract
03191     // two subvectors and do the shuffle. The analysis is done by calculating
03192     // the range of elements the mask access on both vectors.
03193     int MinRange[2] = { static_cast<int>(SrcNumElts),
03194                         static_cast<int>(SrcNumElts)};
03195     int MaxRange[2] = {-1, -1};
03196 
03197     for (unsigned i = 0; i != MaskNumElts; ++i) {
03198       int Idx = Mask[i];
03199       unsigned Input = 0;
03200       if (Idx < 0)
03201         continue;
03202 
03203       if (Idx >= (int)SrcNumElts) {
03204         Input = 1;
03205         Idx -= SrcNumElts;
03206       }
03207       if (Idx > MaxRange[Input])
03208         MaxRange[Input] = Idx;
03209       if (Idx < MinRange[Input])
03210         MinRange[Input] = Idx;
03211     }
03212 
03213     // Check if the access is smaller than the vector size and can we find
03214     // a reasonable extract index.
03215     int RangeUse[2] = { -1, -1 };  // 0 = Unused, 1 = Extract, -1 = Can not
03216                                    // Extract.
03217     int StartIdx[2];  // StartIdx to extract from
03218     for (unsigned Input = 0; Input < 2; ++Input) {
03219       if (MinRange[Input] >= (int)SrcNumElts && MaxRange[Input] < 0) {
03220         RangeUse[Input] = 0; // Unused
03221         StartIdx[Input] = 0;
03222         continue;
03223       }
03224 
03225       // Find a good start index that is a multiple of the mask length. Then
03226       // see if the rest of the elements are in range.
03227       StartIdx[Input] = (MinRange[Input]/MaskNumElts)*MaskNumElts;
03228       if (MaxRange[Input] - StartIdx[Input] < (int)MaskNumElts &&
03229           StartIdx[Input] + MaskNumElts <= SrcNumElts)
03230         RangeUse[Input] = 1; // Extract from a multiple of the mask length.
03231     }
03232 
03233     if (RangeUse[0] == 0 && RangeUse[1] == 0) {
03234       setValue(&I, DAG.getUNDEF(VT)); // Vectors are not used.
03235       return;
03236     }
03237     if (RangeUse[0] >= 0 && RangeUse[1] >= 0) {
03238       // Extract appropriate subvector and generate a vector shuffle
03239       for (unsigned Input = 0; Input < 2; ++Input) {
03240         SDValue &Src = Input == 0 ? Src1 : Src2;
03241         if (RangeUse[Input] == 0)
03242           Src = DAG.getUNDEF(VT);
03243         else
03244           Src = DAG.getNode(
03245               ISD::EXTRACT_SUBVECTOR, getCurSDLoc(), VT, Src,
03246               DAG.getConstant(StartIdx[Input], TLI.getVectorIdxTy()));
03247       }
03248 
03249       // Calculate new mask.
03250       SmallVector<int, 8> MappedOps;
03251       for (unsigned i = 0; i != MaskNumElts; ++i) {
03252         int Idx = Mask[i];
03253         if (Idx >= 0) {
03254           if (Idx < (int)SrcNumElts)
03255             Idx -= StartIdx[0];
03256           else
03257             Idx -= SrcNumElts + StartIdx[1] - MaskNumElts;
03258         }
03259         MappedOps.push_back(Idx);
03260       }
03261 
03262       setValue(&I, DAG.getVectorShuffle(VT, getCurSDLoc(), Src1, Src2,
03263                                         &MappedOps[0]));
03264       return;
03265     }
03266   }
03267 
03268   // We can't use either concat vectors or extract subvectors so fall back to
03269   // replacing the shuffle with extract and build vector.
03270   // to insert and build vector.
03271   EVT EltVT = VT.getVectorElementType();
03272   EVT IdxVT = TLI.getVectorIdxTy();
03273   SmallVector<SDValue,8> Ops;
03274   for (unsigned i = 0; i != MaskNumElts; ++i) {
03275     int Idx = Mask[i];
03276     SDValue Res;
03277 
03278     if (Idx < 0) {
03279       Res = DAG.getUNDEF(EltVT);
03280     } else {
03281       SDValue &Src = Idx < (int)SrcNumElts ? Src1 : Src2;
03282       if (Idx >= (int)SrcNumElts) Idx -= SrcNumElts;
03283 
03284       Res = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, getCurSDLoc(),
03285                         EltVT, Src, DAG.getConstant(Idx, IdxVT));
03286     }
03287 
03288     Ops.push_back(Res);
03289   }
03290 
03291   setValue(&I, DAG.getNode(ISD::BUILD_VECTOR, getCurSDLoc(), VT, Ops));
03292 }
03293 
03294 void SelectionDAGBuilder::visitInsertValue(const InsertValueInst &I) {
03295   const Value *Op0 = I.getOperand(0);
03296   const Value *Op1 = I.getOperand(1);
03297   Type *AggTy = I.getType();
03298   Type *ValTy = Op1->getType();
03299   bool IntoUndef = isa<UndefValue>(Op0);
03300   bool FromUndef = isa<UndefValue>(Op1);
03301 
03302   unsigned LinearIndex = ComputeLinearIndex(AggTy, I.getIndices());
03303 
03304   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
03305   SmallVector<EVT, 4> AggValueVTs;
03306   ComputeValueVTs(TLI, AggTy, AggValueVTs);
03307   SmallVector<EVT, 4> ValValueVTs;
03308   ComputeValueVTs(TLI, ValTy, ValValueVTs);
03309 
03310   unsigned NumAggValues = AggValueVTs.size();
03311   unsigned NumValValues = ValValueVTs.size();
03312   SmallVector<SDValue, 4> Values(NumAggValues);
03313 
03314   // Ignore an insertvalue that produces an empty object
03315   if (!NumAggValues) {
03316     setValue(&I, DAG.getUNDEF(MVT(MVT::Other)));
03317     return;
03318   }
03319 
03320   SDValue Agg = getValue(Op0);
03321   unsigned i = 0;
03322   // Copy the beginning value(s) from the original aggregate.
03323   for (; i != LinearIndex; ++i)
03324     Values[i] = IntoUndef ? DAG.getUNDEF(AggValueVTs[i]) :
03325                 SDValue(Agg.getNode(), Agg.getResNo() + i);
03326   // Copy values from the inserted value(s).
03327   if (NumValValues) {
03328     SDValue Val = getValue(Op1);
03329     for (; i != LinearIndex + NumValValues; ++i)
03330       Values[i] = FromUndef ? DAG.getUNDEF(AggValueVTs[i]) :
03331                   SDValue(Val.getNode(), Val.getResNo() + i - LinearIndex);
03332   }
03333   // Copy remaining value(s) from the original aggregate.
03334   for (; i != NumAggValues; ++i)
03335     Values[i] = IntoUndef ? DAG.getUNDEF(AggValueVTs[i]) :
03336                 SDValue(Agg.getNode(), Agg.getResNo() + i);
03337 
03338   setValue(&I, DAG.getNode(ISD::MERGE_VALUES, getCurSDLoc(),
03339                            DAG.getVTList(AggValueVTs), Values));
03340 }
03341 
03342 void SelectionDAGBuilder::visitExtractValue(const ExtractValueInst &I) {
03343   const Value *Op0 = I.getOperand(0);
03344   Type *AggTy = Op0->getType();
03345   Type *ValTy = I.getType();
03346   bool OutOfUndef = isa<UndefValue>(Op0);
03347 
03348   unsigned LinearIndex = ComputeLinearIndex(AggTy, I.getIndices());
03349 
03350   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
03351   SmallVector<EVT, 4> ValValueVTs;
03352   ComputeValueVTs(TLI, ValTy, ValValueVTs);
03353 
03354   unsigned NumValValues = ValValueVTs.size();
03355 
03356   // Ignore a extractvalue that produces an empty object
03357   if (!NumValValues) {
03358     setValue(&I, DAG.getUNDEF(MVT(MVT::Other)));
03359     return;
03360   }
03361 
03362   SmallVector<SDValue, 4> Values(NumValValues);
03363 
03364   SDValue Agg = getValue(Op0);
03365   // Copy out the selected value(s).
03366   for (unsigned i = LinearIndex; i != LinearIndex + NumValValues; ++i)
03367     Values[i - LinearIndex] =
03368       OutOfUndef ?
03369         DAG.getUNDEF(Agg.getNode()->getValueType(Agg.getResNo() + i)) :
03370         SDValue(Agg.getNode(), Agg.getResNo() + i);
03371 
03372   setValue(&I, DAG.getNode(ISD::MERGE_VALUES, getCurSDLoc(),
03373                            DAG.getVTList(ValValueVTs), Values));
03374 }
03375 
03376 void SelectionDAGBuilder::visitGetElementPtr(const User &I) {
03377   Value *Op0 = I.getOperand(0);
03378   // Note that the pointer operand may be a vector of pointers. Take the scalar
03379   // element which holds a pointer.
03380   Type *Ty = Op0->getType()->getScalarType();
03381   unsigned AS = Ty->getPointerAddressSpace();
03382   SDValue N = getValue(Op0);
03383 
03384   for (GetElementPtrInst::const_op_iterator OI = I.op_begin()+1, E = I.op_end();
03385        OI != E; ++OI) {
03386     const Value *Idx = *OI;
03387     if (StructType *StTy = dyn_cast<StructType>(Ty)) {
03388       unsigned Field = cast<Constant>(Idx)->getUniqueInteger().getZExtValue();
03389       if (Field) {
03390         // N = N + Offset
03391         uint64_t Offset = DL->getStructLayout(StTy)->getElementOffset(Field);
03392         N = DAG.getNode(ISD::ADD, getCurSDLoc(), N.getValueType(), N,
03393                         DAG.getConstant(Offset, N.getValueType()));
03394       }
03395 
03396       Ty = StTy->getElementType(Field);
03397     } else {
03398       Ty = cast<SequentialType>(Ty)->getElementType();
03399       MVT PtrTy = DAG.getTargetLoweringInfo().getPointerTy(AS);
03400       unsigned PtrSize = PtrTy.getSizeInBits();
03401       APInt ElementSize(PtrSize, DL->getTypeAllocSize(Ty));
03402 
03403       // If this is a constant subscript, handle it quickly.
03404       if (const auto *CI = dyn_cast<ConstantInt>(Idx)) {
03405         if (CI->isZero())
03406           continue;
03407         APInt Offs = ElementSize * CI->getValue().sextOrTrunc(PtrSize);
03408         SDValue OffsVal = DAG.getConstant(Offs, PtrTy);
03409         N = DAG.getNode(ISD::ADD, getCurSDLoc(), N.getValueType(), N, OffsVal);
03410         continue;
03411       }
03412 
03413       // N = N + Idx * ElementSize;
03414       SDValue IdxN = getValue(Idx);
03415 
03416       // If the index is smaller or larger than intptr_t, truncate or extend
03417       // it.
03418       IdxN = DAG.getSExtOrTrunc(IdxN, getCurSDLoc(), N.getValueType());
03419 
03420       // If this is a multiply by a power of two, turn it into a shl
03421       // immediately.  This is a very common case.
03422       if (ElementSize != 1) {
03423         if (ElementSize.isPowerOf2()) {
03424           unsigned Amt = ElementSize.logBase2();
03425           IdxN = DAG.getNode(ISD::SHL, getCurSDLoc(),
03426                              N.getValueType(), IdxN,
03427                              DAG.getConstant(Amt, IdxN.getValueType()));
03428         } else {
03429           SDValue Scale = DAG.getConstant(ElementSize, IdxN.getValueType());
03430           IdxN = DAG.getNode(ISD::MUL, getCurSDLoc(),
03431                              N.getValueType(), IdxN, Scale);
03432         }
03433       }
03434 
03435       N = DAG.getNode(ISD::ADD, getCurSDLoc(),
03436                       N.getValueType(), N, IdxN);
03437     }
03438   }
03439 
03440   setValue(&I, N);
03441 }
03442 
03443 void SelectionDAGBuilder::visitAlloca(const AllocaInst &I) {
03444   // If this is a fixed sized alloca in the entry block of the function,
03445   // allocate it statically on the stack.
03446   if (FuncInfo.StaticAllocaMap.count(&I))
03447     return;   // getValue will auto-populate this.
03448 
03449   Type *Ty = I.getAllocatedType();
03450   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
03451   uint64_t TySize = TLI.getDataLayout()->getTypeAllocSize(Ty);
03452   unsigned Align =
03453       std::max((unsigned)TLI.getDataLayout()->getPrefTypeAlignment(Ty),
03454                I.getAlignment());
03455 
03456   SDValue AllocSize = getValue(I.getArraySize());
03457 
03458   EVT IntPtr = TLI.getPointerTy();
03459   if (AllocSize.getValueType() != IntPtr)
03460     AllocSize = DAG.getZExtOrTrunc(AllocSize, getCurSDLoc(), IntPtr);
03461 
03462   AllocSize = DAG.getNode(ISD::MUL, getCurSDLoc(), IntPtr,
03463                           AllocSize,
03464                           DAG.getConstant(TySize, IntPtr));
03465 
03466   // Handle alignment.  If the requested alignment is less than or equal to
03467   // the stack alignment, ignore it.  If the size is greater than or equal to
03468   // the stack alignment, we note this in the DYNAMIC_STACKALLOC node.
03469   unsigned StackAlign =
03470       DAG.getSubtarget().getFrameLowering()->getStackAlignment();
03471   if (Align <= StackAlign)
03472     Align = 0;
03473 
03474   // Round the size of the allocation up to the stack alignment size
03475   // by add SA-1 to the size.
03476   AllocSize = DAG.getNode(ISD::ADD, getCurSDLoc(),
03477                           AllocSize.getValueType(), AllocSize,
03478                           DAG.getIntPtrConstant(StackAlign-1));
03479 
03480   // Mask out the low bits for alignment purposes.
03481   AllocSize = DAG.getNode(ISD::AND, getCurSDLoc(),
03482                           AllocSize.getValueType(), AllocSize,
03483                           DAG.getIntPtrConstant(~(uint64_t)(StackAlign-1)));
03484 
03485   SDValue Ops[] = { getRoot(), AllocSize, DAG.getIntPtrConstant(Align) };
03486   SDVTList VTs = DAG.getVTList(AllocSize.getValueType(), MVT::Other);
03487   SDValue DSA = DAG.getNode(ISD::DYNAMIC_STACKALLOC, getCurSDLoc(), VTs, Ops);
03488   setValue(&I, DSA);
03489   DAG.setRoot(DSA.getValue(1));
03490 
03491   assert(FuncInfo.MF->getFrameInfo()->hasVarSizedObjects());
03492 }
03493 
03494 void SelectionDAGBuilder::visitLoad(const LoadInst &I) {
03495   if (I.isAtomic())
03496     return visitAtomicLoad(I);
03497 
03498   const Value *SV = I.getOperand(0);
03499   SDValue Ptr = getValue(SV);
03500 
03501   Type *Ty = I.getType();
03502 
03503   bool isVolatile = I.isVolatile();
03504   bool isNonTemporal = I.getMetadata(LLVMContext::MD_nontemporal) != nullptr;
03505   bool isInvariant = I.getMetadata(LLVMContext::MD_invariant_load) != nullptr;
03506   unsigned Alignment = I.getAlignment();
03507 
03508   AAMDNodes AAInfo;
03509   I.getAAMetadata(AAInfo);
03510   const MDNode *Ranges = I.getMetadata(LLVMContext::MD_range);
03511 
03512   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
03513   SmallVector<EVT, 4> ValueVTs;
03514   SmallVector<uint64_t, 4> Offsets;
03515   ComputeValueVTs(TLI, Ty, ValueVTs, &Offsets);
03516   unsigned NumValues = ValueVTs.size();
03517   if (NumValues == 0)
03518     return;
03519 
03520   SDValue Root;
03521   bool ConstantMemory = false;
03522   if (isVolatile || NumValues > MaxParallelChains)
03523     // Serialize volatile loads with other side effects.
03524     Root = getRoot();
03525   else if (AA->pointsToConstantMemory(
03526              AliasAnalysis::Location(SV, AA->getTypeStoreSize(Ty), AAInfo))) {
03527     // Do not serialize (non-volatile) loads of constant memory with anything.
03528     Root = DAG.getEntryNode();
03529     ConstantMemory = true;
03530   } else {
03531     // Do not serialize non-volatile loads against each other.
03532     Root = DAG.getRoot();
03533   }
03534 
03535   if (isVolatile)
03536     Root = TLI.prepareVolatileOrAtomicLoad(Root, getCurSDLoc(), DAG);
03537 
03538   SmallVector<SDValue, 4> Values(NumValues);
03539   SmallVector<SDValue, 4> Chains(std::min(unsigned(MaxParallelChains),
03540                                           NumValues));
03541   EVT PtrVT = Ptr.getValueType();
03542   unsigned ChainI = 0;
03543   for (unsigned i = 0; i != NumValues; ++i, ++ChainI) {
03544     // Serializing loads here may result in excessive register pressure, and
03545     // TokenFactor places arbitrary choke points on the scheduler. SD scheduling
03546     // could recover a bit by hoisting nodes upward in the chain by recognizing
03547     // they are side-effect free or do not alias. The optimizer should really
03548     // avoid this case by converting large object/array copies to llvm.memcpy
03549     // (MaxParallelChains should always remain as failsafe).
03550     if (ChainI == MaxParallelChains) {
03551       assert(PendingLoads.empty() && "PendingLoads must be serialized first");
03552       SDValue Chain = DAG.getNode(ISD::TokenFactor, getCurSDLoc(), MVT::Other,
03553                                   makeArrayRef(Chains.data(), ChainI));
03554       Root = Chain;
03555       ChainI = 0;
03556     }
03557     SDValue A = DAG.getNode(ISD::ADD, getCurSDLoc(),
03558                             PtrVT, Ptr,
03559                             DAG.getConstant(Offsets[i], PtrVT));
03560     SDValue L = DAG.getLoad(ValueVTs[i], getCurSDLoc(), Root,
03561                             A, MachinePointerInfo(SV, Offsets[i]), isVolatile,
03562                             isNonTemporal, isInvariant, Alignment, AAInfo,
03563                             Ranges);
03564 
03565     Values[i] = L;
03566     Chains[ChainI] = L.getValue(1);
03567   }
03568 
03569   if (!ConstantMemory) {
03570     SDValue Chain = DAG.getNode(ISD::TokenFactor, getCurSDLoc(), MVT::Other,
03571                                 makeArrayRef(Chains.data(), ChainI));
03572     if (isVolatile)
03573       DAG.setRoot(Chain);
03574     else
03575       PendingLoads.push_back(Chain);
03576   }
03577 
03578   setValue(&I, DAG.getNode(ISD::MERGE_VALUES, getCurSDLoc(),
03579                            DAG.getVTList(ValueVTs), Values));
03580 }
03581 
03582 void SelectionDAGBuilder::visitStore(const StoreInst &I) {
03583   if (I.isAtomic())
03584     return visitAtomicStore(I);
03585 
03586   const Value *SrcV = I.getOperand(0);
03587   const Value *PtrV = I.getOperand(1);
03588 
03589   SmallVector<EVT, 4> ValueVTs;
03590   SmallVector<uint64_t, 4> Offsets;
03591   ComputeValueVTs(DAG.getTargetLoweringInfo(), SrcV->getType(),
03592                   ValueVTs, &Offsets);
03593   unsigned NumValues = ValueVTs.size();
03594   if (NumValues == 0)
03595     return;
03596 
03597   // Get the lowered operands. Note that we do this after
03598   // checking if NumResults is zero, because with zero results
03599   // the operands won't have values in the map.
03600   SDValue Src = getValue(SrcV);
03601   SDValue Ptr = getValue(PtrV);
03602 
03603   SDValue Root = getRoot();
03604   SmallVector<SDValue, 4> Chains(std::min(unsigned(MaxParallelChains),
03605                                           NumValues));
03606   EVT PtrVT = Ptr.getValueType();
03607   bool isVolatile = I.isVolatile();
03608   bool isNonTemporal = I.getMetadata(LLVMContext::MD_nontemporal) != nullptr;
03609   unsigned Alignment = I.getAlignment();
03610 
03611   AAMDNodes AAInfo;
03612   I.getAAMetadata(AAInfo);
03613 
03614   unsigned ChainI = 0;
03615   for (unsigned i = 0; i != NumValues; ++i, ++ChainI) {
03616     // See visitLoad comments.
03617     if (ChainI == MaxParallelChains) {
03618       SDValue Chain = DAG.getNode(ISD::TokenFactor, getCurSDLoc(), MVT::Other,
03619                                   makeArrayRef(Chains.data(), ChainI));
03620       Root = Chain;
03621       ChainI = 0;
03622     }
03623     SDValue Add = DAG.getNode(ISD::ADD, getCurSDLoc(), PtrVT, Ptr,
03624                               DAG.getConstant(Offsets[i], PtrVT));
03625     SDValue St = DAG.getStore(Root, getCurSDLoc(),
03626                               SDValue(Src.getNode(), Src.getResNo() + i),
03627                               Add, MachinePointerInfo(PtrV, Offsets[i]),
03628                               isVolatile, isNonTemporal, Alignment, AAInfo);
03629     Chains[ChainI] = St;
03630   }
03631 
03632   SDValue StoreNode = DAG.getNode(ISD::TokenFactor, getCurSDLoc(), MVT::Other,
03633                                   makeArrayRef(Chains.data(), ChainI));
03634   DAG.setRoot(StoreNode);
03635 }
03636 
03637 void SelectionDAGBuilder::visitMaskedStore(const CallInst &I) {
03638   SDLoc sdl = getCurSDLoc();
03639 
03640   // llvm.masked.store.*(Src0, Ptr, alignemt, Mask)
03641   Value  *PtrOperand = I.getArgOperand(1);
03642   SDValue Ptr = getValue(PtrOperand);
03643   SDValue Src0 = getValue(I.getArgOperand(0));
03644   SDValue Mask = getValue(I.getArgOperand(3));
03645   EVT VT = Src0.getValueType();
03646   unsigned Alignment = (cast<ConstantInt>(I.getArgOperand(2)))->getZExtValue();
03647   if (!Alignment)
03648     Alignment = DAG.getEVTAlignment(VT);
03649 
03650   AAMDNodes AAInfo;
03651   I.getAAMetadata(AAInfo);
03652 
03653   MachineMemOperand *MMO =
03654     DAG.getMachineFunction().
03655     getMachineMemOperand(MachinePointerInfo(PtrOperand),
03656                           MachineMemOperand::MOStore,  VT.getStoreSize(),
03657                           Alignment, AAInfo);
03658   SDValue StoreNode = DAG.getMaskedStore(getRoot(), sdl, Src0, Ptr, Mask, VT,
03659                                          MMO, false);
03660   DAG.setRoot(StoreNode);
03661   setValue(&I, StoreNode);
03662 }
03663 
03664 void SelectionDAGBuilder::visitMaskedLoad(const CallInst &I) {
03665   SDLoc sdl = getCurSDLoc();
03666 
03667   // @llvm.masked.load.*(Ptr, alignment, Mask, Src0)
03668   Value  *PtrOperand = I.getArgOperand(0);
03669   SDValue Ptr = getValue(PtrOperand);
03670   SDValue Src0 = getValue(I.getArgOperand(3));
03671   SDValue Mask = getValue(I.getArgOperand(2));
03672 
03673   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
03674   EVT VT = TLI.getValueType(I.getType());
03675   unsigned Alignment = (cast<ConstantInt>(I.getArgOperand(1)))->getZExtValue();
03676   if (!Alignment)
03677     Alignment = DAG.getEVTAlignment(VT);
03678 
03679   AAMDNodes AAInfo;
03680   I.getAAMetadata(AAInfo);
03681   const MDNode *Ranges = I.getMetadata(LLVMContext::MD_range);
03682 
03683   SDValue InChain = DAG.getRoot();
03684   if (AA->pointsToConstantMemory(
03685       AliasAnalysis::Location(PtrOperand,
03686                               AA->getTypeStoreSize(I.getType()),
03687                               AAInfo))) {
03688     // Do not serialize (non-volatile) loads of constant memory with anything.
03689     InChain = DAG.getEntryNode();
03690   }
03691 
03692   MachineMemOperand *MMO =
03693     DAG.getMachineFunction().
03694     getMachineMemOperand(MachinePointerInfo(PtrOperand),
03695                           MachineMemOperand::MOLoad,  VT.getStoreSize(),
03696                           Alignment, AAInfo, Ranges);
03697 
03698   SDValue Load = DAG.getMaskedLoad(VT, sdl, InChain, Ptr, Mask, Src0, VT, MMO,
03699                                    ISD::NON_EXTLOAD);
03700   SDValue OutChain = Load.getValue(1);
03701   DAG.setRoot(OutChain);
03702   setValue(&I, Load);
03703 }
03704 
03705 void SelectionDAGBuilder::visitAtomicCmpXchg(const AtomicCmpXchgInst &I) {
03706   SDLoc dl = getCurSDLoc();
03707   AtomicOrdering SuccessOrder = I.getSuccessOrdering();
03708   AtomicOrdering FailureOrder = I.getFailureOrdering();
03709   SynchronizationScope Scope = I.getSynchScope();
03710 
03711   SDValue InChain = getRoot();
03712 
03713   MVT MemVT = getValue(I.getCompareOperand()).getSimpleValueType();
03714   SDVTList VTs = DAG.getVTList(MemVT, MVT::i1, MVT::Other);
03715   SDValue L = DAG.getAtomicCmpSwap(
03716       ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS, dl, MemVT, VTs, InChain,
03717       getValue(I.getPointerOperand()), getValue(I.getCompareOperand()),
03718       getValue(I.getNewValOperand()), MachinePointerInfo(I.getPointerOperand()),
03719       /*Alignment=*/ 0, SuccessOrder, FailureOrder, Scope);
03720 
03721   SDValue OutChain = L.getValue(2);
03722 
03723   setValue(&I, L);
03724   DAG.setRoot(OutChain);
03725 }
03726 
03727 void SelectionDAGBuilder::visitAtomicRMW(const AtomicRMWInst &I) {
03728   SDLoc dl = getCurSDLoc();
03729   ISD::NodeType NT;
03730   switch (I.getOperation()) {
03731   default: llvm_unreachable("Unknown atomicrmw operation");
03732   case AtomicRMWInst::Xchg: NT = ISD::ATOMIC_SWAP; break;
03733   case AtomicRMWInst::Add:  NT = ISD::ATOMIC_LOAD_ADD; break;
03734   case AtomicRMWInst::Sub:  NT = ISD::ATOMIC_LOAD_SUB; break;
03735   case AtomicRMWInst::And:  NT = ISD::ATOMIC_LOAD_AND; break;
03736   case AtomicRMWInst::Nand: NT = ISD::ATOMIC_LOAD_NAND; break;
03737   case AtomicRMWInst::Or:   NT = ISD::ATOMIC_LOAD_OR; break;
03738   case AtomicRMWInst::Xor:  NT = ISD::ATOMIC_LOAD_XOR; break;
03739   case AtomicRMWInst::Max:  NT = ISD::ATOMIC_LOAD_MAX; break;
03740   case AtomicRMWInst::Min:  NT = ISD::ATOMIC_LOAD_MIN; break;
03741   case AtomicRMWInst::UMax: NT = ISD::ATOMIC_LOAD_UMAX; break;
03742   case AtomicRMWInst::UMin: NT = ISD::ATOMIC_LOAD_UMIN; break;
03743   }
03744   AtomicOrdering Order = I.getOrdering();
03745   SynchronizationScope Scope = I.getSynchScope();
03746 
03747   SDValue InChain = getRoot();
03748 
03749   SDValue L =
03750     DAG.getAtomic(NT, dl,
03751                   getValue(I.getValOperand()).getSimpleValueType(),
03752                   InChain,
03753                   getValue(I.getPointerOperand()),
03754                   getValue(I.getValOperand()),
03755                   I.getPointerOperand(),
03756                   /* Alignment=*/ 0, Order, Scope);
03757 
03758   SDValue OutChain = L.getValue(1);
03759 
03760   setValue(&I, L);
03761   DAG.setRoot(OutChain);
03762 }
03763 
03764 void SelectionDAGBuilder::visitFence(const FenceInst &I) {
03765   SDLoc dl = getCurSDLoc();
03766   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
03767   SDValue Ops[3];
03768   Ops[0] = getRoot();
03769   Ops[1] = DAG.getConstant(I.getOrdering(), TLI.getPointerTy());
03770   Ops[2] = DAG.getConstant(I.getSynchScope(), TLI.getPointerTy());
03771   DAG.setRoot(DAG.getNode(ISD::ATOMIC_FENCE, dl, MVT::Other, Ops));
03772 }
03773 
03774 void SelectionDAGBuilder::visitAtomicLoad(const LoadInst &I) {
03775   SDLoc dl = getCurSDLoc();
03776   AtomicOrdering Order = I.getOrdering();
03777   SynchronizationScope Scope = I.getSynchScope();
03778 
03779   SDValue InChain = getRoot();
03780 
03781   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
03782   EVT VT = TLI.getValueType(I.getType());
03783 
03784   if (I.getAlignment() < VT.getSizeInBits() / 8)
03785     report_fatal_error("Cannot generate unaligned atomic load");
03786 
03787   MachineMemOperand *MMO =
03788       DAG.getMachineFunction().
03789       getMachineMemOperand(MachinePointerInfo(I.getPointerOperand()),
03790                            MachineMemOperand::MOVolatile |
03791                            MachineMemOperand::MOLoad,
03792                            VT.getStoreSize(),
03793                            I.getAlignment() ? I.getAlignment() :
03794                                               DAG.getEVTAlignment(VT));
03795 
03796   InChain = TLI.prepareVolatileOrAtomicLoad(InChain, dl, DAG);
03797   SDValue L =
03798       DAG.getAtomic(ISD::ATOMIC_LOAD, dl, VT, VT, InChain,
03799                     getValue(I.getPointerOperand()), MMO,
03800                     Order, Scope);
03801 
03802   SDValue OutChain = L.getValue(1);
03803 
03804   setValue(&I, L);
03805   DAG.setRoot(OutChain);
03806 }
03807 
03808 void SelectionDAGBuilder::visitAtomicStore(const StoreInst &I) {
03809   SDLoc dl = getCurSDLoc();
03810 
03811   AtomicOrdering Order = I.getOrdering();
03812   SynchronizationScope Scope = I.getSynchScope();
03813 
03814   SDValue InChain = getRoot();
03815 
03816   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
03817   EVT VT = TLI.getValueType(I.getValueOperand()->getType());
03818 
03819   if (I.getAlignment() < VT.getSizeInBits() / 8)
03820     report_fatal_error("Cannot generate unaligned atomic store");
03821 
03822   SDValue OutChain =
03823     DAG.getAtomic(ISD::ATOMIC_STORE, dl, VT,
03824                   InChain,
03825                   getValue(I.getPointerOperand()),
03826                   getValue(I.getValueOperand()),
03827                   I.getPointerOperand(), I.getAlignment(),
03828                   Order, Scope);
03829 
03830   DAG.setRoot(OutChain);
03831 }
03832 
03833 /// visitTargetIntrinsic - Lower a call of a target intrinsic to an INTRINSIC
03834 /// node.
03835 void SelectionDAGBuilder::visitTargetIntrinsic(const CallInst &I,
03836                                                unsigned Intrinsic) {
03837   bool HasChain = !I.doesNotAccessMemory();
03838   bool OnlyLoad = HasChain && I.onlyReadsMemory();
03839 
03840   // Build the operand list.
03841   SmallVector<SDValue, 8> Ops;
03842   if (HasChain) {  // If this intrinsic has side-effects, chainify it.
03843     if (OnlyLoad) {
03844       // We don't need to serialize loads against other loads.
03845       Ops.push_back(DAG.getRoot());
03846     } else {
03847       Ops.push_back(getRoot());
03848     }
03849   }
03850 
03851   // Info is set by getTgtMemInstrinsic
03852   TargetLowering::IntrinsicInfo Info;
03853   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
03854   bool IsTgtIntrinsic = TLI.getTgtMemIntrinsic(Info, I, Intrinsic);
03855 
03856   // Add the intrinsic ID as an integer operand if it's not a target intrinsic.
03857   if (!IsTgtIntrinsic || Info.opc == ISD::INTRINSIC_VOID ||
03858       Info.opc == ISD::INTRINSIC_W_CHAIN)
03859     Ops.push_back(DAG.getTargetConstant(Intrinsic, TLI.getPointerTy()));
03860 
03861   // Add all operands of the call to the operand list.
03862   for (unsigned i = 0, e = I.getNumArgOperands(); i != e; ++i) {
03863     SDValue Op = getValue(I.getArgOperand(i));
03864     Ops.push_back(Op);
03865   }
03866 
03867   SmallVector<EVT, 4> ValueVTs;
03868   ComputeValueVTs(TLI, I.getType(), ValueVTs);
03869 
03870   if (HasChain)
03871     ValueVTs.push_back(MVT::Other);
03872 
03873   SDVTList VTs = DAG.getVTList(ValueVTs);
03874 
03875   // Create the node.
03876   SDValue Result;
03877   if (IsTgtIntrinsic) {
03878     // This is target intrinsic that touches memory
03879     Result = DAG.getMemIntrinsicNode(Info.opc, getCurSDLoc(),
03880                                      VTs, Ops, Info.memVT,
03881                                    MachinePointerInfo(Info.ptrVal, Info.offset),
03882                                      Info.align, Info.vol,
03883                                      Info.readMem, Info.writeMem, Info.size);
03884   } else if (!HasChain) {
03885     Result = DAG.getNode(ISD::INTRINSIC_WO_CHAIN, getCurSDLoc(), VTs, Ops);
03886   } else if (!I.getType()->isVoidTy()) {
03887     Result = DAG.getNode(ISD::INTRINSIC_W_CHAIN, getCurSDLoc(), VTs, Ops);
03888   } else {
03889     Result = DAG.getNode(ISD::INTRINSIC_VOID, getCurSDLoc(), VTs, Ops);
03890   }
03891 
03892   if (HasChain) {
03893     SDValue Chain = Result.getValue(Result.getNode()->getNumValues()-1);
03894     if (OnlyLoad)
03895       PendingLoads.push_back(Chain);
03896     else
03897       DAG.setRoot(Chain);
03898   }
03899 
03900   if (!I.getType()->isVoidTy()) {
03901     if (VectorType *PTy = dyn_cast<VectorType>(I.getType())) {
03902       EVT VT = TLI.getValueType(PTy);
03903       Result = DAG.getNode(ISD::BITCAST, getCurSDLoc(), VT, Result);
03904     }
03905 
03906     setValue(&I, Result);
03907   }
03908 }
03909 
03910 /// GetSignificand - Get the significand and build it into a floating-point
03911 /// number with exponent of 1:
03912 ///
03913 ///   Op = (Op & 0x007fffff) | 0x3f800000;
03914 ///
03915 /// where Op is the hexadecimal representation of floating point value.
03916 static SDValue
03917 GetSignificand(SelectionDAG &DAG, SDValue Op, SDLoc dl) {
03918   SDValue t1 = DAG.getNode(ISD::AND, dl, MVT::i32, Op,
03919                            DAG.getConstant(0x007fffff, MVT::i32));
03920   SDValue t2 = DAG.getNode(ISD::OR, dl, MVT::i32, t1,
03921                            DAG.getConstant(0x3f800000, MVT::i32));
03922   return DAG.getNode(ISD::BITCAST, dl, MVT::f32, t2);
03923 }
03924 
03925 /// GetExponent - Get the exponent:
03926 ///
03927 ///   (float)(int)(((Op & 0x7f800000) >> 23) - 127);
03928 ///
03929 /// where Op is the hexadecimal representation of floating point value.
03930 static SDValue
03931 GetExponent(SelectionDAG &DAG, SDValue Op, const TargetLowering &TLI,
03932             SDLoc dl) {
03933   SDValue t0 = DAG.getNode(ISD::AND, dl, MVT::i32, Op,
03934                            DAG.getConstant(0x7f800000, MVT::i32));
03935   SDValue t1 = DAG.getNode(ISD::SRL, dl, MVT::i32, t0,
03936                            DAG.getConstant(23, TLI.getPointerTy()));
03937   SDValue t2 = DAG.getNode(ISD::SUB, dl, MVT::i32, t1,
03938                            DAG.getConstant(127, MVT::i32));
03939   return DAG.getNode(ISD::SINT_TO_FP, dl, MVT::f32, t2);
03940 }
03941 
03942 /// getF32Constant - Get 32-bit floating point constant.
03943 static SDValue
03944 getF32Constant(SelectionDAG &DAG, unsigned Flt) {
03945   return DAG.getConstantFP(APFloat(APFloat::IEEEsingle, APInt(32, Flt)),
03946                            MVT::f32);
03947 }
03948 
03949 static SDValue getLimitedPrecisionExp2(SDValue t0, SDLoc dl,
03950                                        SelectionDAG &DAG) {
03951   //   IntegerPartOfX = ((int32_t)(t0);
03952   SDValue IntegerPartOfX = DAG.getNode(ISD::FP_TO_SINT, dl, MVT::i32, t0);
03953 
03954   //   FractionalPartOfX = t0 - (float)IntegerPartOfX;
03955   SDValue t1 = DAG.getNode(ISD::SINT_TO_FP, dl, MVT::f32, IntegerPartOfX);
03956   SDValue X = DAG.getNode(ISD::FSUB, dl, MVT::f32, t0, t1);
03957 
03958   //   IntegerPartOfX <<= 23;
03959   IntegerPartOfX = DAG.getNode(
03960       ISD::SHL, dl, MVT::i32, IntegerPartOfX,
03961       DAG.getConstant(23, DAG.getTargetLoweringInfo().getPointerTy()));
03962 
03963   SDValue TwoToFractionalPartOfX;
03964   if (LimitFloatPrecision <= 6) {
03965     // For floating-point precision of 6:
03966     //
03967     //   TwoToFractionalPartOfX =
03968     //     0.997535578f +
03969     //       (0.735607626f + 0.252464424f * x) * x;
03970     //
03971     // error 0.0144103317, which is 6 bits
03972     SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X,
03973                              getF32Constant(DAG, 0x3e814304));
03974     SDValue t3 = DAG.getNode(ISD::FADD, dl, MVT::f32, t2,
03975                              getF32Constant(DAG, 0x3f3c50c8));
03976     SDValue t4 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t3, X);
03977     TwoToFractionalPartOfX = DAG.getNode(ISD::FADD, dl, MVT::f32, t4,
03978                                          getF32Constant(DAG, 0x3f7f5e7e));
03979   } else if (LimitFloatPrecision <= 12) {
03980     // For floating-point precision of 12:
03981     //
03982     //   TwoToFractionalPartOfX =
03983     //     0.999892986f +
03984     //       (0.696457318f +
03985     //         (0.224338339f + 0.792043434e-1f * x) * x) * x;
03986     //
03987     // error 0.000107046256, which is 13 to 14 bits
03988     SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X,
03989                              getF32Constant(DAG, 0x3da235e3));
03990     SDValue t3 = DAG.getNode(ISD::FADD, dl, MVT::f32, t2,
03991                              getF32Constant(DAG, 0x3e65b8f3));
03992     SDValue t4 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t3, X);
03993     SDValue t5 = DAG.getNode(ISD::FADD, dl, MVT::f32, t4,
03994                              getF32Constant(DAG, 0x3f324b07));
03995     SDValue t6 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t5, X);
03996     TwoToFractionalPartOfX = DAG.getNode(ISD::FADD, dl, MVT::f32, t6,
03997                                          getF32Constant(DAG, 0x3f7ff8fd));
03998   } else { // LimitFloatPrecision <= 18
03999     // For floating-point precision of 18:
04000     //
04001     //   TwoToFractionalPartOfX =
04002     //     0.999999982f +
04003     //       (0.693148872f +
04004     //         (0.240227044f +
04005     //           (0.554906021e-1f +
04006     //             (0.961591928e-2f +
04007     //               (0.136028312e-2f + 0.157059148e-3f *x)*x)*x)*x)*x)*x;
04008     // error 2.47208000*10^(-7), which is better than 18 bits
04009     SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X,
04010                              getF32Constant(DAG, 0x3924b03e));
04011     SDValue t3 = DAG.getNode(ISD::FADD, dl, MVT::f32, t2,
04012                              getF32Constant(DAG, 0x3ab24b87));
04013     SDValue t4 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t3, X);
04014     SDValue t5 = DAG.getNode(ISD::FADD, dl, MVT::f32, t4,
04015                              getF32Constant(DAG, 0x3c1d8c17));
04016     SDValue t6 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t5, X);
04017     SDValue t7 = DAG.getNode(ISD::FADD, dl, MVT::f32, t6,
04018                              getF32Constant(DAG, 0x3d634a1d));
04019     SDValue t8 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t7, X);
04020     SDValue t9 = DAG.getNode(ISD::FADD, dl, MVT::f32, t8,
04021                              getF32Constant(DAG, 0x3e75fe14));
04022     SDValue t10 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t9, X);
04023     SDValue t11 = DAG.getNode(ISD::FADD, dl, MVT::f32, t10,
04024                               getF32Constant(DAG, 0x3f317234));
04025     SDValue t12 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t11, X);
04026     TwoToFractionalPartOfX = DAG.getNode(ISD::FADD, dl, MVT::f32, t12,
04027                                          getF32Constant(DAG, 0x3f800000));
04028   }
04029 
04030   // Add the exponent into the result in integer domain.
04031   SDValue t13 = DAG.getNode(ISD::BITCAST, dl, MVT::i32, TwoToFractionalPartOfX);
04032   return DAG.getNode(ISD::BITCAST, dl, MVT::f32,
04033                      DAG.getNode(ISD::ADD, dl, MVT::i32, t13, IntegerPartOfX));
04034 }
04035 
04036 /// expandExp - Lower an exp intrinsic. Handles the special sequences for
04037 /// limited-precision mode.
04038 static SDValue expandExp(SDLoc dl, SDValue Op, SelectionDAG &DAG,
04039                          const TargetLowering &TLI) {
04040   if (Op.getValueType() == MVT::f32 &&
04041       LimitFloatPrecision > 0 && LimitFloatPrecision <= 18) {
04042 
04043     // Put the exponent in the right bit position for later addition to the
04044     // final result:
04045     //
04046     //   #define LOG2OFe 1.4426950f
04047     //   t0 = Op * LOG2OFe
04048     SDValue t0 = DAG.getNode(ISD::FMUL, dl, MVT::f32, Op,
04049                              getF32Constant(DAG, 0x3fb8aa3b));
04050     return getLimitedPrecisionExp2(t0, dl, DAG);
04051   }
04052 
04053   // No special expansion.
04054   return DAG.getNode(ISD::FEXP, dl, Op.getValueType(), Op);
04055 }
04056 
04057 /// expandLog - Lower a log intrinsic. Handles the special sequences for
04058 /// limited-precision mode.
04059 static SDValue expandLog(SDLoc dl, SDValue Op, SelectionDAG &DAG,
04060                          const TargetLowering &TLI) {
04061   if (Op.getValueType() == MVT::f32 &&
04062       LimitFloatPrecision > 0 && LimitFloatPrecision <= 18) {
04063     SDValue Op1 = DAG.getNode(ISD::BITCAST, dl, MVT::i32, Op);
04064 
04065     // Scale the exponent by log(2) [0.69314718f].
04066     SDValue Exp = GetExponent(DAG, Op1, TLI, dl);
04067     SDValue LogOfExponent = DAG.getNode(ISD::FMUL, dl, MVT::f32, Exp,
04068                                         getF32Constant(DAG, 0x3f317218));
04069 
04070     // Get the significand and build it into a floating-point number with
04071     // exponent of 1.
04072     SDValue X = GetSignificand(DAG, Op1, dl);
04073 
04074     SDValue LogOfMantissa;
04075     if (LimitFloatPrecision <= 6) {
04076       // For floating-point precision of 6:
04077       //
04078       //   LogofMantissa =
04079       //     -1.1609546f +
04080       //       (1.4034025f - 0.23903021f * x) * x;
04081       //
04082       // error 0.0034276066, which is better than 8 bits
04083       SDValue t0 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X,
04084                                getF32Constant(DAG, 0xbe74c456));
04085       SDValue t1 = DAG.getNode(ISD::FADD, dl, MVT::f32, t0,
04086                                getF32Constant(DAG, 0x3fb3a2b1));
04087       SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t1, X);
04088       LogOfMantissa = DAG.getNode(ISD::FSUB, dl, MVT::f32, t2,
04089                                   getF32Constant(DAG, 0x3f949a29));
04090     } else if (LimitFloatPrecision <= 12) {
04091       // For floating-point precision of 12:
04092       //
04093       //   LogOfMantissa =
04094       //     -1.7417939f +
04095       //       (2.8212026f +
04096       //         (-1.4699568f +
04097       //           (0.44717955f - 0.56570851e-1f * x) * x) * x) * x;
04098       //
04099       // error 0.000061011436, which is 14 bits
04100       SDValue t0 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X,
04101                                getF32Constant(DAG, 0xbd67b6d6));
04102       SDValue t1 = DAG.getNode(ISD::FADD, dl, MVT::f32, t0,
04103                                getF32Constant(DAG, 0x3ee4f4b8));
04104       SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t1, X);
04105       SDValue t3 = DAG.getNode(ISD::FSUB, dl, MVT::f32, t2,
04106                                getF32Constant(DAG, 0x3fbc278b));
04107       SDValue t4 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t3, X);
04108       SDValue t5 = DAG.getNode(ISD::FADD, dl, MVT::f32, t4,
04109                                getF32Constant(DAG, 0x40348e95));
04110       SDValue t6 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t5, X);
04111       LogOfMantissa = DAG.getNode(ISD::FSUB, dl, MVT::f32, t6,
04112                                   getF32Constant(DAG, 0x3fdef31a));
04113     } else { // LimitFloatPrecision <= 18
04114       // For floating-point precision of 18:
04115       //
04116       //   LogOfMantissa =
04117       //     -2.1072184f +
04118       //       (4.2372794f +
04119       //         (-3.7029485f +
04120       //           (2.2781945f +
04121       //             (-0.87823314f +
04122       //               (0.19073739f - 0.17809712e-1f * x) * x) * x) * x) * x)*x;
04123       //
04124       // error 0.0000023660568, which is better than 18 bits
04125       SDValue t0 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X,
04126                                getF32Constant(DAG, 0xbc91e5ac));
04127       SDValue t1 = DAG.getNode(ISD::FADD, dl, MVT::f32, t0,
04128                                getF32Constant(DAG, 0x3e4350aa));
04129       SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t1, X);
04130       SDValue t3 = DAG.getNode(ISD::FSUB, dl, MVT::f32, t2,
04131                                getF32Constant(DAG, 0x3f60d3e3));
04132       SDValue t4 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t3, X);
04133       SDValue t5 = DAG.getNode(ISD::FADD, dl, MVT::f32, t4,
04134                                getF32Constant(DAG, 0x4011cdf0));
04135       SDValue t6 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t5, X);
04136       SDValue t7 = DAG.getNode(ISD::FSUB, dl, MVT::f32, t6,
04137                                getF32Constant(DAG, 0x406cfd1c));
04138       SDValue t8 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t7, X);
04139       SDValue t9 = DAG.getNode(ISD::FADD, dl, MVT::f32, t8,
04140                                getF32Constant(DAG, 0x408797cb));
04141       SDValue t10 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t9, X);
04142       LogOfMantissa = DAG.getNode(ISD::FSUB, dl, MVT::f32, t10,
04143                                   getF32Constant(DAG, 0x4006dcab));
04144     }
04145 
04146     return DAG.getNode(ISD::FADD, dl, MVT::f32, LogOfExponent, LogOfMantissa);
04147   }
04148 
04149   // No special expansion.
04150   return DAG.getNode(ISD::FLOG, dl, Op.getValueType(), Op);
04151 }
04152 
04153 /// expandLog2 - Lower a log2 intrinsic. Handles the special sequences for
04154 /// limited-precision mode.
04155 static SDValue expandLog2(SDLoc dl, SDValue Op, SelectionDAG &DAG,
04156                           const TargetLowering &TLI) {
04157   if (Op.getValueType() == MVT::f32 &&
04158       LimitFloatPrecision > 0 && LimitFloatPrecision <= 18) {
04159     SDValue Op1 = DAG.getNode(ISD::BITCAST, dl, MVT::i32, Op);
04160 
04161     // Get the exponent.
04162     SDValue LogOfExponent = GetExponent(DAG, Op1, TLI, dl);
04163 
04164     // Get the significand and build it into a floating-point number with
04165     // exponent of 1.
04166     SDValue X = GetSignificand(DAG, Op1, dl);
04167 
04168     // Different possible minimax approximations of significand in
04169     // floating-point for various degrees of accuracy over [1,2].
04170     SDValue Log2ofMantissa;
04171     if (LimitFloatPrecision <= 6) {
04172       // For floating-point precision of 6:
04173       //
04174       //   Log2ofMantissa = -1.6749035f + (2.0246817f - .34484768f * x) * x;
04175       //
04176       // error 0.0049451742, which is more than 7 bits
04177       SDValue t0 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X,
04178                                getF32Constant(DAG, 0xbeb08fe0));
04179       SDValue t1 = DAG.getNode(ISD::FADD, dl, MVT::f32, t0,
04180                                getF32Constant(DAG, 0x40019463));
04181       SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t1, X);
04182       Log2ofMantissa = DAG.getNode(ISD::FSUB, dl, MVT::f32, t2,
04183                                    getF32Constant(DAG, 0x3fd6633d));
04184     } else if (LimitFloatPrecision <= 12) {
04185       // For floating-point precision of 12:
04186       //
04187       //   Log2ofMantissa =
04188       //     -2.51285454f +
04189       //       (4.07009056f +
04190       //         (-2.12067489f +
04191       //           (.645142248f - 0.816157886e-1f * x) * x) * x) * x;
04192       //
04193       // error 0.0000876136000, which is better than 13 bits
04194       SDValue t0 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X,
04195                                getF32Constant(DAG, 0xbda7262e));
04196       SDValue t1 = DAG.getNode(ISD::FADD, dl, MVT::f32, t0,
04197                                getF32Constant(DAG, 0x3f25280b));
04198       SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t1, X);
04199       SDValue t3 = DAG.getNode(ISD::FSUB, dl, MVT::f32, t2,
04200                                getF32Constant(DAG, 0x4007b923));
04201       SDValue t4 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t3, X);
04202       SDValue t5 = DAG.getNode(ISD::FADD, dl, MVT::f32, t4,
04203                                getF32Constant(DAG, 0x40823e2f));
04204       SDValue t6 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t5, X);
04205       Log2ofMantissa = DAG.getNode(ISD::FSUB, dl, MVT::f32, t6,
04206                                    getF32Constant(DAG, 0x4020d29c));
04207     } else { // LimitFloatPrecision <= 18
04208       // For floating-point precision of 18:
04209       //
04210       //   Log2ofMantissa =
04211       //     -3.0400495f +
04212       //       (6.1129976f +
04213       //         (-5.3420409f +
04214       //           (3.2865683f +
04215       //             (-1.2669343f +
04216       //               (0.27515199f -
04217       //                 0.25691327e-1f * x) * x) * x) * x) * x) * x;
04218       //
04219       // error 0.0000018516, which is better than 18 bits
04220       SDValue t0 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X,
04221                                getF32Constant(DAG, 0xbcd2769e));
04222       SDValue t1 = DAG.getNode(ISD::FADD, dl, MVT::f32, t0,
04223                                getF32Constant(DAG, 0x3e8ce0b9));
04224       SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t1, X);
04225       SDValue t3 = DAG.getNode(ISD::FSUB, dl, MVT::f32, t2,
04226                                getF32Constant(DAG, 0x3fa22ae7));
04227       SDValue t4 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t3, X);
04228       SDValue t5 = DAG.getNode(ISD::FADD, dl, MVT::f32, t4,
04229                                getF32Constant(DAG, 0x40525723));
04230       SDValue t6 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t5, X);
04231       SDValue t7 = DAG.getNode(ISD::FSUB, dl, MVT::f32, t6,
04232                                getF32Constant(DAG, 0x40aaf200));
04233       SDValue t8 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t7, X);
04234       SDValue t9 = DAG.getNode(ISD::FADD, dl, MVT::f32, t8,
04235                                getF32Constant(DAG, 0x40c39dad));
04236       SDValue t10 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t9, X);
04237       Log2ofMantissa = DAG.getNode(ISD::FSUB, dl, MVT::f32, t10,
04238                                    getF32Constant(DAG, 0x4042902c));
04239     }
04240 
04241     return DAG.getNode(ISD::FADD, dl, MVT::f32, LogOfExponent, Log2ofMantissa);
04242   }
04243 
04244   // No special expansion.
04245   return DAG.getNode(ISD::FLOG2, dl, Op.getValueType(), Op);
04246 }
04247 
04248 /// expandLog10 - Lower a log10 intrinsic. Handles the special sequences for
04249 /// limited-precision mode.
04250 static SDValue expandLog10(SDLoc dl, SDValue Op, SelectionDAG &DAG,
04251                            const TargetLowering &TLI) {
04252   if (Op.getValueType() == MVT::f32 &&
04253       LimitFloatPrecision > 0 && LimitFloatPrecision <= 18) {
04254     SDValue Op1 = DAG.getNode(ISD::BITCAST, dl, MVT::i32, Op);
04255 
04256     // Scale the exponent by log10(2) [0.30102999f].
04257     SDValue Exp = GetExponent(DAG, Op1, TLI, dl);
04258     SDValue LogOfExponent = DAG.getNode(ISD::FMUL, dl, MVT::f32, Exp,
04259                                         getF32Constant(DAG, 0x3e9a209a));
04260 
04261     // Get the significand and build it into a floating-point number with
04262     // exponent of 1.
04263     SDValue X = GetSignificand(DAG, Op1, dl);
04264 
04265     SDValue Log10ofMantissa;
04266     if (LimitFloatPrecision <= 6) {
04267       // For floating-point precision of 6:
04268       //
04269       //   Log10ofMantissa =
04270       //     -0.50419619f +
04271       //       (0.60948995f - 0.10380950f * x) * x;
04272       //
04273       // error 0.0014886165, which is 6 bits
04274       SDValue t0 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X,
04275                                getF32Constant(DAG, 0xbdd49a13));
04276       SDValue t1 = DAG.getNode(ISD::FADD, dl, MVT::f32, t0,
04277                                getF32Constant(DAG, 0x3f1c0789));
04278       SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t1, X);
04279       Log10ofMantissa = DAG.getNode(ISD::FSUB, dl, MVT::f32, t2,
04280                                     getF32Constant(DAG, 0x3f011300));
04281     } else if (LimitFloatPrecision <= 12) {
04282       // For floating-point precision of 12:
04283       //
04284       //   Log10ofMantissa =
04285       //     -0.64831180f +
04286       //       (0.91751397f +
04287       //         (-0.31664806f + 0.47637168e-1f * x) * x) * x;
04288       //
04289       // error 0.00019228036, which is better than 12 bits
04290       SDValue t0 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X,
04291                                getF32Constant(DAG, 0x3d431f31));
04292       SDValue t1 = DAG.getNode(ISD::FSUB, dl, MVT::f32, t0,
04293                                getF32Constant(DAG, 0x3ea21fb2));
04294       SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t1, X);
04295       SDValue t3 = DAG.getNode(ISD::FADD, dl, MVT::f32, t2,
04296                                getF32Constant(DAG, 0x3f6ae232));
04297       SDValue t4 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t3, X);
04298       Log10ofMantissa = DAG.getNode(ISD::FSUB, dl, MVT::f32, t4,
04299                                     getF32Constant(DAG, 0x3f25f7c3));
04300     } else { // LimitFloatPrecision <= 18
04301       // For floating-point precision of 18:
04302       //
04303       //   Log10ofMantissa =
04304       //     -0.84299375f +
04305       //       (1.5327582f +
04306       //         (-1.0688956f +
04307       //           (0.49102474f +
04308       //             (-0.12539807f + 0.13508273e-1f * x) * x) * x) * x) * x;
04309       //
04310       // error 0.0000037995730, which is better than 18 bits
04311       SDValue t0 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X,
04312                                getF32Constant(DAG, 0x3c5d51ce));
04313       SDValue t1 = DAG.getNode(ISD::FSUB, dl, MVT::f32, t0,
04314                                getF32Constant(DAG, 0x3e00685a));
04315       SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t1, X);
04316       SDValue t3 = DAG.getNode(ISD::FADD, dl, MVT::f32, t2,
04317                                getF32Constant(DAG, 0x3efb6798));
04318       SDValue t4 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t3, X);
04319       SDValue t5 = DAG.getNode(ISD::FSUB, dl, MVT::f32, t4,
04320                                getF32Constant(DAG, 0x3f88d192));
04321       SDValue t6 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t5, X);
04322       SDValue t7 = DAG.getNode(ISD::FADD, dl, MVT::f32, t6,
04323                                getF32Constant(DAG, 0x3fc4316c));
04324       SDValue t8 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t7, X);
04325       Log10ofMantissa = DAG.getNode(ISD::FSUB, dl, MVT::f32, t8,
04326                                     getF32Constant(DAG, 0x3f57ce70));
04327     }
04328 
04329     return DAG.getNode(ISD::FADD, dl, MVT::f32, LogOfExponent, Log10ofMantissa);
04330   }
04331 
04332   // No special expansion.
04333   return DAG.getNode(ISD::FLOG10, dl, Op.getValueType(), Op);
04334 }
04335 
04336 /// expandExp2 - Lower an exp2 intrinsic. Handles the special sequences for
04337 /// limited-precision mode.
04338 static SDValue expandExp2(SDLoc dl, SDValue Op, SelectionDAG &DAG,
04339                           const TargetLowering &TLI) {
04340   if (Op.getValueType() == MVT::f32 &&
04341       LimitFloatPrecision > 0 && LimitFloatPrecision <= 18)
04342     return getLimitedPrecisionExp2(Op, dl, DAG);
04343 
04344   // No special expansion.
04345   return DAG.getNode(ISD::FEXP2, dl, Op.getValueType(), Op);
04346 }
04347 
04348 /// visitPow - Lower a pow intrinsic. Handles the special sequences for
04349 /// limited-precision mode with x == 10.0f.
04350 static SDValue expandPow(SDLoc dl, SDValue LHS, SDValue RHS,
04351                          SelectionDAG &DAG, const TargetLowering &TLI) {
04352   bool IsExp10 = false;
04353   if (LHS.getValueType() == MVT::f32 && RHS.getValueType() == MVT::f32 &&
04354       LimitFloatPrecision > 0 && LimitFloatPrecision <= 18) {
04355     if (ConstantFPSDNode *LHSC = dyn_cast<ConstantFPSDNode>(LHS)) {
04356       APFloat Ten(10.0f);
04357       IsExp10 = LHSC->isExactlyValue(Ten);
04358     }
04359   }
04360 
04361   if (IsExp10) {
04362     // Put the exponent in the right bit position for later addition to the
04363     // final result:
04364     //
04365     //   #define LOG2OF10 3.3219281f
04366     //   t0 = Op * LOG2OF10;
04367     SDValue t0 = DAG.getNode(ISD::FMUL, dl, MVT::f32, RHS,
04368                              getF32Constant(DAG, 0x40549a78));
04369     return getLimitedPrecisionExp2(t0, dl, DAG);
04370   }
04371 
04372   // No special expansion.
04373   return DAG.getNode(ISD::FPOW, dl, LHS.getValueType(), LHS, RHS);
04374 }
04375 
04376 
04377 /// ExpandPowI - Expand a llvm.powi intrinsic.
04378 static SDValue ExpandPowI(SDLoc DL, SDValue LHS, SDValue RHS,
04379                           SelectionDAG &DAG) {
04380   // If RHS is a constant, we can expand this out to a multiplication tree,
04381   // otherwise we end up lowering to a call to __powidf2 (for example).  When
04382   // optimizing for size, we only want to do this if the expansion would produce
04383   // a small number of multiplies, otherwise we do the full expansion.
04384   if (ConstantSDNode *RHSC = dyn_cast<ConstantSDNode>(RHS)) {
04385     // Get the exponent as a positive value.
04386     unsigned Val = RHSC->getSExtValue();
04387     if ((int)Val < 0) Val = -Val;
04388 
04389     // powi(x, 0) -> 1.0
04390     if (Val == 0)
04391       return DAG.getConstantFP(1.0, LHS.getValueType());
04392 
04393     const Function *F = DAG.getMachineFunction().getFunction();
04394     if (!F->hasFnAttribute(Attribute::OptimizeForSize) ||
04395         // If optimizing for size, don't insert too many multiplies.  This
04396         // inserts up to 5 multiplies.
04397         countPopulation(Val) + Log2_32(Val) < 7) {
04398       // We use the simple binary decomposition method to generate the multiply
04399       // sequence.  There are more optimal ways to do this (for example,
04400       // powi(x,15) generates one more multiply than it should), but this has
04401       // the benefit of being both really simple and much better than a libcall.
04402       SDValue Res;  // Logically starts equal to 1.0
04403       SDValue CurSquare = LHS;
04404       while (Val) {
04405         if (Val & 1) {
04406           if (Res.getNode())
04407             Res = DAG.getNode(ISD::FMUL, DL,Res.getValueType(), Res, CurSquare);
04408           else
04409             Res = CurSquare;  // 1.0*CurSquare.
04410         }
04411 
04412         CurSquare = DAG.getNode(ISD::FMUL, DL, CurSquare.getValueType(),
04413                                 CurSquare, CurSquare);
04414         Val >>= 1;
04415       }
04416 
04417       // If the original was negative, invert the result, producing 1/(x*x*x).
04418       if (RHSC->getSExtValue() < 0)
04419         Res = DAG.getNode(ISD::FDIV, DL, LHS.getValueType(),
04420                           DAG.getConstantFP(1.0, LHS.getValueType()), Res);
04421       return Res;
04422     }
04423   }
04424 
04425   // Otherwise, expand to a libcall.
04426   return DAG.getNode(ISD::FPOWI, DL, LHS.getValueType(), LHS, RHS);
04427 }
04428 
04429 // getTruncatedArgReg - Find underlying register used for an truncated
04430 // argument.
04431 static unsigned getTruncatedArgReg(const SDValue &N) {
04432   if (N.getOpcode() != ISD::TRUNCATE)
04433     return 0;
04434 
04435   const SDValue &Ext = N.getOperand(0);
04436   if (Ext.getOpcode() == ISD::AssertZext ||
04437       Ext.getOpcode() == ISD::AssertSext) {
04438     const SDValue &CFR = Ext.getOperand(0);
04439     if (CFR.getOpcode() == ISD::CopyFromReg)
04440       return cast<RegisterSDNode>(CFR.getOperand(1))->getReg();
04441     if (CFR.getOpcode() == ISD::TRUNCATE)
04442       return getTruncatedArgReg(CFR);
04443   }
04444   return 0;
04445 }
04446 
04447 /// EmitFuncArgumentDbgValue - If the DbgValueInst is a dbg_value of a function
04448 /// argument, create the corresponding DBG_VALUE machine instruction for it now.
04449 /// At the end of instruction selection, they will be inserted to the entry BB.
04450 bool SelectionDAGBuilder::EmitFuncArgumentDbgValue(const Value *V,
04451                                                    MDNode *Variable,
04452                                                    MDNode *Expr, int64_t Offset,
04453                                                    bool IsIndirect,
04454                                                    const SDValue &N) {
04455   const Argument *Arg = dyn_cast<Argument>(V);
04456   if (!Arg)
04457     return false;
04458 
04459   MachineFunction &MF = DAG.getMachineFunction();
04460   const TargetInstrInfo *TII = DAG.getSubtarget().getInstrInfo();
04461 
04462   // Ignore inlined function arguments here.
04463   DIVariable DV(Variable);
04464   if (DV.isInlinedFnArgument(MF.getFunction()))
04465     return false;
04466 
04467   Optional<MachineOperand> Op;
04468   // Some arguments' frame index is recorded during argument lowering.
04469   if (int FI = FuncInfo.getArgumentFrameIndex(Arg))
04470     Op = MachineOperand::CreateFI(FI);
04471 
04472   if (!Op && N.getNode()) {
04473     unsigned Reg;
04474     if (N.getOpcode() == ISD::CopyFromReg)
04475       Reg = cast<RegisterSDNode>(N.getOperand(1))->getReg();
04476     else
04477       Reg = getTruncatedArgReg(N);
04478     if (Reg && TargetRegisterInfo::isVirtualRegister(Reg)) {
04479       MachineRegisterInfo &RegInfo = MF.getRegInfo();
04480       unsigned PR = RegInfo.getLiveInPhysReg(Reg);
04481       if (PR)
04482         Reg = PR;
04483     }
04484     if (Reg)
04485       Op = MachineOperand::CreateReg(Reg, false);
04486   }
04487 
04488   if (!Op) {
04489     // Check if ValueMap has reg number.
04490     DenseMap<const Value *, unsigned>::iterator VMI = FuncInfo.ValueMap.find(V);
04491     if (VMI != FuncInfo.ValueMap.end())
04492       Op = MachineOperand::CreateReg(VMI->second, false);
04493   }
04494 
04495   if (!Op && N.getNode())
04496     // Check if frame index is available.
04497     if (LoadSDNode *LNode = dyn_cast<LoadSDNode>(N.getNode()))
04498       if (FrameIndexSDNode *FINode =
04499           dyn_cast<FrameIndexSDNode>(LNode->getBasePtr().getNode()))
04500         Op = MachineOperand::CreateFI(FINode->getIndex());
04501 
04502   if (!Op)
04503     return false;
04504 
04505   if (Op->isReg())
04506     FuncInfo.ArgDbgValues.push_back(
04507         BuildMI(MF, getCurDebugLoc(), TII->get(TargetOpcode::DBG_VALUE),
04508                 IsIndirect, Op->getReg(), Offset, Variable, Expr));
04509   else
04510     FuncInfo.ArgDbgValues.push_back(
04511         BuildMI(MF, getCurDebugLoc(), TII->get(TargetOpcode::DBG_VALUE))
04512             .addOperand(*Op)
04513             .addImm(Offset)
04514             .addMetadata(Variable)
04515             .addMetadata(Expr));
04516 
04517   return true;
04518 }
04519 
04520 // VisualStudio defines setjmp as _setjmp
04521 #if defined(_MSC_VER) && defined(setjmp) && \
04522                          !defined(setjmp_undefined_for_msvc)
04523 #  pragma push_macro("setjmp")
04524 #  undef setjmp
04525 #  define setjmp_undefined_for_msvc
04526 #endif
04527 
04528 /// visitIntrinsicCall - Lower the call to the specified intrinsic function.  If
04529 /// we want to emit this as a call to a named external function, return the name
04530 /// otherwise lower it and return null.
04531 const char *
04532 SelectionDAGBuilder::visitIntrinsicCall(const CallInst &I, unsigned Intrinsic) {
04533   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
04534   SDLoc sdl = getCurSDLoc();
04535   DebugLoc dl = getCurDebugLoc();
04536   SDValue Res;
04537 
04538   switch (Intrinsic) {
04539   default:
04540     // By default, turn this into a target intrinsic node.
04541     visitTargetIntrinsic(I, Intrinsic);
04542     return nullptr;
04543   case Intrinsic::vastart:  visitVAStart(I); return nullptr;
04544   case Intrinsic::vaend:    visitVAEnd(I); return nullptr;
04545   case Intrinsic::vacopy:   visitVACopy(I); return nullptr;
04546   case Intrinsic::returnaddress:
04547     setValue(&I, DAG.getNode(ISD::RETURNADDR, sdl, TLI.getPointerTy(),
04548                              getValue(I.getArgOperand(0))));
04549     return nullptr;
04550   case Intrinsic::frameaddress:
04551     setValue(&I, DAG.getNode(ISD::FRAMEADDR, sdl, TLI.getPointerTy(),
04552                              getValue(I.getArgOperand(0))));
04553     return nullptr;
04554   case Intrinsic::read_register: {
04555     Value *Reg = I.getArgOperand(0);
04556     SDValue RegName =
04557         DAG.getMDNode(cast<MDNode>(cast<MetadataAsValue>(Reg)->getMetadata()));
04558     EVT VT = TLI.getValueType(I.getType());
04559     setValue(&I, DAG.getNode(ISD::READ_REGISTER, sdl, VT, RegName));
04560     return nullptr;
04561   }
04562   case Intrinsic::write_register: {
04563     Value *Reg = I.getArgOperand(0);
04564     Value *RegValue = I.getArgOperand(1);
04565     SDValue Chain = getValue(RegValue).getOperand(0);
04566     SDValue RegName =
04567         DAG.getMDNode(cast<MDNode>(cast<MetadataAsValue>(Reg)->getMetadata()));
04568     DAG.setRoot(DAG.getNode(ISD::WRITE_REGISTER, sdl, MVT::Other, Chain,
04569                             RegName, getValue(RegValue)));
04570     return nullptr;
04571   }
04572   case Intrinsic::setjmp:
04573     return &"_setjmp"[!TLI.usesUnderscoreSetJmp()];
04574   case Intrinsic::longjmp:
04575     return &"_longjmp"[!TLI.usesUnderscoreLongJmp()];
04576   case Intrinsic::memcpy: {
04577     // FIXME: this definition of "user defined address space" is x86-specific
04578     // Assert for address < 256 since we support only user defined address
04579     // spaces.
04580     assert(cast<PointerType>(I.getArgOperand(0)->getType())->getAddressSpace()
04581            < 256 &&
04582            cast<PointerType>(I.getArgOperand(1)->getType())->getAddressSpace()
04583            < 256 &&
04584            "Unknown address space");
04585     SDValue Op1 = getValue(I.getArgOperand(0));
04586     SDValue Op2 = getValue(I.getArgOperand(1));
04587     SDValue Op3 = getValue(I.getArgOperand(2));
04588     unsigned Align = cast<ConstantInt>(I.getArgOperand(3))->getZExtValue();
04589     if (!Align)
04590       Align = 1; // @llvm.memcpy defines 0 and 1 to both mean no alignment.
04591     bool isVol = cast<ConstantInt>(I.getArgOperand(4))->getZExtValue();
04592     DAG.setRoot(DAG.getMemcpy(getRoot(), sdl, Op1, Op2, Op3, Align, isVol, false,
04593                               MachinePointerInfo(I.getArgOperand(0)),
04594                               MachinePointerInfo(I.getArgOperand(1))));
04595     return nullptr;
04596   }
04597   case Intrinsic::memset: {
04598     // FIXME: this definition of "user defined address space" is x86-specific
04599     // Assert for address < 256 since we support only user defined address
04600     // spaces.
04601     assert(cast<PointerType>(I.getArgOperand(0)->getType())->getAddressSpace()
04602            < 256 &&
04603            "Unknown address space");
04604     SDValue Op1 = getValue(I.getArgOperand(0));
04605     SDValue Op2 = getValue(I.getArgOperand(1));
04606     SDValue Op3 = getValue(I.getArgOperand(2));
04607     unsigned Align = cast<ConstantInt>(I.getArgOperand(3))->getZExtValue();
04608     if (!Align)
04609       Align = 1; // @llvm.memset defines 0 and 1 to both mean no alignment.
04610     bool isVol = cast<ConstantInt>(I.getArgOperand(4))->getZExtValue();
04611     DAG.setRoot(DAG.getMemset(getRoot(), sdl, Op1, Op2, Op3, Align, isVol,
04612                               MachinePointerInfo(I.getArgOperand(0))));
04613     return nullptr;
04614   }
04615   case Intrinsic::memmove: {
04616     // FIXME: this definition of "user defined address space" is x86-specific
04617     // Assert for address < 256 since we support only user defined address
04618     // spaces.
04619     assert(cast<PointerType>(I.getArgOperand(0)->getType())->getAddressSpace()
04620            < 256 &&
04621            cast<PointerType>(I.getArgOperand(1)->getType())->getAddressSpace()
04622            < 256 &&
04623            "Unknown address space");
04624     SDValue Op1 = getValue(I.getArgOperand(0));
04625     SDValue Op2 = getValue(I.getArgOperand(1));
04626     SDValue Op3 = getValue(I.getArgOperand(2));
04627     unsigned Align = cast<ConstantInt>(I.getArgOperand(3))->getZExtValue();
04628     if (!Align)
04629       Align = 1; // @llvm.memmove defines 0 and 1 to both mean no alignment.
04630     bool isVol = cast<ConstantInt>(I.getArgOperand(4))->getZExtValue();
04631     DAG.setRoot(DAG.getMemmove(getRoot(), sdl, Op1, Op2, Op3, Align, isVol,
04632                                MachinePointerInfo(I.getArgOperand(0)),
04633                                MachinePointerInfo(I.getArgOperand(1))));
04634     return nullptr;
04635   }
04636   case Intrinsic::dbg_declare: {
04637     const DbgDeclareInst &DI = cast<DbgDeclareInst>(I);
04638     MDNode *Variable = DI.getVariable();
04639     MDNode *Expression = DI.getExpression();
04640     const Value *Address = DI.getAddress();
04641     DIVariable DIVar(Variable);
04642     assert((!DIVar || DIVar.isVariable()) &&
04643       "Variable in DbgDeclareInst should be either null or a DIVariable.");
04644     if (!Address || !DIVar) {
04645       DEBUG(dbgs() << "Dropping debug info for " << DI << "\n");
04646       return nullptr;
04647     }
04648 
04649     // Check if address has undef value.
04650     if (isa<UndefValue>(Address) ||
04651         (Address->use_empty() && !isa<Argument>(Address))) {
04652       DEBUG(dbgs() << "Dropping debug info for " << DI << "\n");
04653       return nullptr;
04654     }
04655 
04656     SDValue &N = NodeMap[Address];
04657     if (!N.getNode() && isa<Argument>(Address))
04658       // Check unused arguments map.
04659       N = UnusedArgNodeMap[Address];
04660     SDDbgValue *SDV;
04661     if (N.getNode()) {
04662       if (const BitCastInst *BCI = dyn_cast<BitCastInst>(Address))
04663         Address = BCI->getOperand(0);
04664       // Parameters are handled specially.
04665       bool isParameter =
04666         (DIVariable(Variable).getTag() == dwarf::DW_TAG_arg_variable ||
04667          isa<Argument>(Address));
04668 
04669       const AllocaInst *AI = dyn_cast<AllocaInst>(Address);
04670 
04671       if (isParameter && !AI) {
04672         FrameIndexSDNode *FINode = dyn_cast<FrameIndexSDNode>(N.getNode());
04673         if (FINode)
04674           // Byval parameter.  We have a frame index at this point.
04675           SDV = DAG.getFrameIndexDbgValue(
04676               Variable, Expression, FINode->getIndex(), 0, dl, SDNodeOrder);
04677         else {
04678           // Address is an argument, so try to emit its dbg value using
04679           // virtual register info from the FuncInfo.ValueMap.
04680           EmitFuncArgumentDbgValue(Address, Variable, Expression, 0, false, N);
04681           return nullptr;
04682         }
04683       } else if (AI)
04684         SDV = DAG.getDbgValue(Variable, Expression, N.getNode(), N.getResNo(),
04685                               true, 0, dl, SDNodeOrder);
04686       else {
04687         // Can't do anything with other non-AI cases yet.
04688         DEBUG(dbgs() << "Dropping debug info for " << DI << "\n");
04689         DEBUG(dbgs() << "non-AllocaInst issue for Address: \n\t");
04690         DEBUG(Address->dump());
04691         return nullptr;
04692       }
04693       DAG.AddDbgValue(SDV, N.getNode(), isParameter);
04694     } else {
04695       // If Address is an argument then try to emit its dbg value using
04696       // virtual register info from the FuncInfo.ValueMap.
04697       if (!EmitFuncArgumentDbgValue(Address, Variable, Expression, 0, false,
04698                                     N)) {
04699         // If variable is pinned by a alloca in dominating bb then
04700         // use StaticAllocaMap.
04701         if (const AllocaInst *AI = dyn_cast<AllocaInst>(Address)) {
04702           if (AI->getParent() != DI.getParent()) {
04703             DenseMap<const AllocaInst*, int>::iterator SI =
04704               FuncInfo.StaticAllocaMap.find(AI);
04705             if (SI != FuncInfo.StaticAllocaMap.end()) {
04706               SDV = DAG.getFrameIndexDbgValue(Variable, Expression, SI->second,
04707                                               0, dl, SDNodeOrder);
04708               DAG.AddDbgValue(SDV, nullptr, false);
04709               return nullptr;
04710             }
04711           }
04712         }
04713         DEBUG(dbgs() << "Dropping debug info for " << DI << "\n");
04714       }
04715     }
04716     return nullptr;
04717   }
04718   case Intrinsic::dbg_value: {
04719     const DbgValueInst &DI = cast<DbgValueInst>(I);
04720     DIVariable DIVar(DI.getVariable());
04721     assert((!DIVar || DIVar.isVariable()) &&
04722       "Variable in DbgValueInst should be either null or a DIVariable.");
04723     if (!DIVar)
04724       return nullptr;
04725 
04726     MDNode *Variable = DI.getVariable();
04727     MDNode *Expression = DI.getExpression();
04728     uint64_t Offset = DI.getOffset();
04729     const Value *V = DI.getValue();
04730     if (!V)
04731       return nullptr;
04732 
04733     SDDbgValue *SDV;
04734     if (isa<ConstantInt>(V) || isa<ConstantFP>(V) || isa<UndefValue>(V)) {
04735       SDV = DAG.getConstantDbgValue(Variable, Expression, V, Offset, dl,
04736                                     SDNodeOrder);
04737       DAG.AddDbgValue(SDV, nullptr, false);
04738     } else {
04739       // Do not use getValue() in here; we don't want to generate code at
04740       // this point if it hasn't been done yet.
04741       SDValue N = NodeMap[V];
04742       if (!N.getNode() && isa<Argument>(V))
04743         // Check unused arguments map.
04744         N = UnusedArgNodeMap[V];
04745       if (N.getNode()) {
04746         // A dbg.value for an alloca is always indirect.
04747         bool IsIndirect = isa<AllocaInst>(V) || Offset != 0;
04748         if (!EmitFuncArgumentDbgValue(V, Variable, Expression, Offset,
04749                                       IsIndirect, N)) {
04750           SDV = DAG.getDbgValue(Variable, Expression, N.getNode(), N.getResNo(),
04751                                 IsIndirect, Offset, dl, SDNodeOrder);
04752           DAG.AddDbgValue(SDV, N.getNode(), false);
04753         }
04754       } else if (!V->use_empty() ) {
04755         // Do not call getValue(V) yet, as we don't want to generate code.
04756         // Remember it for later.
04757         DanglingDebugInfo DDI(&DI, dl, SDNodeOrder);
04758         DanglingDebugInfoMap[V] = DDI;
04759       } else {
04760         // We may expand this to cover more cases.  One case where we have no
04761         // data available is an unreferenced parameter.
04762         DEBUG(dbgs() << "Dropping debug info for " << DI << "\n");
04763       }
04764     }
04765 
04766     // Build a debug info table entry.
04767     if (const BitCastInst *BCI = dyn_cast<BitCastInst>(V))
04768       V = BCI->getOperand(0);
04769     const AllocaInst *AI = dyn_cast<AllocaInst>(V);
04770     // Don't handle byval struct arguments or VLAs, for example.
04771     if (!AI) {
04772       DEBUG(dbgs() << "Dropping debug location info for:\n  " << DI << "\n");
04773       DEBUG(dbgs() << "  Last seen at:\n    " << *V << "\n");
04774       return nullptr;
04775     }
04776     DenseMap<const AllocaInst*, int>::iterator SI =
04777       FuncInfo.StaticAllocaMap.find(AI);
04778     if (SI == FuncInfo.StaticAllocaMap.end())
04779       return nullptr; // VLAs.
04780     return nullptr;
04781   }
04782 
04783   case Intrinsic::eh_typeid_for: {
04784     // Find the type id for the given typeinfo.
04785     GlobalValue *GV = ExtractTypeInfo(I.getArgOperand(0));
04786     unsigned TypeID = DAG.getMachineFunction().getMMI().getTypeIDFor(GV);
04787     Res = DAG.getConstant(TypeID, MVT::i32);
04788     setValue(&I, Res);
04789     return nullptr;
04790   }
04791 
04792   case Intrinsic::eh_return_i32:
04793   case Intrinsic::eh_return_i64:
04794     DAG.getMachineFunction().getMMI().setCallsEHReturn(true);
04795     DAG.setRoot(DAG.getNode(ISD::EH_RETURN, sdl,
04796                             MVT::Other,
04797                             getControlRoot(),
04798                             getValue(I.getArgOperand(0)),
04799                             getValue(I.getArgOperand(1))));
04800     return nullptr;
04801   case Intrinsic::eh_unwind_init:
04802     DAG.getMachineFunction().getMMI().setCallsUnwindInit(true);
04803     return nullptr;
04804   case Intrinsic::eh_dwarf_cfa: {
04805     SDValue CfaArg = DAG.getSExtOrTrunc(getValue(I.getArgOperand(0)), sdl,
04806                                         TLI.getPointerTy());
04807     SDValue Offset = DAG.getNode(ISD::ADD, sdl,
04808                                  CfaArg.getValueType(),
04809                                  DAG.getNode(ISD::FRAME_TO_ARGS_OFFSET, sdl,
04810                                              CfaArg.getValueType()),
04811                                  CfaArg);
04812     SDValue FA = DAG.getNode(ISD::FRAMEADDR, sdl, TLI.getPointerTy(),
04813                              DAG.getConstant(0, TLI.getPointerTy()));
04814     setValue(&I, DAG.getNode(ISD::ADD, sdl, FA.getValueType(),
04815                              FA, Offset));
04816     return nullptr;
04817   }
04818   case Intrinsic::eh_sjlj_callsite: {
04819     MachineModuleInfo &MMI = DAG.getMachineFunction().getMMI();
04820     ConstantInt *CI = dyn_cast<ConstantInt>(I.getArgOperand(0));
04821     assert(CI && "Non-constant call site value in eh.sjlj.callsite!");
04822     assert(MMI.getCurrentCallSite() == 0 && "Overlapping call sites!");
04823 
04824     MMI.setCurrentCallSite(CI->getZExtValue());
04825     return nullptr;
04826   }
04827   case Intrinsic::eh_sjlj_functioncontext: {
04828     // Get and store the index of the function context.
04829     MachineFrameInfo *MFI = DAG.getMachineFunction().getFrameInfo();
04830     AllocaInst *FnCtx =
04831       cast<AllocaInst>(I.getArgOperand(0)->stripPointerCasts());
04832     int FI = FuncInfo.StaticAllocaMap[FnCtx];
04833     MFI->setFunctionContextIndex(FI);
04834     return nullptr;
04835   }
04836   case Intrinsic::eh_sjlj_setjmp: {
04837     SDValue Ops[2];
04838     Ops[0] = getRoot();
04839     Ops[1] = getValue(I.getArgOperand(0));
04840     SDValue Op = DAG.getNode(ISD::EH_SJLJ_SETJMP, sdl,
04841                              DAG.getVTList(MVT::i32, MVT::Other), Ops);
04842     setValue(&I, Op.getValue(0));
04843     DAG.setRoot(Op.getValue(1));
04844     return nullptr;
04845   }
04846   case Intrinsic::eh_sjlj_longjmp: {
04847     DAG.setRoot(DAG.getNode(ISD::EH_SJLJ_LONGJMP, sdl, MVT::Other,
04848                             getRoot(), getValue(I.getArgOperand(0))));
04849     return nullptr;
04850   }
04851 
04852   case Intrinsic::masked_load:
04853     visitMaskedLoad(I);
04854     return nullptr;
04855   case Intrinsic::masked_store:
04856     visitMaskedStore(I);
04857     return nullptr;
04858   case Intrinsic::x86_mmx_pslli_w:
04859   case Intrinsic::x86_mmx_pslli_d:
04860   case Intrinsic::x86_mmx_pslli_q:
04861   case Intrinsic::x86_mmx_psrli_w:
04862   case Intrinsic::x86_mmx_psrli_d:
04863   case Intrinsic::x86_mmx_psrli_q:
04864   case Intrinsic::x86_mmx_psrai_w:
04865   case Intrinsic::x86_mmx_psrai_d: {
04866     SDValue ShAmt = getValue(I.getArgOperand(1));
04867     if (isa<ConstantSDNode>(ShAmt)) {
04868       visitTargetIntrinsic(I, Intrinsic);
04869       return nullptr;
04870     }
04871     unsigned NewIntrinsic = 0;
04872     EVT ShAmtVT = MVT::v2i32;
04873     switch (Intrinsic) {
04874     case Intrinsic::x86_mmx_pslli_w:
04875       NewIntrinsic = Intrinsic::x86_mmx_psll_w;
04876       break;
04877     case Intrinsic::x86_mmx_pslli_d:
04878       NewIntrinsic = Intrinsic::x86_mmx_psll_d;
04879       break;
04880     case Intrinsic::x86_mmx_pslli_q:
04881       NewIntrinsic = Intrinsic::x86_mmx_psll_q;
04882       break;
04883     case Intrinsic::x86_mmx_psrli_w:
04884       NewIntrinsic = Intrinsic::x86_mmx_psrl_w;
04885       break;
04886     case Intrinsic::x86_mmx_psrli_d:
04887       NewIntrinsic = Intrinsic::x86_mmx_psrl_d;
04888       break;
04889     case Intrinsic::x86_mmx_psrli_q:
04890       NewIntrinsic = Intrinsic::x86_mmx_psrl_q;
04891       break;
04892     case Intrinsic::x86_mmx_psrai_w:
04893       NewIntrinsic = Intrinsic::x86_mmx_psra_w;
04894       break;
04895     case Intrinsic::x86_mmx_psrai_d:
04896       NewIntrinsic = Intrinsic::x86_mmx_psra_d;
04897       break;
04898     default: llvm_unreachable("Impossible intrinsic");  // Can't reach here.
04899     }
04900 
04901     // The vector shift intrinsics with scalars uses 32b shift amounts but
04902     // the sse2/mmx shift instructions reads 64 bits. Set the upper 32 bits
04903     // to be zero.
04904     // We must do this early because v2i32 is not a legal type.
04905     SDValue ShOps[2];
04906     ShOps[0] = ShAmt;
04907     ShOps[1] = DAG.getConstant(0, MVT::i32);
04908     ShAmt =  DAG.getNode(ISD::BUILD_VECTOR, sdl, ShAmtVT, ShOps);
04909     EVT DestVT = TLI.getValueType(I.getType());
04910     ShAmt = DAG.getNode(ISD::BITCAST, sdl, DestVT, ShAmt);
04911     Res = DAG.getNode(ISD::INTRINSIC_WO_CHAIN, sdl, DestVT,
04912                        DAG.getConstant(NewIntrinsic, MVT::i32),
04913                        getValue(I.getArgOperand(0)), ShAmt);
04914     setValue(&I, Res);
04915     return nullptr;
04916   }
04917   case Intrinsic::convertff:
04918   case Intrinsic::convertfsi:
04919   case Intrinsic::convertfui:
04920   case Intrinsic::convertsif:
04921   case Intrinsic::convertuif:
04922   case Intrinsic::convertss:
04923   case Intrinsic::convertsu:
04924   case Intrinsic::convertus:
04925   case Intrinsic::convertuu: {
04926     ISD::CvtCode Code = ISD::CVT_INVALID;
04927     switch (Intrinsic) {
04928     default: llvm_unreachable("Impossible intrinsic");  // Can't reach here.
04929     case Intrinsic::convertff:  Code = ISD::CVT_FF; break;
04930     case Intrinsic::convertfsi: Code = ISD::CVT_FS; break;
04931     case Intrinsic::convertfui: Code = ISD::CVT_FU; break;
04932     case Intrinsic::convertsif: Code = ISD::CVT_SF; break;
04933     case Intrinsic::convertuif: Code = ISD::CVT_UF; break;
04934     case Intrinsic::convertss:  Code = ISD::CVT_SS; break;
04935     case Intrinsic::convertsu:  Code = ISD::CVT_SU; break;
04936     case Intrinsic::convertus:  Code = ISD::CVT_US; break;
04937     case Intrinsic::convertuu:  Code = ISD::CVT_UU; break;
04938     }
04939     EVT DestVT = TLI.getValueType(I.getType());
04940     const Value *Op1 = I.getArgOperand(0);
04941     Res = DAG.getConvertRndSat(DestVT, sdl, getValue(Op1),
04942                                DAG.getValueType(DestVT),
04943                                DAG.getValueType(getValue(Op1).getValueType()),
04944                                getValue(I.getArgOperand(1)),
04945                                getValue(I.getArgOperand(2)),
04946                                Code);
04947     setValue(&I, Res);
04948     return nullptr;
04949   }
04950   case Intrinsic::powi:
04951     setValue(&I, ExpandPowI(sdl, getValue(I.getArgOperand(0)),
04952                             getValue(I.getArgOperand(1)), DAG));
04953     return nullptr;
04954   case Intrinsic::log:
04955     setValue(&I, expandLog(sdl, getValue(I.getArgOperand(0)), DAG, TLI));
04956     return nullptr;
04957   case Intrinsic::log2:
04958     setValue(&I, expandLog2(sdl, getValue(I.getArgOperand(0)), DAG, TLI));
04959     return nullptr;
04960   case Intrinsic::log10:
04961     setValue(&I, expandLog10(sdl, getValue(I.getArgOperand(0)), DAG, TLI));
04962     return nullptr;
04963   case Intrinsic::exp:
04964     setValue(&I, expandExp(sdl, getValue(I.getArgOperand(0)), DAG, TLI));
04965     return nullptr;
04966   case Intrinsic::exp2:
04967     setValue(&I, expandExp2(sdl, getValue(I.getArgOperand(0)), DAG, TLI));
04968     return nullptr;
04969   case Intrinsic::pow:
04970     setValue(&I, expandPow(sdl, getValue(I.getArgOperand(0)),
04971                            getValue(I.getArgOperand(1)), DAG, TLI));
04972     return nullptr;
04973   case Intrinsic::sqrt:
04974   case Intrinsic::fabs:
04975   case Intrinsic::sin:
04976   case Intrinsic::cos:
04977   case Intrinsic::floor:
04978   case Intrinsic::ceil:
04979   case Intrinsic::trunc:
04980   case Intrinsic::rint:
04981   case Intrinsic::nearbyint:
04982   case Intrinsic::round: {
04983     unsigned Opcode;
04984     switch (Intrinsic) {
04985     default: llvm_unreachable("Impossible intrinsic");  // Can't reach here.
04986     case Intrinsic::sqrt:      Opcode = ISD::FSQRT;      break;
04987     case Intrinsic::fabs:      Opcode = ISD::FABS;       break;
04988     case Intrinsic::sin:       Opcode = ISD::FSIN;       break;
04989     case Intrinsic::cos:       Opcode = ISD::FCOS;       break;
04990     case Intrinsic::floor:     Opcode = ISD::FFLOOR;     break;
04991     case Intrinsic::ceil:      Opcode = ISD::FCEIL;      break;
04992     case Intrinsic::trunc:     Opcode = ISD::FTRUNC;     break;
04993     case Intrinsic::rint:      Opcode = ISD::FRINT;      break;
04994     case Intrinsic::nearbyint: Opcode = ISD::FNEARBYINT; break;
04995     case Intrinsic::round:     Opcode = ISD::FROUND;     break;
04996     }
04997 
04998     setValue(&I, DAG.getNode(Opcode, sdl,
04999                              getValue(I.getArgOperand(0)).getValueType(),
05000                              getValue(I.getArgOperand(0))));
05001     return nullptr;
05002   }
05003   case Intrinsic::minnum:
05004     setValue(&I, DAG.getNode(ISD::FMINNUM, sdl,
05005                              getValue(I.getArgOperand(0)).getValueType(),
05006                              getValue(I.getArgOperand(0)),
05007                              getValue(I.getArgOperand(1))));
05008     return nullptr;
05009   case Intrinsic::maxnum:
05010     setValue(&I, DAG.getNode(ISD::FMAXNUM, sdl,
05011                              getValue(I.getArgOperand(0)).getValueType(),
05012                              getValue(I.getArgOperand(0)),
05013                              getValue(I.getArgOperand(1))));
05014     return nullptr;
05015   case Intrinsic::copysign:
05016     setValue(&I, DAG.getNode(ISD::FCOPYSIGN, sdl,
05017                              getValue(I.getArgOperand(0)).getValueType(),
05018                              getValue(I.getArgOperand(0)),
05019                              getValue(I.getArgOperand(1))));
05020     return nullptr;
05021   case Intrinsic::fma:
05022     setValue(&I, DAG.getNode(ISD::FMA, sdl,
05023                              getValue(I.getArgOperand(0)).getValueType(),
05024                              getValue(I.getArgOperand(0)),
05025                              getValue(I.getArgOperand(1)),
05026                              getValue(I.getArgOperand(2))));
05027     return nullptr;
05028   case Intrinsic::fmuladd: {
05029     EVT VT = TLI.getValueType(I.getType());
05030     if (TM.Options.AllowFPOpFusion != FPOpFusion::Strict &&
05031         TLI.isFMAFasterThanFMulAndFAdd(VT)) {
05032       setValue(&I, DAG.getNode(ISD::FMA, sdl,
05033                                getValue(I.getArgOperand(0)).getValueType(),
05034                                getValue(I.getArgOperand(0)),
05035                                getValue(I.getArgOperand(1)),
05036                                getValue(I.getArgOperand(2))));
05037     } else {
05038       SDValue Mul = DAG.getNode(ISD::FMUL, sdl,
05039                                 getValue(I.getArgOperand(0)).getValueType(),
05040                                 getValue(I.getArgOperand(0)),
05041                                 getValue(I.getArgOperand(1)));
05042       SDValue Add = DAG.getNode(ISD::FADD, sdl,
05043                                 getValue(I.getArgOperand(0)).getValueType(),
05044                                 Mul,
05045                                 getValue(I.getArgOperand(2)));
05046       setValue(&I, Add);
05047     }
05048     return nullptr;
05049   }
05050   case Intrinsic::convert_to_fp16:
05051     setValue(&I, DAG.getNode(ISD::BITCAST, sdl, MVT::i16,
05052                              DAG.getNode(ISD::FP_ROUND, sdl, MVT::f16,
05053                                          getValue(I.getArgOperand(0)),
05054                                          DAG.getTargetConstant(0, MVT::i32))));
05055     return nullptr;
05056   case Intrinsic::convert_from_fp16:
05057     setValue(&I,
05058              DAG.getNode(ISD::FP_EXTEND, sdl, TLI.getValueType(I.getType()),
05059                          DAG.getNode(ISD::BITCAST, sdl, MVT::f16,
05060                                      getValue(I.getArgOperand(0)))));
05061     return nullptr;
05062   case Intrinsic::pcmarker: {
05063     SDValue Tmp = getValue(I.getArgOperand(0));
05064     DAG.setRoot(DAG.getNode(ISD::PCMARKER, sdl, MVT::Other, getRoot(), Tmp));
05065     return nullptr;
05066   }
05067   case Intrinsic::readcyclecounter: {
05068     SDValue Op = getRoot();
05069     Res = DAG.getNode(ISD::READCYCLECOUNTER, sdl,
05070                       DAG.getVTList(MVT::i64, MVT::Other), Op);
05071     setValue(&I, Res);
05072     DAG.setRoot(Res.getValue(1));
05073     return nullptr;
05074   }
05075   case Intrinsic::bswap:
05076     setValue(&I, DAG.getNode(ISD::BSWAP, sdl,
05077                              getValue(I.getArgOperand(0)).getValueType(),
05078                              getValue(I.getArgOperand(0))));
05079     return nullptr;
05080   case Intrinsic::cttz: {
05081     SDValue Arg = getValue(I.getArgOperand(0));
05082     ConstantInt *CI = cast<ConstantInt>(I.getArgOperand(1));
05083     EVT Ty = Arg.getValueType();
05084     setValue(&I, DAG.getNode(CI->isZero() ? ISD::CTTZ : ISD::CTTZ_ZERO_UNDEF,
05085                              sdl, Ty, Arg));
05086     return nullptr;
05087   }
05088   case Intrinsic::ctlz: {
05089     SDValue Arg = getValue(I.getArgOperand(0));
05090     ConstantInt *CI = cast<ConstantInt>(I.getArgOperand(1));
05091     EVT Ty = Arg.getValueType();
05092     setValue(&I, DAG.getNode(CI->isZero() ? ISD::CTLZ : ISD::CTLZ_ZERO_UNDEF,
05093                              sdl, Ty, Arg));
05094     return nullptr;
05095   }
05096   case Intrinsic::ctpop: {
05097     SDValue Arg = getValue(I.getArgOperand(0));
05098     EVT Ty = Arg.getValueType();
05099     setValue(&I, DAG.getNode(ISD::CTPOP, sdl, Ty, Arg));
05100     return nullptr;
05101   }
05102   case Intrinsic::stacksave: {
05103     SDValue Op = getRoot();
05104     Res = DAG.getNode(ISD::STACKSAVE, sdl,
05105                       DAG.getVTList(TLI.getPointerTy(), MVT::Other), Op);
05106     setValue(&I, Res);
05107     DAG.setRoot(Res.getValue(1));
05108     return nullptr;
05109   }
05110   case Intrinsic::stackrestore: {
05111     Res = getValue(I.getArgOperand(0));
05112     DAG.setRoot(DAG.getNode(ISD::STACKRESTORE, sdl, MVT::Other, getRoot(), Res));
05113     return nullptr;
05114   }
05115   case Intrinsic::stackprotector: {
05116     // Emit code into the DAG to store the stack guard onto the stack.
05117     MachineFunction &MF = DAG.getMachineFunction();
05118     MachineFrameInfo *MFI = MF.getFrameInfo();
05119     EVT PtrTy = TLI.getPointerTy();
05120     SDValue Src, Chain = getRoot();
05121     const Value *Ptr = cast<LoadInst>(I.getArgOperand(0))->getPointerOperand();
05122     const GlobalVariable *GV = dyn_cast<GlobalVariable>(Ptr);
05123 
05124     // See if Ptr is a bitcast. If it is, look through it and see if we can get
05125     // global variable __stack_chk_guard.
05126     if (!GV)
05127       if (const Operator *BC = dyn_cast<Operator>(Ptr))
05128         if (BC->getOpcode() == Instruction::BitCast)
05129           GV = dyn_cast<GlobalVariable>(BC->getOperand(0));
05130 
05131     if (GV && TLI.useLoadStackGuardNode()) {
05132       // Emit a LOAD_STACK_GUARD node.
05133       MachineSDNode *Node = DAG.getMachineNode(TargetOpcode::LOAD_STACK_GUARD,
05134                                                sdl, PtrTy, Chain);
05135       MachinePointerInfo MPInfo(GV);
05136       MachineInstr::mmo_iterator MemRefs = MF.allocateMemRefsArray(1);
05137       unsigned Flags = MachineMemOperand::MOLoad |
05138                        MachineMemOperand::MOInvariant;
05139       *MemRefs = MF.getMachineMemOperand(MPInfo, Flags,
05140                                          PtrTy.getSizeInBits() / 8,
05141                                          DAG.getEVTAlignment(PtrTy));
05142       Node->setMemRefs(MemRefs, MemRefs + 1);
05143 
05144       // Copy the guard value to a virtual register so that it can be
05145       // retrieved in the epilogue.
05146       Src = SDValue(Node, 0);
05147       const TargetRegisterClass *RC =
05148           TLI.getRegClassFor(Src.getSimpleValueType());
05149       unsigned Reg = MF.getRegInfo().createVirtualRegister(RC);
05150 
05151       SPDescriptor.setGuardReg(Reg);
05152       Chain = DAG.getCopyToReg(Chain, sdl, Reg, Src);
05153     } else {
05154       Src = getValue(I.getArgOperand(0));   // The guard's value.
05155     }
05156 
05157     AllocaInst *Slot = cast<AllocaInst>(I.getArgOperand(1));
05158 
05159     int FI = FuncInfo.StaticAllocaMap[Slot];
05160     MFI->setStackProtectorIndex(FI);
05161 
05162     SDValue FIN = DAG.getFrameIndex(FI, PtrTy);
05163 
05164     // Store the stack protector onto the stack.
05165     Res = DAG.getStore(Chain, sdl, Src, FIN,
05166                        MachinePointerInfo::getFixedStack(FI),
05167                        true, false, 0);
05168     setValue(&I, Res);
05169     DAG.setRoot(Res);
05170     return nullptr;
05171   }
05172   case Intrinsic::objectsize: {
05173     // If we don't know by now, we're never going to know.
05174     ConstantInt *CI = dyn_cast<ConstantInt>(I.getArgOperand(1));
05175 
05176     assert(CI && "Non-constant type in __builtin_object_size?");
05177 
05178     SDValue Arg = getValue(I.getCalledValue());
05179     EVT Ty = Arg.getValueType();
05180 
05181     if (CI->isZero())
05182       Res = DAG.getConstant(-1ULL, Ty);
05183     else
05184       Res = DAG.getConstant(0, Ty);
05185 
05186     setValue(&I, Res);
05187     return nullptr;
05188   }
05189   case Intrinsic::annotation:
05190   case Intrinsic::ptr_annotation:
05191     // Drop the intrinsic, but forward the value
05192     setValue(&I, getValue(I.getOperand(0)));
05193     return nullptr;
05194   case Intrinsic::assume:
05195   case Intrinsic::var_annotation:
05196     // Discard annotate attributes and assumptions
05197     return nullptr;
05198 
05199   case Intrinsic::init_trampoline: {
05200     const Function *F = cast<Function>(I.getArgOperand(1)->stripPointerCasts());
05201 
05202     SDValue Ops[6];
05203     Ops[0] = getRoot();
05204     Ops[1] = getValue(I.getArgOperand(0));
05205     Ops[2] = getValue(I.getArgOperand(1));
05206     Ops[3] = getValue(I.getArgOperand(2));
05207     Ops[4] = DAG.getSrcValue(I.getArgOperand(0));
05208     Ops[5] = DAG.getSrcValue(F);
05209 
05210     Res = DAG.getNode(ISD::INIT_TRAMPOLINE, sdl, MVT::Other, Ops);
05211 
05212     DAG.setRoot(Res);
05213     return nullptr;
05214   }
05215   case Intrinsic::adjust_trampoline: {
05216     setValue(&I, DAG.getNode(ISD::ADJUST_TRAMPOLINE, sdl,
05217                              TLI.getPointerTy(),
05218                              getValue(I.getArgOperand(0))));
05219     return nullptr;
05220   }
05221   case Intrinsic::gcroot:
05222     if (GFI) {
05223       const Value *Alloca = I.getArgOperand(0)->stripPointerCasts();
05224       const Constant *TypeMap = cast<Constant>(I.getArgOperand(1));
05225 
05226       FrameIndexSDNode *FI = cast<FrameIndexSDNode>(getValue(Alloca).getNode());
05227       GFI->addStackRoot(FI->getIndex(), TypeMap);
05228     }
05229     return nullptr;
05230   case Intrinsic::gcread:
05231   case Intrinsic::gcwrite:
05232     llvm_unreachable("GC failed to lower gcread/gcwrite intrinsics!");
05233   case Intrinsic::flt_rounds:
05234     setValue(&I, DAG.getNode(ISD::FLT_ROUNDS_, sdl, MVT::i32));
05235     return nullptr;
05236 
05237   case Intrinsic::expect: {
05238     // Just replace __builtin_expect(exp, c) with EXP.
05239     setValue(&I, getValue(I.getArgOperand(0)));
05240     return nullptr;
05241   }
05242 
05243   case Intrinsic::debugtrap:
05244   case Intrinsic::trap: {
05245     StringRef TrapFuncName = TM.Options.getTrapFunctionName();
05246     if (TrapFuncName.empty()) {
05247       ISD::NodeType Op = (Intrinsic == Intrinsic::trap) ?
05248         ISD::TRAP : ISD::DEBUGTRAP;
05249       DAG.setRoot(DAG.getNode(Op, sdl,MVT::Other, getRoot()));
05250       return nullptr;
05251     }
05252     TargetLowering::ArgListTy Args;
05253 
05254     TargetLowering::CallLoweringInfo CLI(DAG);
05255     CLI.setDebugLoc(sdl).setChain(getRoot())
05256       .setCallee(CallingConv::C, I.getType(),
05257                  DAG.getExternalSymbol(TrapFuncName.data(), TLI.getPointerTy()),
05258                  std::move(Args), 0);
05259 
05260     std::pair<SDValue, SDValue> Result = TLI.LowerCallTo(CLI);
05261     DAG.setRoot(Result.second);
05262     return nullptr;
05263   }
05264 
05265   case Intrinsic::uadd_with_overflow:
05266   case Intrinsic::sadd_with_overflow:
05267   case Intrinsic::usub_with_overflow:
05268   case Intrinsic::ssub_with_overflow:
05269   case Intrinsic::umul_with_overflow:
05270   case Intrinsic::smul_with_overflow: {
05271     ISD::NodeType Op;
05272     switch (Intrinsic) {
05273     default: llvm_unreachable("Impossible intrinsic");  // Can't reach here.
05274     case Intrinsic::uadd_with_overflow: Op = ISD::UADDO; break;
05275     case Intrinsic::sadd_with_overflow: Op = ISD::SADDO; break;
05276     case Intrinsic::usub_with_overflow: Op = ISD::USUBO; break;
05277     case Intrinsic::ssub_with_overflow: Op = ISD::SSUBO; break;
05278     case Intrinsic::umul_with_overflow: Op = ISD::UMULO; break;
05279     case Intrinsic::smul_with_overflow: Op = ISD::SMULO; break;
05280     }
05281     SDValue Op1 = getValue(I.getArgOperand(0));
05282     SDValue Op2 = getValue(I.getArgOperand(1));
05283 
05284     SDVTList VTs = DAG.getVTList(Op1.getValueType(), MVT::i1);
05285     setValue(&I, DAG.getNode(Op, sdl, VTs, Op1, Op2));
05286     return nullptr;
05287   }
05288   case Intrinsic::prefetch: {
05289     SDValue Ops[5];
05290     unsigned rw = cast<ConstantInt>(I.getArgOperand(1))->getZExtValue();
05291     Ops[0] = getRoot();
05292     Ops[1] = getValue(I.getArgOperand(0));
05293     Ops[2] = getValue(I.getArgOperand(1));
05294     Ops[3] = getValue(I.getArgOperand(2));
05295     Ops[4] = getValue(I.getArgOperand(3));
05296     DAG.setRoot(DAG.getMemIntrinsicNode(ISD::PREFETCH, sdl,
05297                                         DAG.getVTList(MVT::Other), Ops,
05298                                         EVT::getIntegerVT(*Context, 8),
05299                                         MachinePointerInfo(I.getArgOperand(0)),
05300                                         0, /* align */
05301                                         false, /* volatile */
05302                                         rw==0, /* read */
05303                                         rw==1)); /* write */
05304     return nullptr;
05305   }
05306   case Intrinsic::lifetime_start:
05307   case Intrinsic::lifetime_end: {
05308     bool IsStart = (Intrinsic == Intrinsic::lifetime_start);
05309     // Stack coloring is not enabled in O0, discard region information.
05310     if (TM.getOptLevel() == CodeGenOpt::None)
05311       return nullptr;
05312 
05313     SmallVector<Value *, 4> Allocas;
05314     GetUnderlyingObjects(I.getArgOperand(1), Allocas, *DL);
05315 
05316     for (SmallVectorImpl<Value*>::iterator Object = Allocas.begin(),
05317            E = Allocas.end(); Object != E; ++Object) {
05318       AllocaInst *LifetimeObject = dyn_cast_or_null<AllocaInst>(*Object);
05319 
05320       // Could not find an Alloca.
05321       if (!LifetimeObject)
05322         continue;
05323 
05324       // First check that the Alloca is static, otherwise it won't have a
05325       // valid frame index.
05326       auto SI = FuncInfo.StaticAllocaMap.find(LifetimeObject);
05327       if (SI == FuncInfo.StaticAllocaMap.end())
05328         return nullptr;
05329 
05330       int FI = SI->second;
05331 
05332       SDValue Ops[2];
05333       Ops[0] = getRoot();
05334       Ops[1] = DAG.getFrameIndex(FI, TLI.getPointerTy(), true);
05335       unsigned Opcode = (IsStart ? ISD::LIFETIME_START : ISD::LIFETIME_END);
05336 
05337       Res = DAG.getNode(Opcode, sdl, MVT::Other, Ops);
05338       DAG.setRoot(Res);
05339     }
05340     return nullptr;
05341   }
05342   case Intrinsic::invariant_start:
05343     // Discard region information.
05344     setValue(&I, DAG.getUNDEF(TLI.getPointerTy()));
05345     return nullptr;
05346   case Intrinsic::invariant_end:
05347     // Discard region information.
05348     return nullptr;
05349   case Intrinsic::stackprotectorcheck: {
05350     // Do not actually emit anything for this basic block. Instead we initialize
05351     // the stack protector descriptor and export the guard variable so we can
05352     // access it in FinishBasicBlock.
05353     const BasicBlock *BB = I.getParent();
05354     SPDescriptor.initialize(BB, FuncInfo.MBBMap[BB], I);
05355     ExportFromCurrentBlock(SPDescriptor.getGuard());
05356 
05357     // Flush our exports since we are going to process a terminator.
05358     (void)getControlRoot();
05359     return nullptr;
05360   }
05361   case Intrinsic::clear_cache:
05362     return TLI.getClearCacheBuiltinName();
05363   case Intrinsic::donothing:
05364     // ignore
05365     return nullptr;
05366   case Intrinsic::experimental_stackmap: {
05367     visitStackmap(I);
05368     return nullptr;
05369   }
05370   case Intrinsic::experimental_patchpoint_void:
05371   case Intrinsic::experimental_patchpoint_i64: {
05372     visitPatchpoint(&I);
05373     return nullptr;
05374   }
05375   case Intrinsic::experimental_gc_statepoint: {
05376     visitStatepoint(I);
05377     return nullptr;
05378   }
05379   case Intrinsic::experimental_gc_result_int:
05380   case Intrinsic::experimental_gc_result_float:
05381   case Intrinsic::experimental_gc_result_ptr:
05382   case Intrinsic::experimental_gc_result: {
05383     visitGCResult(I);
05384     return nullptr;
05385   }
05386   case Intrinsic::experimental_gc_relocate: {
05387     visitGCRelocate(I);
05388     return nullptr;
05389   }
05390   case Intrinsic::instrprof_increment:
05391     llvm_unreachable("instrprof failed to lower an increment");
05392 
05393   case Intrinsic::frameescape: {
05394     MachineFunction &MF = DAG.getMachineFunction();
05395     const TargetInstrInfo *TII = DAG.getSubtarget().getInstrInfo();
05396 
05397     // Directly emit some FRAME_ALLOC machine instrs. Label assignment emission
05398     // is the same on all targets.
05399     for (unsigned Idx = 0, E = I.getNumArgOperands(); Idx < E; ++Idx) {
05400       AllocaInst *Slot =
05401           cast<AllocaInst>(I.getArgOperand(Idx)->stripPointerCasts());
05402       assert(FuncInfo.StaticAllocaMap.count(Slot) &&
05403              "can only escape static allocas");
05404       int FI = FuncInfo.StaticAllocaMap[Slot];
05405       MCSymbol *FrameAllocSym =
05406           MF.getMMI().getContext().getOrCreateFrameAllocSymbol(MF.getName(),
05407                                                                Idx);
05408       BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, dl,
05409               TII->get(TargetOpcode::FRAME_ALLOC))
05410           .addSym(FrameAllocSym)
05411           .addFrameIndex(FI);
05412     }
05413 
05414     return nullptr;
05415   }
05416 
05417   case Intrinsic::framerecover: {
05418     // i8* @llvm.framerecover(i8* %fn, i8* %fp, i32 %idx)
05419     MachineFunction &MF = DAG.getMachineFunction();
05420     MVT PtrVT = TLI.getPointerTy(0);
05421 
05422     // Get the symbol that defines the frame offset.
05423     auto *Fn = cast<Function>(I.getArgOperand(0)->stripPointerCasts());
05424     auto *Idx = cast<ConstantInt>(I.getArgOperand(2));
05425     unsigned IdxVal = unsigned(Idx->getLimitedValue(INT_MAX));
05426     MCSymbol *FrameAllocSym =
05427         MF.getMMI().getContext().getOrCreateFrameAllocSymbol(Fn->getName(),
05428                                                              IdxVal);
05429 
05430     // Create a TargetExternalSymbol for the label to avoid any target lowering
05431     // that would make this PC relative.
05432     StringRef Name = FrameAllocSym->getName();
05433     assert(Name.data()[Name.size()] == '\0' && "not null terminated");
05434     SDValue OffsetSym = DAG.getTargetExternalSymbol(Name.data(), PtrVT);
05435     SDValue OffsetVal =
05436         DAG.getNode(ISD::FRAME_ALLOC_RECOVER, sdl, PtrVT, OffsetSym);
05437 
05438     // Add the offset to the FP.
05439     Value *FP = I.getArgOperand(1);
05440     SDValue FPVal = getValue(FP);
05441     SDValue Add = DAG.getNode(ISD::ADD, sdl, PtrVT, FPVal, OffsetVal);
05442     setValue(&I, Add);
05443 
05444     return nullptr;
05445   }
05446   case Intrinsic::eh_begincatch:
05447   case Intrinsic::eh_endcatch:
05448     llvm_unreachable("begin/end catch intrinsics not lowered in codegen");
05449   case Intrinsic::eh_parentframe: {
05450     AllocaInst *Slot =
05451         cast<AllocaInst>(I.getArgOperand(0)->stripPointerCasts());
05452     assert(FuncInfo.StaticAllocaMap.count(Slot) &&
05453            "can only use static allocas with llvm.eh.parentframe");
05454     int FI = FuncInfo.StaticAllocaMap[Slot];
05455     // TODO: Save this in the not-yet-existent WinEHFuncInfo struct.
05456     (void)FI;
05457     return nullptr;
05458   }
05459   case Intrinsic::eh_unwindhelp: {
05460     AllocaInst *Slot =
05461         cast<AllocaInst>(I.getArgOperand(0)->stripPointerCasts());
05462     assert(FuncInfo.StaticAllocaMap.count(Slot) &&
05463            "can only use static allocas with llvm.eh.unwindhelp");
05464     int FI = FuncInfo.StaticAllocaMap[Slot];
05465     // TODO: Save this in the not-yet-existent WinEHFuncInfo struct.
05466     (void)FI;
05467     return nullptr;
05468   }
05469   }
05470 }
05471 
05472 std::pair<SDValue, SDValue>
05473 SelectionDAGBuilder::lowerInvokable(TargetLowering::CallLoweringInfo &CLI,
05474                                     MachineBasicBlock *LandingPad) {
05475   MachineModuleInfo &MMI = DAG.getMachineFunction().getMMI();
05476   MCSymbol *BeginLabel = nullptr;
05477 
05478   if (LandingPad) {
05479     // Insert a label before the invoke call to mark the try range.  This can be
05480     // used to detect deletion of the invoke via the MachineModuleInfo.
05481     BeginLabel = MMI.getContext().CreateTempSymbol();
05482 
05483     // For SjLj, keep track of which landing pads go with which invokes
05484     // so as to maintain the ordering of pads in the LSDA.
05485     unsigned CallSiteIndex = MMI.getCurrentCallSite();
05486     if (CallSiteIndex) {
05487       MMI.setCallSiteBeginLabel(BeginLabel, CallSiteIndex);
05488       LPadToCallSiteMap[LandingPad].push_back(CallSiteIndex);
05489 
05490       // Now that the call site is handled, stop tracking it.
05491       MMI.setCurrentCallSite(0);
05492     }
05493 
05494     // Both PendingLoads and PendingExports must be flushed here;
05495     // this call might not return.
05496     (void)getRoot();
05497     DAG.setRoot(DAG.getEHLabel(getCurSDLoc(), getControlRoot(), BeginLabel));
05498 
05499     CLI.setChain(getRoot());
05500   }
05501   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
05502   std::pair<SDValue, SDValue> Result = TLI.LowerCallTo(CLI);
05503 
05504   assert((CLI.IsTailCall || Result.second.getNode()) &&
05505          "Non-null chain expected with non-tail call!");
05506   assert((Result.second.getNode() || !Result.first.getNode()) &&
05507          "Null value expected with tail call!");
05508 
05509   if (!Result.second.getNode()) {
05510     // As a special case, a null chain means that a tail call has been emitted
05511     // and the DAG root is already updated.
05512     HasTailCall = true;
05513 
05514     // Since there's no actual continuation from this block, nothing can be
05515     // relying on us setting vregs for them.
05516     PendingExports.clear();
05517   } else {
05518     DAG.setRoot(Result.second);
05519   }
05520 
05521   if (LandingPad) {
05522     // Insert a label at the end of the invoke call to mark the try range.  This
05523     // can be used to detect deletion of the invoke via the MachineModuleInfo.
05524     MCSymbol *EndLabel = MMI.getContext().CreateTempSymbol();
05525     DAG.setRoot(DAG.getEHLabel(getCurSDLoc(), getRoot(), EndLabel));
05526 
05527     // Inform MachineModuleInfo of range.
05528     MMI.addInvoke(LandingPad, BeginLabel, EndLabel);
05529   }
05530 
05531   return Result;
05532 }
05533 
05534 void SelectionDAGBuilder::LowerCallTo(ImmutableCallSite CS, SDValue Callee,
05535                                       bool isTailCall,
05536                                       MachineBasicBlock *LandingPad) {
05537   PointerType *PT = cast<PointerType>(CS.getCalledValue()->getType());
05538   FunctionType *FTy = cast<FunctionType>(PT->getElementType());
05539   Type *RetTy = FTy->getReturnType();
05540 
05541   TargetLowering::ArgListTy Args;
05542   TargetLowering::ArgListEntry Entry;
05543   Args.reserve(CS.arg_size());
05544 
05545   for (ImmutableCallSite::arg_iterator i = CS.arg_begin(), e = CS.arg_end();
05546        i != e; ++i) {
05547     const Value *V = *i;
05548 
05549     // Skip empty types
05550     if (V->getType()->isEmptyTy())
05551       continue;
05552 
05553     SDValue ArgNode = getValue(V);
05554     Entry.Node = ArgNode; Entry.Ty = V->getType();
05555 
05556     // Skip the first return-type Attribute to get to params.
05557     Entry.setAttributes(&CS, i - CS.arg_begin() + 1);
05558     Args.push_back(Entry);
05559 
05560     // If we have an explicit sret argument that is an Instruction, (i.e., it
05561     // might point to function-local memory), we can't meaningfully tail-call.
05562     if (Entry.isSRet && isa<Instruction>(V))
05563       isTailCall = false;
05564   }
05565 
05566   // Check if target-independent constraints permit a tail call here.
05567   // Target-dependent constraints are checked within TLI->LowerCallTo.
05568   if (isTailCall && !isInTailCallPosition(CS, DAG.getTarget()))
05569     isTailCall = false;
05570 
05571   TargetLowering::CallLoweringInfo CLI(DAG);
05572   CLI.setDebugLoc(getCurSDLoc()).setChain(getRoot())
05573     .setCallee(RetTy, FTy, Callee, std::move(Args), CS)
05574     .setTailCall(isTailCall);
05575   std::pair<SDValue,SDValue> Result = lowerInvokable(CLI, LandingPad);
05576 
05577   if (Result.first.getNode())
05578     setValue(CS.getInstruction(), Result.first);
05579 }
05580 
05581 /// IsOnlyUsedInZeroEqualityComparison - Return true if it only matters that the
05582 /// value is equal or not-equal to zero.
05583 static bool IsOnlyUsedInZeroEqualityComparison(const Value *V) {
05584   for (const User *U : V->users()) {
05585     if (const ICmpInst *IC = dyn_cast<ICmpInst>(U))
05586       if (IC->isEquality())
05587         if (const Constant *C = dyn_cast<Constant>(IC->getOperand(1)))
05588           if (C->isNullValue())
05589             continue;
05590     // Unknown instruction.
05591     return false;
05592   }
05593   return true;
05594 }
05595 
05596 static SDValue getMemCmpLoad(const Value *PtrVal, MVT LoadVT,
05597                              Type *LoadTy,
05598                              SelectionDAGBuilder &Builder) {
05599 
05600   // Check to see if this load can be trivially constant folded, e.g. if the
05601   // input is from a string literal.
05602   if (const Constant *LoadInput = dyn_cast<Constant>(PtrVal)) {
05603     // Cast pointer to the type we really want to load.
05604     LoadInput = ConstantExpr::getBitCast(const_cast<Constant *>(LoadInput),
05605                                          PointerType::getUnqual(LoadTy));
05606 
05607     if (const Constant *LoadCst = ConstantFoldLoadFromConstPtr(
05608             const_cast<Constant *>(LoadInput), *Builder.DL))
05609       return Builder.getValue(LoadCst);
05610   }
05611 
05612   // Otherwise, we have to emit the load.  If the pointer is to unfoldable but
05613   // still constant memory, the input chain can be the entry node.
05614   SDValue Root;
05615   bool ConstantMemory = false;
05616 
05617   // Do not serialize (non-volatile) loads of constant memory with anything.
05618   if (Builder.AA->pointsToConstantMemory(PtrVal)) {
05619     Root = Builder.DAG.getEntryNode();
05620     ConstantMemory = true;
05621   } else {
05622     // Do not serialize non-volatile loads against each other.
05623     Root = Builder.DAG.getRoot();
05624   }
05625 
05626   SDValue Ptr = Builder.getValue(PtrVal);
05627   SDValue LoadVal = Builder.DAG.getLoad(LoadVT, Builder.getCurSDLoc(), Root,
05628                                         Ptr, MachinePointerInfo(PtrVal),
05629                                         false /*volatile*/,
05630                                         false /*nontemporal*/,
05631                                         false /*isinvariant*/, 1 /* align=1 */);
05632 
05633   if (!ConstantMemory)
05634     Builder.PendingLoads.push_back(LoadVal.getValue(1));
05635   return LoadVal;
05636 }
05637 
05638 /// processIntegerCallValue - Record the value for an instruction that
05639 /// produces an integer result, converting the type where necessary.
05640 void SelectionDAGBuilder::processIntegerCallValue(const Instruction &I,
05641                                                   SDValue Value,
05642                                                   bool IsSigned) {
05643   EVT VT = DAG.getTargetLoweringInfo().getValueType(I.getType(), true);
05644   if (IsSigned)
05645     Value = DAG.getSExtOrTrunc(Value, getCurSDLoc(), VT);
05646   else
05647     Value = DAG.getZExtOrTrunc(Value, getCurSDLoc(), VT);
05648   setValue(&I, Value);
05649 }
05650 
05651 /// visitMemCmpCall - See if we can lower a call to memcmp in an optimized form.
05652 /// If so, return true and lower it, otherwise return false and it will be
05653 /// lowered like a normal call.
05654 bool SelectionDAGBuilder::visitMemCmpCall(const CallInst &I) {
05655   // Verify that the prototype makes sense.  int memcmp(void*,void*,size_t)
05656   if (I.getNumArgOperands() != 3)
05657     return false;
05658 
05659   const Value *LHS = I.getArgOperand(0), *RHS = I.getArgOperand(1);
05660   if (!LHS->getType()->isPointerTy() || !RHS->getType()->isPointerTy() ||
05661       !I.getArgOperand(2)->getType()->isIntegerTy() ||
05662       !I.getType()->isIntegerTy())
05663     return false;
05664 
05665   const Value *Size = I.getArgOperand(2);
05666   const ConstantInt *CSize = dyn_cast<ConstantInt>(Size);
05667   if (CSize && CSize->getZExtValue() == 0) {
05668     EVT CallVT = DAG.getTargetLoweringInfo().getValueType(I.getType(), true);
05669     setValue(&I, DAG.getConstant(0, CallVT));
05670     return true;
05671   }
05672 
05673   const TargetSelectionDAGInfo &TSI = DAG.getSelectionDAGInfo();
05674   std::pair<SDValue, SDValue> Res =
05675     TSI.EmitTargetCodeForMemcmp(DAG, getCurSDLoc(), DAG.getRoot(),
05676                                 getValue(LHS), getValue(RHS), getValue(Size),
05677                                 MachinePointerInfo(LHS),
05678                                 MachinePointerInfo(RHS));
05679   if (Res.first.getNode()) {
05680     processIntegerCallValue(I, Res.first, true);
05681     PendingLoads.push_back(Res.second);
05682     return true;
05683   }
05684 
05685   // memcmp(S1,S2,2) != 0 -> (*(short*)LHS != *(short*)RHS)  != 0
05686   // memcmp(S1,S2,4) != 0 -> (*(int*)LHS != *(int*)RHS)  != 0
05687   if (CSize && IsOnlyUsedInZeroEqualityComparison(&I)) {
05688     bool ActuallyDoIt = true;
05689     MVT LoadVT;
05690     Type *LoadTy;
05691     switch (CSize->getZExtValue()) {
05692     default:
05693       LoadVT = MVT::Other;
05694       LoadTy = nullptr;
05695       ActuallyDoIt = false;
05696       break;
05697     case 2:
05698       LoadVT = MVT::i16;
05699       LoadTy = Type::getInt16Ty(CSize->getContext());
05700       break;
05701     case 4:
05702       LoadVT = MVT::i32;
05703       LoadTy = Type::getInt32Ty(CSize->getContext());
05704       break;
05705     case 8:
05706       LoadVT = MVT::i64;
05707       LoadTy = Type::getInt64Ty(CSize->getContext());
05708       break;
05709         /*
05710     case 16:
05711       LoadVT = MVT::v4i32;
05712       LoadTy = Type::getInt32Ty(CSize->getContext());
05713       LoadTy = VectorType::get(LoadTy, 4);
05714       break;
05715          */
05716     }
05717 
05718     // This turns into unaligned loads.  We only do this if the target natively
05719     // supports the MVT we'll be loading or if it is small enough (<= 4) that
05720     // we'll only produce a small number of byte loads.
05721 
05722     // Require that we can find a legal MVT, and only do this if the target
05723     // supports unaligned loads of that type.  Expanding into byte loads would
05724     // bloat the code.
05725     const TargetLowering &TLI = DAG.getTargetLoweringInfo();
05726     if (ActuallyDoIt && CSize->getZExtValue() > 4) {
05727       unsigned DstAS = LHS->getType()->getPointerAddressSpace();
05728       unsigned SrcAS = RHS->getType()->getPointerAddressSpace();
05729       // TODO: Handle 5 byte compare as 4-byte + 1 byte.
05730       // TODO: Handle 8 byte compare on x86-32 as two 32-bit loads.
05731       // TODO: Check alignment of src and dest ptrs.
05732       if (!TLI.isTypeLegal(LoadVT) ||
05733           !TLI.allowsMisalignedMemoryAccesses(LoadVT, SrcAS) ||
05734           !TLI.allowsMisalignedMemoryAccesses(LoadVT, DstAS))
05735         ActuallyDoIt = false;
05736     }
05737 
05738     if (ActuallyDoIt) {
05739       SDValue LHSVal = getMemCmpLoad(LHS, LoadVT, LoadTy, *this);
05740       SDValue RHSVal = getMemCmpLoad(RHS, LoadVT, LoadTy, *this);
05741 
05742       SDValue Res = DAG.getSetCC(getCurSDLoc(), MVT::i1, LHSVal, RHSVal,
05743                                  ISD::SETNE);
05744       processIntegerCallValue(I, Res, false);
05745       return true;
05746     }
05747   }
05748 
05749 
05750   return false;
05751 }
05752 
05753 /// visitMemChrCall -- See if we can lower a memchr call into an optimized
05754 /// form.  If so, return true and lower it, otherwise return false and it
05755 /// will be lowered like a normal call.
05756 bool SelectionDAGBuilder::visitMemChrCall(const CallInst &I) {
05757   // Verify that the prototype makes sense.  void *memchr(void *, int, size_t)
05758   if (I.getNumArgOperands() != 3)
05759     return false;
05760 
05761   const Value *Src = I.getArgOperand(0);
05762   const Value *Char = I.getArgOperand(1);
05763   const Value *Length = I.getArgOperand(2);
05764   if (!Src->getType()->isPointerTy() ||
05765       !Char->getType()->isIntegerTy() ||
05766       !Length->getType()->isIntegerTy() ||
05767       !I.getType()->isPointerTy())
05768     return false;
05769 
05770   const TargetSelectionDAGInfo &TSI = DAG.getSelectionDAGInfo();
05771   std::pair<SDValue, SDValue> Res =
05772     TSI.EmitTargetCodeForMemchr(DAG, getCurSDLoc(), DAG.getRoot(),
05773                                 getValue(Src), getValue(Char), getValue(Length),
05774                                 MachinePointerInfo(Src));
05775   if (Res.first.getNode()) {
05776     setValue(&I, Res.first);
05777     PendingLoads.push_back(Res.second);
05778     return true;
05779   }
05780 
05781   return false;
05782 }
05783 
05784 /// visitStrCpyCall -- See if we can lower a strcpy or stpcpy call into an
05785 /// optimized form.  If so, return true and lower it, otherwise return false
05786 /// and it will be lowered like a normal call.
05787 bool SelectionDAGBuilder::visitStrCpyCall(const CallInst &I, bool isStpcpy) {
05788   // Verify that the prototype makes sense.  char *strcpy(char *, char *)
05789   if (I.getNumArgOperands() != 2)
05790     return false;
05791 
05792   const Value *Arg0 = I.getArgOperand(0), *Arg1 = I.getArgOperand(1);
05793   if (!Arg0->getType()->isPointerTy() ||
05794       !Arg1->getType()->isPointerTy() ||
05795       !I.getType()->isPointerTy())
05796     return false;
05797 
05798   const TargetSelectionDAGInfo &TSI = DAG.getSelectionDAGInfo();
05799   std::pair<SDValue, SDValue> Res =
05800     TSI.EmitTargetCodeForStrcpy(DAG, getCurSDLoc(), getRoot(),
05801                                 getValue(Arg0), getValue(Arg1),
05802                                 MachinePointerInfo(Arg0),
05803                                 MachinePointerInfo(Arg1), isStpcpy);
05804   if (Res.first.getNode()) {
05805     setValue(&I, Res.first);
05806     DAG.setRoot(Res.second);
05807     return true;
05808   }
05809 
05810   return false;
05811 }
05812 
05813 /// visitStrCmpCall - See if we can lower a call to strcmp in an optimized form.
05814 /// If so, return true and lower it, otherwise return false and it will be
05815 /// lowered like a normal call.
05816 bool SelectionDAGBuilder::visitStrCmpCall(const CallInst &I) {
05817   // Verify that the prototype makes sense.  int strcmp(void*,void*)
05818   if (I.getNumArgOperands() != 2)
05819     return false;
05820 
05821   const Value *Arg0 = I.getArgOperand(0), *Arg1 = I.getArgOperand(1);
05822   if (!Arg0->getType()->isPointerTy() ||
05823       !Arg1->getType()->isPointerTy() ||
05824       !I.getType()->isIntegerTy())
05825     return false;
05826 
05827   const TargetSelectionDAGInfo &TSI = DAG.getSelectionDAGInfo();
05828   std::pair<SDValue, SDValue> Res =
05829     TSI.EmitTargetCodeForStrcmp(DAG, getCurSDLoc(), DAG.getRoot(),
05830                                 getValue(Arg0), getValue(Arg1),
05831                                 MachinePointerInfo(Arg0),
05832                                 MachinePointerInfo(Arg1));
05833   if (Res.first.getNode()) {
05834     processIntegerCallValue(I, Res.first, true);
05835     PendingLoads.push_back(Res.second);
05836     return true;
05837   }
05838 
05839   return false;
05840 }
05841 
05842 /// visitStrLenCall -- See if we can lower a strlen call into an optimized
05843 /// form.  If so, return true and lower it, otherwise return false and it
05844 /// will be lowered like a normal call.
05845 bool SelectionDAGBuilder::visitStrLenCall(const CallInst &I) {
05846   // Verify that the prototype makes sense.  size_t strlen(char *)
05847   if (I.getNumArgOperands() != 1)
05848     return false;
05849 
05850   const Value *Arg0 = I.getArgOperand(0);
05851   if (!Arg0->getType()->isPointerTy() || !I.getType()->isIntegerTy())
05852     return false;
05853 
05854   const TargetSelectionDAGInfo &TSI = DAG.getSelectionDAGInfo();
05855   std::pair<SDValue, SDValue> Res =
05856     TSI.EmitTargetCodeForStrlen(DAG, getCurSDLoc(), DAG.getRoot(),
05857                                 getValue(Arg0), MachinePointerInfo(Arg0));
05858   if (Res.first.getNode()) {
05859     processIntegerCallValue(I, Res.first, false);
05860     PendingLoads.push_back(Res.second);
05861     return true;
05862   }
05863 
05864   return false;
05865 }
05866 
05867 /// visitStrNLenCall -- See if we can lower a strnlen call into an optimized
05868 /// form.  If so, return true and lower it, otherwise return false and it
05869 /// will be lowered like a normal call.
05870 bool SelectionDAGBuilder::visitStrNLenCall(const CallInst &I) {
05871   // Verify that the prototype makes sense.  size_t strnlen(char *, size_t)
05872   if (I.getNumArgOperands() != 2)
05873     return false;
05874 
05875   const Value *Arg0 = I.getArgOperand(0), *Arg1 = I.getArgOperand(1);
05876   if (!Arg0->getType()->isPointerTy() ||
05877       !Arg1->getType()->isIntegerTy() ||
05878       !I.getType()->isIntegerTy())
05879     return false;
05880 
05881   const TargetSelectionDAGInfo &TSI = DAG.getSelectionDAGInfo();
05882   std::pair<SDValue, SDValue> Res =
05883     TSI.EmitTargetCodeForStrnlen(DAG, getCurSDLoc(), DAG.getRoot(),
05884                                  getValue(Arg0), getValue(Arg1),
05885                                  MachinePointerInfo(Arg0));
05886   if (Res.first.getNode()) {
05887     processIntegerCallValue(I, Res.first, false);
05888     PendingLoads.push_back(Res.second);
05889     return true;
05890   }
05891 
05892   return false;
05893 }
05894 
05895 /// visitUnaryFloatCall - If a call instruction is a unary floating-point
05896 /// operation (as expected), translate it to an SDNode with the specified opcode
05897 /// and return true.
05898 bool SelectionDAGBuilder::visitUnaryFloatCall(const CallInst &I,
05899                                               unsigned Opcode) {
05900   // Sanity check that it really is a unary floating-point call.
05901   if (I.getNumArgOperands() != 1 ||
05902       !I.getArgOperand(0)->getType()->isFloatingPointTy() ||
05903       I.getType() != I.getArgOperand(0)->getType() ||
05904       !I.onlyReadsMemory())
05905     return false;
05906 
05907   SDValue Tmp = getValue(I.getArgOperand(0));
05908   setValue(&I, DAG.getNode(Opcode, getCurSDLoc(), Tmp.getValueType(), Tmp));
05909   return true;
05910 }
05911 
05912 /// visitBinaryFloatCall - If a call instruction is a binary floating-point
05913 /// operation (as expected), translate it to an SDNode with the specified opcode
05914 /// and return true.
05915 bool SelectionDAGBuilder::visitBinaryFloatCall(const CallInst &I,
05916                                                unsigned Opcode) {
05917   // Sanity check that it really is a binary floating-point call.
05918   if (I.getNumArgOperands() != 2 ||
05919       !I.getArgOperand(0)->getType()->isFloatingPointTy() ||
05920       I.getType() != I.getArgOperand(0)->getType() ||
05921       I.getType() != I.getArgOperand(1)->getType() ||
05922       !I.onlyReadsMemory())
05923     return false;
05924 
05925   SDValue Tmp0 = getValue(I.getArgOperand(0));
05926   SDValue Tmp1 = getValue(I.getArgOperand(1));
05927   EVT VT = Tmp0.getValueType();
05928   setValue(&I, DAG.getNode(Opcode, getCurSDLoc(), VT, Tmp0, Tmp1));
05929   return true;
05930 }
05931 
05932 void SelectionDAGBuilder::visitCall(const CallInst &I) {
05933   // Handle inline assembly differently.
05934   if (isa<InlineAsm>(I.getCalledValue())) {
05935     visitInlineAsm(&I);
05936     return;
05937   }
05938 
05939   MachineModuleInfo &MMI = DAG.getMachineFunction().getMMI();
05940   ComputeUsesVAFloatArgument(I, &MMI);
05941 
05942   const char *RenameFn = nullptr;
05943   if (Function *F = I.getCalledFunction()) {
05944     if (F->isDeclaration()) {
05945       if (const TargetIntrinsicInfo *II = TM.getIntrinsicInfo()) {
05946         if (unsigned IID = II->getIntrinsicID(F)) {
05947           RenameFn = visitIntrinsicCall(I, IID);
05948           if (!RenameFn)
05949             return;
05950         }
05951       }
05952       if (unsigned IID = F->getIntrinsicID()) {
05953         RenameFn = visitIntrinsicCall(I, IID);
05954         if (!RenameFn)
05955           return;
05956       }
05957     }
05958 
05959     // Check for well-known libc/libm calls.  If the function is internal, it
05960     // can't be a library call.
05961     LibFunc::Func Func;
05962     if (!F->hasLocalLinkage() && F->hasName() &&
05963         LibInfo->getLibFunc(F->getName(), Func) &&
05964         LibInfo->hasOptimizedCodeGen(Func)) {
05965       switch (Func) {
05966       default: break;
05967       case LibFunc::copysign:
05968       case LibFunc::copysignf:
05969       case LibFunc::copysignl:
05970         if (I.getNumArgOperands() == 2 &&   // Basic sanity checks.
05971             I.getArgOperand(0)->getType()->isFloatingPointTy() &&
05972             I.getType() == I.getArgOperand(0)->getType() &&
05973             I.getType() == I.getArgOperand(1)->getType() &&
05974             I.onlyReadsMemory()) {
05975           SDValue LHS = getValue(I.getArgOperand(0));
05976           SDValue RHS = getValue(I.getArgOperand(1));
05977           setValue(&I, DAG.getNode(ISD::FCOPYSIGN, getCurSDLoc(),
05978                                    LHS.getValueType(), LHS, RHS));
05979           return;
05980         }
05981         break;
05982       case LibFunc::fabs:
05983       case LibFunc::fabsf:
05984       case LibFunc::fabsl:
05985         if (visitUnaryFloatCall(I, ISD::FABS))
05986           return;
05987         break;
05988       case LibFunc::fmin:
05989       case LibFunc::fminf:
05990       case LibFunc::fminl:
05991         if (visitBinaryFloatCall(I, ISD::FMINNUM))
05992           return;
05993         break;
05994       case LibFunc::fmax:
05995       case LibFunc::fmaxf:
05996       case LibFunc::fmaxl:
05997         if (visitBinaryFloatCall(I, ISD::FMAXNUM))
05998           return;
05999         break;
06000       case LibFunc::sin:
06001       case LibFunc::sinf:
06002       case LibFunc::sinl:
06003         if (visitUnaryFloatCall(I, ISD::FSIN))
06004           return;
06005         break;
06006       case LibFunc::cos:
06007       case LibFunc::cosf:
06008       case LibFunc::cosl:
06009         if (visitUnaryFloatCall(I, ISD::FCOS))
06010           return;
06011         break;
06012       case LibFunc::sqrt:
06013       case LibFunc::sqrtf:
06014       case LibFunc::sqrtl:
06015       case LibFunc::sqrt_finite:
06016       case LibFunc::sqrtf_finite:
06017       case LibFunc::sqrtl_finite:
06018         if (visitUnaryFloatCall(I, ISD::FSQRT))
06019           return;
06020         break;
06021       case LibFunc::floor:
06022       case LibFunc::floorf:
06023       case LibFunc::floorl:
06024         if (visitUnaryFloatCall(I, ISD::FFLOOR))
06025           return;
06026         break;
06027       case LibFunc::nearbyint:
06028       case LibFunc::nearbyintf:
06029       case LibFunc::nearbyintl:
06030         if (visitUnaryFloatCall(I, ISD::FNEARBYINT))
06031           return;
06032         break;
06033       case LibFunc::ceil:
06034       case LibFunc::ceilf:
06035       case LibFunc::ceill:
06036         if (visitUnaryFloatCall(I, ISD::FCEIL))
06037           return;
06038         break;
06039       case LibFunc::rint:
06040       case LibFunc::rintf:
06041       case LibFunc::rintl:
06042         if (visitUnaryFloatCall(I, ISD::FRINT))
06043           return;
06044         break;
06045       case LibFunc::round:
06046       case LibFunc::roundf:
06047       case LibFunc::roundl:
06048         if (visitUnaryFloatCall(I, ISD::FROUND))
06049           return;
06050         break;
06051       case LibFunc::trunc:
06052       case LibFunc::truncf:
06053       case LibFunc::truncl:
06054         if (visitUnaryFloatCall(I, ISD::FTRUNC))
06055           return;
06056         break;
06057       case LibFunc::log2:
06058       case LibFunc::log2f:
06059       case LibFunc::log2l:
06060         if (visitUnaryFloatCall(I, ISD::FLOG2))
06061           return;
06062         break;
06063       case LibFunc::exp2:
06064       case LibFunc::exp2f:
06065       case LibFunc::exp2l:
06066         if (visitUnaryFloatCall(I, ISD::FEXP2))
06067           return;
06068         break;
06069       case LibFunc::memcmp:
06070         if (visitMemCmpCall(I))
06071           return;
06072         break;
06073       case LibFunc::memchr:
06074         if (visitMemChrCall(I))
06075           return;
06076         break;
06077       case LibFunc::strcpy:
06078         if (visitStrCpyCall(I, false))
06079           return;
06080         break;
06081       case LibFunc::stpcpy:
06082         if (visitStrCpyCall(I, true))
06083           return;
06084         break;
06085       case LibFunc::strcmp:
06086         if (visitStrCmpCall(I))
06087           return;
06088         break;
06089       case LibFunc::strlen:
06090         if (visitStrLenCall(I))
06091           return;
06092         break;
06093       case LibFunc::strnlen:
06094         if (visitStrNLenCall(I))
06095           return;
06096         break;
06097       }
06098     }
06099   }
06100 
06101   SDValue Callee;
06102   if (!RenameFn)
06103     Callee = getValue(I.getCalledValue());
06104   else
06105     Callee = DAG.getExternalSymbol(RenameFn,
06106                                    DAG.getTargetLoweringInfo().getPointerTy());
06107 
06108   // Check if we can potentially perform a tail call. More detailed checking is
06109   // be done within LowerCallTo, after more information about the call is known.
06110   LowerCallTo(&I, Callee, I.isTailCall());
06111 }
06112 
06113 namespace {
06114 
06115 /// AsmOperandInfo - This contains information for each constraint that we are
06116 /// lowering.
06117 class SDISelAsmOperandInfo : public TargetLowering::AsmOperandInfo {
06118 public:
06119   /// CallOperand - If this is the result output operand or a clobber
06120   /// this is null, otherwise it is the incoming operand to the CallInst.
06121   /// This gets modified as the asm is processed.
06122   SDValue CallOperand;
06123 
06124   /// AssignedRegs - If this is a register or register class operand, this
06125   /// contains the set of register corresponding to the operand.
06126   RegsForValue AssignedRegs;
06127 
06128   explicit SDISelAsmOperandInfo(const TargetLowering::AsmOperandInfo &info)
06129     : TargetLowering::AsmOperandInfo(info), CallOperand(nullptr,0) {
06130   }
06131 
06132   /// getCallOperandValEVT - Return the EVT of the Value* that this operand
06133   /// corresponds to.  If there is no Value* for this operand, it returns
06134   /// MVT::Other.
06135   EVT getCallOperandValEVT(LLVMContext &Context,
06136                            const TargetLowering &TLI,
06137                            const DataLayout *DL) const {
06138     if (!CallOperandVal) return MVT::Other;
06139 
06140     if (isa<BasicBlock>(CallOperandVal))
06141       return TLI.getPointerTy();
06142 
06143     llvm::Type *OpTy = CallOperandVal->getType();
06144 
06145     // FIXME: code duplicated from TargetLowering::ParseConstraints().
06146     // If this is an indirect operand, the operand is a pointer to the
06147     // accessed type.
06148     if (isIndirect) {
06149       llvm::PointerType *PtrTy = dyn_cast<PointerType>(OpTy);
06150       if (!PtrTy)
06151         report_fatal_error("Indirect operand for inline asm not a pointer!");
06152       OpTy = PtrTy->getElementType();
06153     }
06154 
06155     // Look for vector wrapped in a struct. e.g. { <16 x i8> }.
06156     if (StructType *STy = dyn_cast<StructType>(OpTy))
06157       if (STy->getNumElements() == 1)
06158         OpTy = STy->getElementType(0);
06159 
06160     // If OpTy is not a single value, it may be a struct/union that we
06161     // can tile with integers.
06162     if (!OpTy->isSingleValueType() && OpTy->isSized()) {
06163       unsigned BitSize = DL->getTypeSizeInBits(OpTy);
06164       switch (BitSize) {
06165       default: break;
06166       case 1:
06167       case 8:
06168       case 16:
06169       case 32:
06170       case 64:
06171       case 128:
06172         OpTy = IntegerType::get(Context, BitSize);
06173         break;
06174       }
06175     }
06176 
06177     return TLI.getValueType(OpTy, true);
06178   }
06179 };
06180 
06181 typedef SmallVector<SDISelAsmOperandInfo,16> SDISelAsmOperandInfoVector;
06182 
06183 } // end anonymous namespace
06184 
06185 /// GetRegistersForValue - Assign registers (virtual or physical) for the
06186 /// specified operand.  We prefer to assign virtual registers, to allow the
06187 /// register allocator to handle the assignment process.  However, if the asm
06188 /// uses features that we can't model on machineinstrs, we have SDISel do the
06189 /// allocation.  This produces generally horrible, but correct, code.
06190 ///
06191 ///   OpInfo describes the operand.
06192 ///
06193 static void GetRegistersForValue(SelectionDAG &DAG,
06194                                  const TargetLowering &TLI,
06195                                  SDLoc DL,
06196                                  SDISelAsmOperandInfo &OpInfo) {
06197   LLVMContext &Context = *DAG.getContext();
06198 
06199   MachineFunction &MF = DAG.getMachineFunction();
06200   SmallVector<unsigned, 4> Regs;
06201 
06202   // If this is a constraint for a single physreg, or a constraint for a
06203   // register class, find it.
06204   std::pair<unsigned, const TargetRegisterClass *> PhysReg =
06205       TLI.getRegForInlineAsmConstraint(MF.getSubtarget().getRegisterInfo(),
06206                                        OpInfo.ConstraintCode,
06207                                        OpInfo.ConstraintVT);
06208 
06209   unsigned NumRegs = 1;
06210   if (OpInfo.ConstraintVT != MVT::Other) {
06211     // If this is a FP input in an integer register (or visa versa) insert a bit
06212     // cast of the input value.  More generally, handle any case where the input
06213     // value disagrees with the register class we plan to stick this in.
06214     if (OpInfo.Type == InlineAsm::isInput &&
06215         PhysReg.second && !PhysReg.second->hasType(OpInfo.ConstraintVT)) {
06216       // Try to convert to the first EVT that the reg class contains.  If the
06217       // types are identical size, use a bitcast to convert (e.g. two differing
06218       // vector types).
06219       MVT RegVT = *PhysReg.second->vt_begin();
06220       if (RegVT.getSizeInBits() == OpInfo.CallOperand.getValueSizeInBits()) {
06221         OpInfo.CallOperand = DAG.getNode(ISD::BITCAST, DL,
06222                                          RegVT, OpInfo.CallOperand);
06223         OpInfo.ConstraintVT = RegVT;
06224       } else if (RegVT.isInteger() && OpInfo.ConstraintVT.isFloatingPoint()) {
06225         // If the input is a FP value and we want it in FP registers, do a
06226         // bitcast to the corresponding integer type.  This turns an f64 value
06227         // into i64, which can be passed with two i32 values on a 32-bit
06228         // machine.
06229         RegVT = MVT::getIntegerVT(OpInfo.ConstraintVT.getSizeInBits());
06230         OpInfo.CallOperand = DAG.getNode(ISD::BITCAST, DL,
06231                                          RegVT, OpInfo.CallOperand);
06232         OpInfo.ConstraintVT = RegVT;
06233       }
06234     }
06235 
06236     NumRegs = TLI.getNumRegisters(Context, OpInfo.ConstraintVT);
06237   }
06238 
06239   MVT RegVT;
06240   EVT ValueVT = OpInfo.ConstraintVT;
06241 
06242   // If this is a constraint for a specific physical register, like {r17},
06243   // assign it now.
06244   if (unsigned AssignedReg = PhysReg.first) {
06245     const TargetRegisterClass *RC = PhysReg.second;
06246     if (OpInfo.ConstraintVT == MVT::Other)
06247       ValueVT = *RC->vt_begin();
06248 
06249     // Get the actual register value type.  This is important, because the user
06250     // may have asked for (e.g.) the AX register in i32 type.  We need to
06251     // remember that AX is actually i16 to get the right extension.
06252     RegVT = *RC->vt_begin();
06253 
06254     // This is a explicit reference to a physical register.
06255     Regs.push_back(AssignedReg);
06256 
06257     // If this is an expanded reference, add the rest of the regs to Regs.
06258     if (NumRegs != 1) {
06259       TargetRegisterClass::iterator I = RC->begin();
06260       for (; *I != AssignedReg; ++I)
06261         assert(I != RC->end() && "Didn't find reg!");
06262 
06263       // Already added the first reg.
06264       --NumRegs; ++I;
06265       for (; NumRegs; --NumRegs, ++I) {
06266         assert(I != RC->end() && "Ran out of registers to allocate!");
06267         Regs.push_back(*I);
06268       }
06269     }
06270 
06271     OpInfo.AssignedRegs = RegsForValue(Regs, RegVT, ValueVT);
06272     return;
06273   }
06274 
06275   // Otherwise, if this was a reference to an LLVM register class, create vregs
06276   // for this reference.
06277   if (const TargetRegisterClass *RC = PhysReg.second) {
06278     RegVT = *RC->vt_begin();
06279     if (OpInfo.ConstraintVT == MVT::Other)
06280       ValueVT = RegVT;
06281 
06282     // Create the appropriate number of virtual registers.
06283     MachineRegisterInfo &RegInfo = MF.getRegInfo();
06284     for (; NumRegs; --NumRegs)
06285       Regs.push_back(RegInfo.createVirtualRegister(RC));
06286 
06287     OpInfo.AssignedRegs = RegsForValue(Regs, RegVT, ValueVT);
06288     return;
06289   }
06290 
06291   // Otherwise, we couldn't allocate enough registers for this.
06292 }
06293 
06294 /// visitInlineAsm - Handle a call to an InlineAsm object.
06295 ///
06296 void SelectionDAGBuilder::visitInlineAsm(ImmutableCallSite CS) {
06297   const InlineAsm *IA = cast<InlineAsm>(CS.getCalledValue());
06298 
06299   /// ConstraintOperands - Information about all of the constraints.
06300   SDISelAsmOperandInfoVector ConstraintOperands;
06301 
06302   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
06303   TargetLowering::AsmOperandInfoVector TargetConstraints =
06304       TLI.ParseConstraints(DAG.getSubtarget().getRegisterInfo(), CS);
06305 
06306   bool hasMemory = false;
06307 
06308   unsigned ArgNo = 0;   // ArgNo - The argument of the CallInst.
06309   unsigned ResNo = 0;   // ResNo - The result number of the next output.
06310   for (unsigned i = 0, e = TargetConstraints.size(); i != e; ++i) {
06311     ConstraintOperands.push_back(SDISelAsmOperandInfo(TargetConstraints[i]));
06312     SDISelAsmOperandInfo &OpInfo = ConstraintOperands.back();
06313 
06314     MVT OpVT = MVT::Other;
06315 
06316     // Compute the value type for each operand.
06317     switch (OpInfo.Type) {
06318     case InlineAsm::isOutput:
06319       // Indirect outputs just consume an argument.
06320       if (OpInfo.isIndirect) {
06321         OpInfo.CallOperandVal = const_cast<Value *>(CS.getArgument(ArgNo++));
06322         break;
06323       }
06324 
06325       // The return value of the call is this value.  As such, there is no
06326       // corresponding argument.
06327       assert(!CS.getType()->isVoidTy() && "Bad inline asm!");
06328       if (StructType *STy = dyn_cast<StructType>(CS.getType())) {
06329         OpVT = TLI.getSimpleValueType(STy->getElementType(ResNo));
06330       } else {
06331         assert(ResNo == 0 && "Asm only has one result!");
06332         OpVT = TLI.getSimpleValueType(CS.getType());
06333       }
06334       ++ResNo;
06335       break;
06336     case InlineAsm::isInput:
06337       OpInfo.CallOperandVal = const_cast<Value *>(CS.getArgument(ArgNo++));
06338       break;
06339     case InlineAsm::isClobber:
06340       // Nothing to do.
06341       break;
06342     }
06343 
06344     // If this is an input or an indirect output, process the call argument.
06345     // BasicBlocks are labels, currently appearing only in asm's.
06346     if (OpInfo.CallOperandVal) {
06347       if (const BasicBlock *BB = dyn_cast<BasicBlock>(OpInfo.CallOperandVal)) {
06348         OpInfo.CallOperand = DAG.getBasicBlock(FuncInfo.