LLVM API Documentation

SelectionDAGBuilder.cpp
Go to the documentation of this file.
00001 //===-- SelectionDAGBuilder.cpp - Selection-DAG building ------------------===//
00002 //
00003 //                     The LLVM Compiler Infrastructure
00004 //
00005 // This file is distributed under the University of Illinois Open Source
00006 // License. See LICENSE.TXT for details.
00007 //
00008 //===----------------------------------------------------------------------===//
00009 //
00010 // This implements routines for translating from LLVM IR into SelectionDAG IR.
00011 //
00012 //===----------------------------------------------------------------------===//
00013 
00014 #include "SelectionDAGBuilder.h"
00015 #include "SDNodeDbgValue.h"
00016 #include "llvm/ADT/BitVector.h"
00017 #include "llvm/ADT/Optional.h"
00018 #include "llvm/ADT/SmallSet.h"
00019 #include "llvm/ADT/Statistic.h"
00020 #include "llvm/Analysis/AliasAnalysis.h"
00021 #include "llvm/Analysis/BranchProbabilityInfo.h"
00022 #include "llvm/Analysis/ConstantFolding.h"
00023 #include "llvm/Analysis/ValueTracking.h"
00024 #include "llvm/CodeGen/Analysis.h"
00025 #include "llvm/CodeGen/FastISel.h"
00026 #include "llvm/CodeGen/FunctionLoweringInfo.h"
00027 #include "llvm/CodeGen/GCMetadata.h"
00028 #include "llvm/CodeGen/GCStrategy.h"
00029 #include "llvm/CodeGen/MachineFrameInfo.h"
00030 #include "llvm/CodeGen/MachineFunction.h"
00031 #include "llvm/CodeGen/MachineInstrBuilder.h"
00032 #include "llvm/CodeGen/MachineJumpTableInfo.h"
00033 #include "llvm/CodeGen/MachineModuleInfo.h"
00034 #include "llvm/CodeGen/MachineRegisterInfo.h"
00035 #include "llvm/CodeGen/SelectionDAG.h"
00036 #include "llvm/CodeGen/StackMaps.h"
00037 #include "llvm/IR/CallingConv.h"
00038 #include "llvm/IR/Constants.h"
00039 #include "llvm/IR/DataLayout.h"
00040 #include "llvm/IR/DebugInfo.h"
00041 #include "llvm/IR/DerivedTypes.h"
00042 #include "llvm/IR/Function.h"
00043 #include "llvm/IR/GlobalVariable.h"
00044 #include "llvm/IR/InlineAsm.h"
00045 #include "llvm/IR/Instructions.h"
00046 #include "llvm/IR/IntrinsicInst.h"
00047 #include "llvm/IR/Intrinsics.h"
00048 #include "llvm/IR/LLVMContext.h"
00049 #include "llvm/IR/Module.h"
00050 #include "llvm/IR/Statepoint.h"
00051 #include "llvm/Support/CommandLine.h"
00052 #include "llvm/Support/Debug.h"
00053 #include "llvm/Support/ErrorHandling.h"
00054 #include "llvm/Support/MathExtras.h"
00055 #include "llvm/Support/raw_ostream.h"
00056 #include "llvm/Target/TargetFrameLowering.h"
00057 #include "llvm/Target/TargetInstrInfo.h"
00058 #include "llvm/Target/TargetIntrinsicInfo.h"
00059 #include "llvm/Target/TargetLibraryInfo.h"
00060 #include "llvm/Target/TargetLowering.h"
00061 #include "llvm/Target/TargetOptions.h"
00062 #include "llvm/Target/TargetSelectionDAGInfo.h"
00063 #include "llvm/Target/TargetSubtargetInfo.h"
00064 #include <algorithm>
00065 using namespace llvm;
00066 
00067 #define DEBUG_TYPE "isel"
00068 
00069 /// LimitFloatPrecision - Generate low-precision inline sequences for
00070 /// some float libcalls (6, 8 or 12 bits).
00071 static unsigned LimitFloatPrecision;
00072 
00073 static cl::opt<unsigned, true>
00074 LimitFPPrecision("limit-float-precision",
00075                  cl::desc("Generate low-precision inline sequences "
00076                           "for some float libcalls"),
00077                  cl::location(LimitFloatPrecision),
00078                  cl::init(0));
00079 
00080 // Limit the width of DAG chains. This is important in general to prevent
00081 // prevent DAG-based analysis from blowing up. For example, alias analysis and
00082 // load clustering may not complete in reasonable time. It is difficult to
00083 // recognize and avoid this situation within each individual analysis, and
00084 // future analyses are likely to have the same behavior. Limiting DAG width is
00085 // the safe approach, and will be especially important with global DAGs.
00086 //
00087 // MaxParallelChains default is arbitrarily high to avoid affecting
00088 // optimization, but could be lowered to improve compile time. Any ld-ld-st-st
00089 // sequence over this should have been converted to llvm.memcpy by the
00090 // frontend. It easy to induce this behavior with .ll code such as:
00091 // %buffer = alloca [4096 x i8]
00092 // %data = load [4096 x i8]* %argPtr
00093 // store [4096 x i8] %data, [4096 x i8]* %buffer
00094 static const unsigned MaxParallelChains = 64;
00095 
00096 static SDValue getCopyFromPartsVector(SelectionDAG &DAG, SDLoc DL,
00097                                       const SDValue *Parts, unsigned NumParts,
00098                                       MVT PartVT, EVT ValueVT, const Value *V);
00099 
00100 /// getCopyFromParts - Create a value that contains the specified legal parts
00101 /// combined into the value they represent.  If the parts combine to a type
00102 /// larger then ValueVT then AssertOp can be used to specify whether the extra
00103 /// bits are known to be zero (ISD::AssertZext) or sign extended from ValueVT
00104 /// (ISD::AssertSext).
00105 static SDValue getCopyFromParts(SelectionDAG &DAG, SDLoc DL,
00106                                 const SDValue *Parts,
00107                                 unsigned NumParts, MVT PartVT, EVT ValueVT,
00108                                 const Value *V,
00109                                 ISD::NodeType AssertOp = ISD::DELETED_NODE) {
00110   if (ValueVT.isVector())
00111     return getCopyFromPartsVector(DAG, DL, Parts, NumParts,
00112                                   PartVT, ValueVT, V);
00113 
00114   assert(NumParts > 0 && "No parts to assemble!");
00115   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
00116   SDValue Val = Parts[0];
00117 
00118   if (NumParts > 1) {
00119     // Assemble the value from multiple parts.
00120     if (ValueVT.isInteger()) {
00121       unsigned PartBits = PartVT.getSizeInBits();
00122       unsigned ValueBits = ValueVT.getSizeInBits();
00123 
00124       // Assemble the power of 2 part.
00125       unsigned RoundParts = NumParts & (NumParts - 1) ?
00126         1 << Log2_32(NumParts) : NumParts;
00127       unsigned RoundBits = PartBits * RoundParts;
00128       EVT RoundVT = RoundBits == ValueBits ?
00129         ValueVT : EVT::getIntegerVT(*DAG.getContext(), RoundBits);
00130       SDValue Lo, Hi;
00131 
00132       EVT HalfVT = EVT::getIntegerVT(*DAG.getContext(), RoundBits/2);
00133 
00134       if (RoundParts > 2) {
00135         Lo = getCopyFromParts(DAG, DL, Parts, RoundParts / 2,
00136                               PartVT, HalfVT, V);
00137         Hi = getCopyFromParts(DAG, DL, Parts + RoundParts / 2,
00138                               RoundParts / 2, PartVT, HalfVT, V);
00139       } else {
00140         Lo = DAG.getNode(ISD::BITCAST, DL, HalfVT, Parts[0]);
00141         Hi = DAG.getNode(ISD::BITCAST, DL, HalfVT, Parts[1]);
00142       }
00143 
00144       if (TLI.isBigEndian())
00145         std::swap(Lo, Hi);
00146 
00147       Val = DAG.getNode(ISD::BUILD_PAIR, DL, RoundVT, Lo, Hi);
00148 
00149       if (RoundParts < NumParts) {
00150         // Assemble the trailing non-power-of-2 part.
00151         unsigned OddParts = NumParts - RoundParts;
00152         EVT OddVT = EVT::getIntegerVT(*DAG.getContext(), OddParts * PartBits);
00153         Hi = getCopyFromParts(DAG, DL,
00154                               Parts + RoundParts, OddParts, PartVT, OddVT, V);
00155 
00156         // Combine the round and odd parts.
00157         Lo = Val;
00158         if (TLI.isBigEndian())
00159           std::swap(Lo, Hi);
00160         EVT TotalVT = EVT::getIntegerVT(*DAG.getContext(), NumParts * PartBits);
00161         Hi = DAG.getNode(ISD::ANY_EXTEND, DL, TotalVT, Hi);
00162         Hi = DAG.getNode(ISD::SHL, DL, TotalVT, Hi,
00163                          DAG.getConstant(Lo.getValueType().getSizeInBits(),
00164                                          TLI.getPointerTy()));
00165         Lo = DAG.getNode(ISD::ZERO_EXTEND, DL, TotalVT, Lo);
00166         Val = DAG.getNode(ISD::OR, DL, TotalVT, Lo, Hi);
00167       }
00168     } else if (PartVT.isFloatingPoint()) {
00169       // FP split into multiple FP parts (for ppcf128)
00170       assert(ValueVT == EVT(MVT::ppcf128) && PartVT == MVT::f64 &&
00171              "Unexpected split");
00172       SDValue Lo, Hi;
00173       Lo = DAG.getNode(ISD::BITCAST, DL, EVT(MVT::f64), Parts[0]);
00174       Hi = DAG.getNode(ISD::BITCAST, DL, EVT(MVT::f64), Parts[1]);
00175       if (TLI.hasBigEndianPartOrdering(ValueVT))
00176         std::swap(Lo, Hi);
00177       Val = DAG.getNode(ISD::BUILD_PAIR, DL, ValueVT, Lo, Hi);
00178     } else {
00179       // FP split into integer parts (soft fp)
00180       assert(ValueVT.isFloatingPoint() && PartVT.isInteger() &&
00181              !PartVT.isVector() && "Unexpected split");
00182       EVT IntVT = EVT::getIntegerVT(*DAG.getContext(), ValueVT.getSizeInBits());
00183       Val = getCopyFromParts(DAG, DL, Parts, NumParts, PartVT, IntVT, V);
00184     }
00185   }
00186 
00187   // There is now one part, held in Val.  Correct it to match ValueVT.
00188   EVT PartEVT = Val.getValueType();
00189 
00190   if (PartEVT == ValueVT)
00191     return Val;
00192 
00193   if (PartEVT.isInteger() && ValueVT.isInteger()) {
00194     if (ValueVT.bitsLT(PartEVT)) {
00195       // For a truncate, see if we have any information to
00196       // indicate whether the truncated bits will always be
00197       // zero or sign-extension.
00198       if (AssertOp != ISD::DELETED_NODE)
00199         Val = DAG.getNode(AssertOp, DL, PartEVT, Val,
00200                           DAG.getValueType(ValueVT));
00201       return DAG.getNode(ISD::TRUNCATE, DL, ValueVT, Val);
00202     }
00203     return DAG.getNode(ISD::ANY_EXTEND, DL, ValueVT, Val);
00204   }
00205 
00206   if (PartEVT.isFloatingPoint() && ValueVT.isFloatingPoint()) {
00207     // FP_ROUND's are always exact here.
00208     if (ValueVT.bitsLT(Val.getValueType()))
00209       return DAG.getNode(ISD::FP_ROUND, DL, ValueVT, Val,
00210                          DAG.getTargetConstant(1, TLI.getPointerTy()));
00211 
00212     return DAG.getNode(ISD::FP_EXTEND, DL, ValueVT, Val);
00213   }
00214 
00215   if (PartEVT.getSizeInBits() == ValueVT.getSizeInBits())
00216     return DAG.getNode(ISD::BITCAST, DL, ValueVT, Val);
00217 
00218   llvm_unreachable("Unknown mismatch!");
00219 }
00220 
00221 static void diagnosePossiblyInvalidConstraint(LLVMContext &Ctx, const Value *V,
00222                                               const Twine &ErrMsg) {
00223   const Instruction *I = dyn_cast_or_null<Instruction>(V);
00224   if (!V)
00225     return Ctx.emitError(ErrMsg);
00226 
00227   const char *AsmError = ", possible invalid constraint for vector type";
00228   if (const CallInst *CI = dyn_cast<CallInst>(I))
00229     if (isa<InlineAsm>(CI->getCalledValue()))
00230       return Ctx.emitError(I, ErrMsg + AsmError);
00231 
00232   return Ctx.emitError(I, ErrMsg);
00233 }
00234 
00235 /// getCopyFromPartsVector - Create a value that contains the specified legal
00236 /// parts combined into the value they represent.  If the parts combine to a
00237 /// type larger then ValueVT then AssertOp can be used to specify whether the
00238 /// extra bits are known to be zero (ISD::AssertZext) or sign extended from
00239 /// ValueVT (ISD::AssertSext).
00240 static SDValue getCopyFromPartsVector(SelectionDAG &DAG, SDLoc DL,
00241                                       const SDValue *Parts, unsigned NumParts,
00242                                       MVT PartVT, EVT ValueVT, const Value *V) {
00243   assert(ValueVT.isVector() && "Not a vector value");
00244   assert(NumParts > 0 && "No parts to assemble!");
00245   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
00246   SDValue Val = Parts[0];
00247 
00248   // Handle a multi-element vector.
00249   if (NumParts > 1) {
00250     EVT IntermediateVT;
00251     MVT RegisterVT;
00252     unsigned NumIntermediates;
00253     unsigned NumRegs =
00254     TLI.getVectorTypeBreakdown(*DAG.getContext(), ValueVT, IntermediateVT,
00255                                NumIntermediates, RegisterVT);
00256     assert(NumRegs == NumParts && "Part count doesn't match vector breakdown!");
00257     NumParts = NumRegs; // Silence a compiler warning.
00258     assert(RegisterVT == PartVT && "Part type doesn't match vector breakdown!");
00259     assert(RegisterVT == Parts[0].getSimpleValueType() &&
00260            "Part type doesn't match part!");
00261 
00262     // Assemble the parts into intermediate operands.
00263     SmallVector<SDValue, 8> Ops(NumIntermediates);
00264     if (NumIntermediates == NumParts) {
00265       // If the register was not expanded, truncate or copy the value,
00266       // as appropriate.
00267       for (unsigned i = 0; i != NumParts; ++i)
00268         Ops[i] = getCopyFromParts(DAG, DL, &Parts[i], 1,
00269                                   PartVT, IntermediateVT, V);
00270     } else if (NumParts > 0) {
00271       // If the intermediate type was expanded, build the intermediate
00272       // operands from the parts.
00273       assert(NumParts % NumIntermediates == 0 &&
00274              "Must expand into a divisible number of parts!");
00275       unsigned Factor = NumParts / NumIntermediates;
00276       for (unsigned i = 0; i != NumIntermediates; ++i)
00277         Ops[i] = getCopyFromParts(DAG, DL, &Parts[i * Factor], Factor,
00278                                   PartVT, IntermediateVT, V);
00279     }
00280 
00281     // Build a vector with BUILD_VECTOR or CONCAT_VECTORS from the
00282     // intermediate operands.
00283     Val = DAG.getNode(IntermediateVT.isVector() ? ISD::CONCAT_VECTORS
00284                                                 : ISD::BUILD_VECTOR,
00285                       DL, ValueVT, Ops);
00286   }
00287 
00288   // There is now one part, held in Val.  Correct it to match ValueVT.
00289   EVT PartEVT = Val.getValueType();
00290 
00291   if (PartEVT == ValueVT)
00292     return Val;
00293 
00294   if (PartEVT.isVector()) {
00295     // If the element type of the source/dest vectors are the same, but the
00296     // parts vector has more elements than the value vector, then we have a
00297     // vector widening case (e.g. <2 x float> -> <4 x float>).  Extract the
00298     // elements we want.
00299     if (PartEVT.getVectorElementType() == ValueVT.getVectorElementType()) {
00300       assert(PartEVT.getVectorNumElements() > ValueVT.getVectorNumElements() &&
00301              "Cannot narrow, it would be a lossy transformation");
00302       return DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, ValueVT, Val,
00303                          DAG.getConstant(0, TLI.getVectorIdxTy()));
00304     }
00305 
00306     // Vector/Vector bitcast.
00307     if (ValueVT.getSizeInBits() == PartEVT.getSizeInBits())
00308       return DAG.getNode(ISD::BITCAST, DL, ValueVT, Val);
00309 
00310     assert(PartEVT.getVectorNumElements() == ValueVT.getVectorNumElements() &&
00311       "Cannot handle this kind of promotion");
00312     // Promoted vector extract
00313     bool Smaller = ValueVT.bitsLE(PartEVT);
00314     return DAG.getNode((Smaller ? ISD::TRUNCATE : ISD::ANY_EXTEND),
00315                        DL, ValueVT, Val);
00316 
00317   }
00318 
00319   // Trivial bitcast if the types are the same size and the destination
00320   // vector type is legal.
00321   if (PartEVT.getSizeInBits() == ValueVT.getSizeInBits() &&
00322       TLI.isTypeLegal(ValueVT))
00323     return DAG.getNode(ISD::BITCAST, DL, ValueVT, Val);
00324 
00325   // Handle cases such as i8 -> <1 x i1>
00326   if (ValueVT.getVectorNumElements() != 1) {
00327     diagnosePossiblyInvalidConstraint(*DAG.getContext(), V,
00328                                       "non-trivial scalar-to-vector conversion");
00329     return DAG.getUNDEF(ValueVT);
00330   }
00331 
00332   if (ValueVT.getVectorNumElements() == 1 &&
00333       ValueVT.getVectorElementType() != PartEVT) {
00334     bool Smaller = ValueVT.bitsLE(PartEVT);
00335     Val = DAG.getNode((Smaller ? ISD::TRUNCATE : ISD::ANY_EXTEND),
00336                        DL, ValueVT.getScalarType(), Val);
00337   }
00338 
00339   return DAG.getNode(ISD::BUILD_VECTOR, DL, ValueVT, Val);
00340 }
00341 
00342 static void getCopyToPartsVector(SelectionDAG &DAG, SDLoc dl,
00343                                  SDValue Val, SDValue *Parts, unsigned NumParts,
00344                                  MVT PartVT, const Value *V);
00345 
00346 /// getCopyToParts - Create a series of nodes that contain the specified value
00347 /// split into legal parts.  If the parts contain more bits than Val, then, for
00348 /// integers, ExtendKind can be used to specify how to generate the extra bits.
00349 static void getCopyToParts(SelectionDAG &DAG, SDLoc DL,
00350                            SDValue Val, SDValue *Parts, unsigned NumParts,
00351                            MVT PartVT, const Value *V,
00352                            ISD::NodeType ExtendKind = ISD::ANY_EXTEND) {
00353   EVT ValueVT = Val.getValueType();
00354 
00355   // Handle the vector case separately.
00356   if (ValueVT.isVector())
00357     return getCopyToPartsVector(DAG, DL, Val, Parts, NumParts, PartVT, V);
00358 
00359   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
00360   unsigned PartBits = PartVT.getSizeInBits();
00361   unsigned OrigNumParts = NumParts;
00362   assert(TLI.isTypeLegal(PartVT) && "Copying to an illegal type!");
00363 
00364   if (NumParts == 0)
00365     return;
00366 
00367   assert(!ValueVT.isVector() && "Vector case handled elsewhere");
00368   EVT PartEVT = PartVT;
00369   if (PartEVT == ValueVT) {
00370     assert(NumParts == 1 && "No-op copy with multiple parts!");
00371     Parts[0] = Val;
00372     return;
00373   }
00374 
00375   if (NumParts * PartBits > ValueVT.getSizeInBits()) {
00376     // If the parts cover more bits than the value has, promote the value.
00377     if (PartVT.isFloatingPoint() && ValueVT.isFloatingPoint()) {
00378       assert(NumParts == 1 && "Do not know what to promote to!");
00379       Val = DAG.getNode(ISD::FP_EXTEND, DL, PartVT, Val);
00380     } else {
00381       assert((PartVT.isInteger() || PartVT == MVT::x86mmx) &&
00382              ValueVT.isInteger() &&
00383              "Unknown mismatch!");
00384       ValueVT = EVT::getIntegerVT(*DAG.getContext(), NumParts * PartBits);
00385       Val = DAG.getNode(ExtendKind, DL, ValueVT, Val);
00386       if (PartVT == MVT::x86mmx)
00387         Val = DAG.getNode(ISD::BITCAST, DL, PartVT, Val);
00388     }
00389   } else if (PartBits == ValueVT.getSizeInBits()) {
00390     // Different types of the same size.
00391     assert(NumParts == 1 && PartEVT != ValueVT);
00392     Val = DAG.getNode(ISD::BITCAST, DL, PartVT, Val);
00393   } else if (NumParts * PartBits < ValueVT.getSizeInBits()) {
00394     // If the parts cover less bits than value has, truncate the value.
00395     assert((PartVT.isInteger() || PartVT == MVT::x86mmx) &&
00396            ValueVT.isInteger() &&
00397            "Unknown mismatch!");
00398     ValueVT = EVT::getIntegerVT(*DAG.getContext(), NumParts * PartBits);
00399     Val = DAG.getNode(ISD::TRUNCATE, DL, ValueVT, Val);
00400     if (PartVT == MVT::x86mmx)
00401       Val = DAG.getNode(ISD::BITCAST, DL, PartVT, Val);
00402   }
00403 
00404   // The value may have changed - recompute ValueVT.
00405   ValueVT = Val.getValueType();
00406   assert(NumParts * PartBits == ValueVT.getSizeInBits() &&
00407          "Failed to tile the value with PartVT!");
00408 
00409   if (NumParts == 1) {
00410     if (PartEVT != ValueVT)
00411       diagnosePossiblyInvalidConstraint(*DAG.getContext(), V,
00412                                         "scalar-to-vector conversion failed");
00413 
00414     Parts[0] = Val;
00415     return;
00416   }
00417 
00418   // Expand the value into multiple parts.
00419   if (NumParts & (NumParts - 1)) {
00420     // The number of parts is not a power of 2.  Split off and copy the tail.
00421     assert(PartVT.isInteger() && ValueVT.isInteger() &&
00422            "Do not know what to expand to!");
00423     unsigned RoundParts = 1 << Log2_32(NumParts);
00424     unsigned RoundBits = RoundParts * PartBits;
00425     unsigned OddParts = NumParts - RoundParts;
00426     SDValue OddVal = DAG.getNode(ISD::SRL, DL, ValueVT, Val,
00427                                  DAG.getIntPtrConstant(RoundBits));
00428     getCopyToParts(DAG, DL, OddVal, Parts + RoundParts, OddParts, PartVT, V);
00429 
00430     if (TLI.isBigEndian())
00431       // The odd parts were reversed by getCopyToParts - unreverse them.
00432       std::reverse(Parts + RoundParts, Parts + NumParts);
00433 
00434     NumParts = RoundParts;
00435     ValueVT = EVT::getIntegerVT(*DAG.getContext(), NumParts * PartBits);
00436     Val = DAG.getNode(ISD::TRUNCATE, DL, ValueVT, Val);
00437   }
00438 
00439   // The number of parts is a power of 2.  Repeatedly bisect the value using
00440   // EXTRACT_ELEMENT.
00441   Parts[0] = DAG.getNode(ISD::BITCAST, DL,
00442                          EVT::getIntegerVT(*DAG.getContext(),
00443                                            ValueVT.getSizeInBits()),
00444                          Val);
00445 
00446   for (unsigned StepSize = NumParts; StepSize > 1; StepSize /= 2) {
00447     for (unsigned i = 0; i < NumParts; i += StepSize) {
00448       unsigned ThisBits = StepSize * PartBits / 2;
00449       EVT ThisVT = EVT::getIntegerVT(*DAG.getContext(), ThisBits);
00450       SDValue &Part0 = Parts[i];
00451       SDValue &Part1 = Parts[i+StepSize/2];
00452 
00453       Part1 = DAG.getNode(ISD::EXTRACT_ELEMENT, DL,
00454                           ThisVT, Part0, DAG.getIntPtrConstant(1));
00455       Part0 = DAG.getNode(ISD::EXTRACT_ELEMENT, DL,
00456                           ThisVT, Part0, DAG.getIntPtrConstant(0));
00457 
00458       if (ThisBits == PartBits && ThisVT != PartVT) {
00459         Part0 = DAG.getNode(ISD::BITCAST, DL, PartVT, Part0);
00460         Part1 = DAG.getNode(ISD::BITCAST, DL, PartVT, Part1);
00461       }
00462     }
00463   }
00464 
00465   if (TLI.isBigEndian())
00466     std::reverse(Parts, Parts + OrigNumParts);
00467 }
00468 
00469 
00470 /// getCopyToPartsVector - Create a series of nodes that contain the specified
00471 /// value split into legal parts.
00472 static void getCopyToPartsVector(SelectionDAG &DAG, SDLoc DL,
00473                                  SDValue Val, SDValue *Parts, unsigned NumParts,
00474                                  MVT PartVT, const Value *V) {
00475   EVT ValueVT = Val.getValueType();
00476   assert(ValueVT.isVector() && "Not a vector");
00477   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
00478 
00479   if (NumParts == 1) {
00480     EVT PartEVT = PartVT;
00481     if (PartEVT == ValueVT) {
00482       // Nothing to do.
00483     } else if (PartVT.getSizeInBits() == ValueVT.getSizeInBits()) {
00484       // Bitconvert vector->vector case.
00485       Val = DAG.getNode(ISD::BITCAST, DL, PartVT, Val);
00486     } else if (PartVT.isVector() &&
00487                PartEVT.getVectorElementType() == ValueVT.getVectorElementType() &&
00488                PartEVT.getVectorNumElements() > ValueVT.getVectorNumElements()) {
00489       EVT ElementVT = PartVT.getVectorElementType();
00490       // Vector widening case, e.g. <2 x float> -> <4 x float>.  Shuffle in
00491       // undef elements.
00492       SmallVector<SDValue, 16> Ops;
00493       for (unsigned i = 0, e = ValueVT.getVectorNumElements(); i != e; ++i)
00494         Ops.push_back(DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL,
00495                                   ElementVT, Val, DAG.getConstant(i,
00496                                                   TLI.getVectorIdxTy())));
00497 
00498       for (unsigned i = ValueVT.getVectorNumElements(),
00499            e = PartVT.getVectorNumElements(); i != e; ++i)
00500         Ops.push_back(DAG.getUNDEF(ElementVT));
00501 
00502       Val = DAG.getNode(ISD::BUILD_VECTOR, DL, PartVT, Ops);
00503 
00504       // FIXME: Use CONCAT for 2x -> 4x.
00505 
00506       //SDValue UndefElts = DAG.getUNDEF(VectorTy);
00507       //Val = DAG.getNode(ISD::CONCAT_VECTORS, DL, PartVT, Val, UndefElts);
00508     } else if (PartVT.isVector() &&
00509                PartEVT.getVectorElementType().bitsGE(
00510                  ValueVT.getVectorElementType()) &&
00511                PartEVT.getVectorNumElements() == ValueVT.getVectorNumElements()) {
00512 
00513       // Promoted vector extract
00514       bool Smaller = PartEVT.bitsLE(ValueVT);
00515       Val = DAG.getNode((Smaller ? ISD::TRUNCATE : ISD::ANY_EXTEND),
00516                         DL, PartVT, Val);
00517     } else{
00518       // Vector -> scalar conversion.
00519       assert(ValueVT.getVectorNumElements() == 1 &&
00520              "Only trivial vector-to-scalar conversions should get here!");
00521       Val = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL,
00522                         PartVT, Val, DAG.getConstant(0, TLI.getVectorIdxTy()));
00523 
00524       bool Smaller = ValueVT.bitsLE(PartVT);
00525       Val = DAG.getNode((Smaller ? ISD::TRUNCATE : ISD::ANY_EXTEND),
00526                          DL, PartVT, Val);
00527     }
00528 
00529     Parts[0] = Val;
00530     return;
00531   }
00532 
00533   // Handle a multi-element vector.
00534   EVT IntermediateVT;
00535   MVT RegisterVT;
00536   unsigned NumIntermediates;
00537   unsigned NumRegs = TLI.getVectorTypeBreakdown(*DAG.getContext(), ValueVT,
00538                                                 IntermediateVT,
00539                                                 NumIntermediates, RegisterVT);
00540   unsigned NumElements = ValueVT.getVectorNumElements();
00541 
00542   assert(NumRegs == NumParts && "Part count doesn't match vector breakdown!");
00543   NumParts = NumRegs; // Silence a compiler warning.
00544   assert(RegisterVT == PartVT && "Part type doesn't match vector breakdown!");
00545 
00546   // Split the vector into intermediate operands.
00547   SmallVector<SDValue, 8> Ops(NumIntermediates);
00548   for (unsigned i = 0; i != NumIntermediates; ++i) {
00549     if (IntermediateVT.isVector())
00550       Ops[i] = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL,
00551                            IntermediateVT, Val,
00552                    DAG.getConstant(i * (NumElements / NumIntermediates),
00553                                    TLI.getVectorIdxTy()));
00554     else
00555       Ops[i] = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL,
00556                            IntermediateVT, Val,
00557                            DAG.getConstant(i, TLI.getVectorIdxTy()));
00558   }
00559 
00560   // Split the intermediate operands into legal parts.
00561   if (NumParts == NumIntermediates) {
00562     // If the register was not expanded, promote or copy the value,
00563     // as appropriate.
00564     for (unsigned i = 0; i != NumParts; ++i)
00565       getCopyToParts(DAG, DL, Ops[i], &Parts[i], 1, PartVT, V);
00566   } else if (NumParts > 0) {
00567     // If the intermediate type was expanded, split each the value into
00568     // legal parts.
00569     assert(NumIntermediates != 0 && "division by zero");
00570     assert(NumParts % NumIntermediates == 0 &&
00571            "Must expand into a divisible number of parts!");
00572     unsigned Factor = NumParts / NumIntermediates;
00573     for (unsigned i = 0; i != NumIntermediates; ++i)
00574       getCopyToParts(DAG, DL, Ops[i], &Parts[i*Factor], Factor, PartVT, V);
00575   }
00576 }
00577 
00578 namespace {
00579   /// RegsForValue - This struct represents the registers (physical or virtual)
00580   /// that a particular set of values is assigned, and the type information
00581   /// about the value. The most common situation is to represent one value at a
00582   /// time, but struct or array values are handled element-wise as multiple
00583   /// values.  The splitting of aggregates is performed recursively, so that we
00584   /// never have aggregate-typed registers. The values at this point do not
00585   /// necessarily have legal types, so each value may require one or more
00586   /// registers of some legal type.
00587   ///
00588   struct RegsForValue {
00589     /// ValueVTs - The value types of the values, which may not be legal, and
00590     /// may need be promoted or synthesized from one or more registers.
00591     ///
00592     SmallVector<EVT, 4> ValueVTs;
00593 
00594     /// RegVTs - The value types of the registers. This is the same size as
00595     /// ValueVTs and it records, for each value, what the type of the assigned
00596     /// register or registers are. (Individual values are never synthesized
00597     /// from more than one type of register.)
00598     ///
00599     /// With virtual registers, the contents of RegVTs is redundant with TLI's
00600     /// getRegisterType member function, however when with physical registers
00601     /// it is necessary to have a separate record of the types.
00602     ///
00603     SmallVector<MVT, 4> RegVTs;
00604 
00605     /// Regs - This list holds the registers assigned to the values.
00606     /// Each legal or promoted value requires one register, and each
00607     /// expanded value requires multiple registers.
00608     ///
00609     SmallVector<unsigned, 4> Regs;
00610 
00611     RegsForValue() {}
00612 
00613     RegsForValue(const SmallVector<unsigned, 4> &regs,
00614                  MVT regvt, EVT valuevt)
00615       : ValueVTs(1, valuevt), RegVTs(1, regvt), Regs(regs) {}
00616 
00617     RegsForValue(LLVMContext &Context, const TargetLowering &tli,
00618                  unsigned Reg, Type *Ty) {
00619       ComputeValueVTs(tli, Ty, ValueVTs);
00620 
00621       for (unsigned Value = 0, e = ValueVTs.size(); Value != e; ++Value) {
00622         EVT ValueVT = ValueVTs[Value];
00623         unsigned NumRegs = tli.getNumRegisters(Context, ValueVT);
00624         MVT RegisterVT = tli.getRegisterType(Context, ValueVT);
00625         for (unsigned i = 0; i != NumRegs; ++i)
00626           Regs.push_back(Reg + i);
00627         RegVTs.push_back(RegisterVT);
00628         Reg += NumRegs;
00629       }
00630     }
00631 
00632     /// append - Add the specified values to this one.
00633     void append(const RegsForValue &RHS) {
00634       ValueVTs.append(RHS.ValueVTs.begin(), RHS.ValueVTs.end());
00635       RegVTs.append(RHS.RegVTs.begin(), RHS.RegVTs.end());
00636       Regs.append(RHS.Regs.begin(), RHS.Regs.end());
00637     }
00638 
00639     /// getCopyFromRegs - Emit a series of CopyFromReg nodes that copies from
00640     /// this value and returns the result as a ValueVTs value.  This uses
00641     /// Chain/Flag as the input and updates them for the output Chain/Flag.
00642     /// If the Flag pointer is NULL, no flag is used.
00643     SDValue getCopyFromRegs(SelectionDAG &DAG, FunctionLoweringInfo &FuncInfo,
00644                             SDLoc dl,
00645                             SDValue &Chain, SDValue *Flag,
00646                             const Value *V = nullptr) const;
00647 
00648     /// getCopyToRegs - Emit a series of CopyToReg nodes that copies the
00649     /// specified value into the registers specified by this object.  This uses
00650     /// Chain/Flag as the input and updates them for the output Chain/Flag.
00651     /// If the Flag pointer is NULL, no flag is used.
00652     void
00653     getCopyToRegs(SDValue Val, SelectionDAG &DAG, SDLoc dl, SDValue &Chain,
00654                   SDValue *Flag, const Value *V,
00655                   ISD::NodeType PreferredExtendType = ISD::ANY_EXTEND) const;
00656 
00657     /// AddInlineAsmOperands - Add this value to the specified inlineasm node
00658     /// operand list.  This adds the code marker, matching input operand index
00659     /// (if applicable), and includes the number of values added into it.
00660     void AddInlineAsmOperands(unsigned Kind,
00661                               bool HasMatching, unsigned MatchingIdx,
00662                               SelectionDAG &DAG,
00663                               std::vector<SDValue> &Ops) const;
00664   };
00665 }
00666 
00667 /// getCopyFromRegs - Emit a series of CopyFromReg nodes that copies from
00668 /// this value and returns the result as a ValueVT value.  This uses
00669 /// Chain/Flag as the input and updates them for the output Chain/Flag.
00670 /// If the Flag pointer is NULL, no flag is used.
00671 SDValue RegsForValue::getCopyFromRegs(SelectionDAG &DAG,
00672                                       FunctionLoweringInfo &FuncInfo,
00673                                       SDLoc dl,
00674                                       SDValue &Chain, SDValue *Flag,
00675                                       const Value *V) const {
00676   // A Value with type {} or [0 x %t] needs no registers.
00677   if (ValueVTs.empty())
00678     return SDValue();
00679 
00680   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
00681 
00682   // Assemble the legal parts into the final values.
00683   SmallVector<SDValue, 4> Values(ValueVTs.size());
00684   SmallVector<SDValue, 8> Parts;
00685   for (unsigned Value = 0, Part = 0, e = ValueVTs.size(); Value != e; ++Value) {
00686     // Copy the legal parts from the registers.
00687     EVT ValueVT = ValueVTs[Value];
00688     unsigned NumRegs = TLI.getNumRegisters(*DAG.getContext(), ValueVT);
00689     MVT RegisterVT = RegVTs[Value];
00690 
00691     Parts.resize(NumRegs);
00692     for (unsigned i = 0; i != NumRegs; ++i) {
00693       SDValue P;
00694       if (!Flag) {
00695         P = DAG.getCopyFromReg(Chain, dl, Regs[Part+i], RegisterVT);
00696       } else {
00697         P = DAG.getCopyFromReg(Chain, dl, Regs[Part+i], RegisterVT, *Flag);
00698         *Flag = P.getValue(2);
00699       }
00700 
00701       Chain = P.getValue(1);
00702       Parts[i] = P;
00703 
00704       // If the source register was virtual and if we know something about it,
00705       // add an assert node.
00706       if (!TargetRegisterInfo::isVirtualRegister(Regs[Part+i]) ||
00707           !RegisterVT.isInteger() || RegisterVT.isVector())
00708         continue;
00709 
00710       const FunctionLoweringInfo::LiveOutInfo *LOI =
00711         FuncInfo.GetLiveOutRegInfo(Regs[Part+i]);
00712       if (!LOI)
00713         continue;
00714 
00715       unsigned RegSize = RegisterVT.getSizeInBits();
00716       unsigned NumSignBits = LOI->NumSignBits;
00717       unsigned NumZeroBits = LOI->KnownZero.countLeadingOnes();
00718 
00719       if (NumZeroBits == RegSize) {
00720         // The current value is a zero.
00721         // Explicitly express that as it would be easier for
00722         // optimizations to kick in.
00723         Parts[i] = DAG.getConstant(0, RegisterVT);
00724         continue;
00725       }
00726 
00727       // FIXME: We capture more information than the dag can represent.  For
00728       // now, just use the tightest assertzext/assertsext possible.
00729       bool isSExt = true;
00730       EVT FromVT(MVT::Other);
00731       if (NumSignBits == RegSize)
00732         isSExt = true, FromVT = MVT::i1;   // ASSERT SEXT 1
00733       else if (NumZeroBits >= RegSize-1)
00734         isSExt = false, FromVT = MVT::i1;  // ASSERT ZEXT 1
00735       else if (NumSignBits > RegSize-8)
00736         isSExt = true, FromVT = MVT::i8;   // ASSERT SEXT 8
00737       else if (NumZeroBits >= RegSize-8)
00738         isSExt = false, FromVT = MVT::i8;  // ASSERT ZEXT 8
00739       else if (NumSignBits > RegSize-16)
00740         isSExt = true, FromVT = MVT::i16;  // ASSERT SEXT 16
00741       else if (NumZeroBits >= RegSize-16)
00742         isSExt = false, FromVT = MVT::i16; // ASSERT ZEXT 16
00743       else if (NumSignBits > RegSize-32)
00744         isSExt = true, FromVT = MVT::i32;  // ASSERT SEXT 32
00745       else if (NumZeroBits >= RegSize-32)
00746         isSExt = false, FromVT = MVT::i32; // ASSERT ZEXT 32
00747       else
00748         continue;
00749 
00750       // Add an assertion node.
00751       assert(FromVT != MVT::Other);
00752       Parts[i] = DAG.getNode(isSExt ? ISD::AssertSext : ISD::AssertZext, dl,
00753                              RegisterVT, P, DAG.getValueType(FromVT));
00754     }
00755 
00756     Values[Value] = getCopyFromParts(DAG, dl, Parts.begin(),
00757                                      NumRegs, RegisterVT, ValueVT, V);
00758     Part += NumRegs;
00759     Parts.clear();
00760   }
00761 
00762   return DAG.getNode(ISD::MERGE_VALUES, dl, DAG.getVTList(ValueVTs), Values);
00763 }
00764 
00765 /// getCopyToRegs - Emit a series of CopyToReg nodes that copies the
00766 /// specified value into the registers specified by this object.  This uses
00767 /// Chain/Flag as the input and updates them for the output Chain/Flag.
00768 /// If the Flag pointer is NULL, no flag is used.
00769 void RegsForValue::getCopyToRegs(SDValue Val, SelectionDAG &DAG, SDLoc dl,
00770                                  SDValue &Chain, SDValue *Flag, const Value *V,
00771                                  ISD::NodeType PreferredExtendType) const {
00772   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
00773   ISD::NodeType ExtendKind = PreferredExtendType;
00774 
00775   // Get the list of the values's legal parts.
00776   unsigned NumRegs = Regs.size();
00777   SmallVector<SDValue, 8> Parts(NumRegs);
00778   for (unsigned Value = 0, Part = 0, e = ValueVTs.size(); Value != e; ++Value) {
00779     EVT ValueVT = ValueVTs[Value];
00780     unsigned NumParts = TLI.getNumRegisters(*DAG.getContext(), ValueVT);
00781     MVT RegisterVT = RegVTs[Value];
00782 
00783     if (ExtendKind == ISD::ANY_EXTEND && TLI.isZExtFree(Val, RegisterVT))
00784       ExtendKind = ISD::ZERO_EXTEND;
00785 
00786     getCopyToParts(DAG, dl, Val.getValue(Val.getResNo() + Value),
00787                    &Parts[Part], NumParts, RegisterVT, V, ExtendKind);
00788     Part += NumParts;
00789   }
00790 
00791   // Copy the parts into the registers.
00792   SmallVector<SDValue, 8> Chains(NumRegs);
00793   for (unsigned i = 0; i != NumRegs; ++i) {
00794     SDValue Part;
00795     if (!Flag) {
00796       Part = DAG.getCopyToReg(Chain, dl, Regs[i], Parts[i]);
00797     } else {
00798       Part = DAG.getCopyToReg(Chain, dl, Regs[i], Parts[i], *Flag);
00799       *Flag = Part.getValue(1);
00800     }
00801 
00802     Chains[i] = Part.getValue(0);
00803   }
00804 
00805   if (NumRegs == 1 || Flag)
00806     // If NumRegs > 1 && Flag is used then the use of the last CopyToReg is
00807     // flagged to it. That is the CopyToReg nodes and the user are considered
00808     // a single scheduling unit. If we create a TokenFactor and return it as
00809     // chain, then the TokenFactor is both a predecessor (operand) of the
00810     // user as well as a successor (the TF operands are flagged to the user).
00811     // c1, f1 = CopyToReg
00812     // c2, f2 = CopyToReg
00813     // c3     = TokenFactor c1, c2
00814     // ...
00815     //        = op c3, ..., f2
00816     Chain = Chains[NumRegs-1];
00817   else
00818     Chain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Chains);
00819 }
00820 
00821 /// AddInlineAsmOperands - Add this value to the specified inlineasm node
00822 /// operand list.  This adds the code marker and includes the number of
00823 /// values added into it.
00824 void RegsForValue::AddInlineAsmOperands(unsigned Code, bool HasMatching,
00825                                         unsigned MatchingIdx,
00826                                         SelectionDAG &DAG,
00827                                         std::vector<SDValue> &Ops) const {
00828   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
00829 
00830   unsigned Flag = InlineAsm::getFlagWord(Code, Regs.size());
00831   if (HasMatching)
00832     Flag = InlineAsm::getFlagWordForMatchingOp(Flag, MatchingIdx);
00833   else if (!Regs.empty() &&
00834            TargetRegisterInfo::isVirtualRegister(Regs.front())) {
00835     // Put the register class of the virtual registers in the flag word.  That
00836     // way, later passes can recompute register class constraints for inline
00837     // assembly as well as normal instructions.
00838     // Don't do this for tied operands that can use the regclass information
00839     // from the def.
00840     const MachineRegisterInfo &MRI = DAG.getMachineFunction().getRegInfo();
00841     const TargetRegisterClass *RC = MRI.getRegClass(Regs.front());
00842     Flag = InlineAsm::getFlagWordForRegClass(Flag, RC->getID());
00843   }
00844 
00845   SDValue Res = DAG.getTargetConstant(Flag, MVT::i32);
00846   Ops.push_back(Res);
00847 
00848   unsigned SP = TLI.getStackPointerRegisterToSaveRestore();
00849   for (unsigned Value = 0, Reg = 0, e = ValueVTs.size(); Value != e; ++Value) {
00850     unsigned NumRegs = TLI.getNumRegisters(*DAG.getContext(), ValueVTs[Value]);
00851     MVT RegisterVT = RegVTs[Value];
00852     for (unsigned i = 0; i != NumRegs; ++i) {
00853       assert(Reg < Regs.size() && "Mismatch in # registers expected");
00854       unsigned TheReg = Regs[Reg++];
00855       Ops.push_back(DAG.getRegister(TheReg, RegisterVT));
00856 
00857       if (TheReg == SP && Code == InlineAsm::Kind_Clobber) {
00858         // If we clobbered the stack pointer, MFI should know about it.
00859         assert(DAG.getMachineFunction().getFrameInfo()->
00860             hasInlineAsmWithSPAdjust());
00861       }
00862     }
00863   }
00864 }
00865 
00866 void SelectionDAGBuilder::init(GCFunctionInfo *gfi, AliasAnalysis &aa,
00867                                const TargetLibraryInfo *li) {
00868   AA = &aa;
00869   GFI = gfi;
00870   LibInfo = li;
00871   DL = DAG.getSubtarget().getDataLayout();
00872   Context = DAG.getContext();
00873   LPadToCallSiteMap.clear();
00874 }
00875 
00876 /// clear - Clear out the current SelectionDAG and the associated
00877 /// state and prepare this SelectionDAGBuilder object to be used
00878 /// for a new block. This doesn't clear out information about
00879 /// additional blocks that are needed to complete switch lowering
00880 /// or PHI node updating; that information is cleared out as it is
00881 /// consumed.
00882 void SelectionDAGBuilder::clear() {
00883   NodeMap.clear();
00884   UnusedArgNodeMap.clear();
00885   PendingLoads.clear();
00886   PendingExports.clear();
00887   CurInst = nullptr;
00888   HasTailCall = false;
00889   SDNodeOrder = LowestSDNodeOrder;
00890   StatepointLowering.clear();
00891 }
00892 
00893 /// clearDanglingDebugInfo - Clear the dangling debug information
00894 /// map. This function is separated from the clear so that debug
00895 /// information that is dangling in a basic block can be properly
00896 /// resolved in a different basic block. This allows the
00897 /// SelectionDAG to resolve dangling debug information attached
00898 /// to PHI nodes.
00899 void SelectionDAGBuilder::clearDanglingDebugInfo() {
00900   DanglingDebugInfoMap.clear();
00901 }
00902 
00903 /// getRoot - Return the current virtual root of the Selection DAG,
00904 /// flushing any PendingLoad items. This must be done before emitting
00905 /// a store or any other node that may need to be ordered after any
00906 /// prior load instructions.
00907 ///
00908 SDValue SelectionDAGBuilder::getRoot() {
00909   if (PendingLoads.empty())
00910     return DAG.getRoot();
00911 
00912   if (PendingLoads.size() == 1) {
00913     SDValue Root = PendingLoads[0];
00914     DAG.setRoot(Root);
00915     PendingLoads.clear();
00916     return Root;
00917   }
00918 
00919   // Otherwise, we have to make a token factor node.
00920   SDValue Root = DAG.getNode(ISD::TokenFactor, getCurSDLoc(), MVT::Other,
00921                              PendingLoads);
00922   PendingLoads.clear();
00923   DAG.setRoot(Root);
00924   return Root;
00925 }
00926 
00927 /// getControlRoot - Similar to getRoot, but instead of flushing all the
00928 /// PendingLoad items, flush all the PendingExports items. It is necessary
00929 /// to do this before emitting a terminator instruction.
00930 ///
00931 SDValue SelectionDAGBuilder::getControlRoot() {
00932   SDValue Root = DAG.getRoot();
00933 
00934   if (PendingExports.empty())
00935     return Root;
00936 
00937   // Turn all of the CopyToReg chains into one factored node.
00938   if (Root.getOpcode() != ISD::EntryToken) {
00939     unsigned i = 0, e = PendingExports.size();
00940     for (; i != e; ++i) {
00941       assert(PendingExports[i].getNode()->getNumOperands() > 1);
00942       if (PendingExports[i].getNode()->getOperand(0) == Root)
00943         break;  // Don't add the root if we already indirectly depend on it.
00944     }
00945 
00946     if (i == e)
00947       PendingExports.push_back(Root);
00948   }
00949 
00950   Root = DAG.getNode(ISD::TokenFactor, getCurSDLoc(), MVT::Other,
00951                      PendingExports);
00952   PendingExports.clear();
00953   DAG.setRoot(Root);
00954   return Root;
00955 }
00956 
00957 void SelectionDAGBuilder::visit(const Instruction &I) {
00958   // Set up outgoing PHI node register values before emitting the terminator.
00959   if (isa<TerminatorInst>(&I))
00960     HandlePHINodesInSuccessorBlocks(I.getParent());
00961 
00962   ++SDNodeOrder;
00963 
00964   CurInst = &I;
00965 
00966   visit(I.getOpcode(), I);
00967 
00968   if (!isa<TerminatorInst>(&I) && !HasTailCall)
00969     CopyToExportRegsIfNeeded(&I);
00970 
00971   CurInst = nullptr;
00972 }
00973 
00974 void SelectionDAGBuilder::visitPHI(const PHINode &) {
00975   llvm_unreachable("SelectionDAGBuilder shouldn't visit PHI nodes!");
00976 }
00977 
00978 void SelectionDAGBuilder::visit(unsigned Opcode, const User &I) {
00979   // Note: this doesn't use InstVisitor, because it has to work with
00980   // ConstantExpr's in addition to instructions.
00981   switch (Opcode) {
00982   default: llvm_unreachable("Unknown instruction type encountered!");
00983     // Build the switch statement using the Instruction.def file.
00984 #define HANDLE_INST(NUM, OPCODE, CLASS) \
00985     case Instruction::OPCODE: visit##OPCODE((const CLASS&)I); break;
00986 #include "llvm/IR/Instruction.def"
00987   }
00988 }
00989 
00990 // resolveDanglingDebugInfo - if we saw an earlier dbg_value referring to V,
00991 // generate the debug data structures now that we've seen its definition.
00992 void SelectionDAGBuilder::resolveDanglingDebugInfo(const Value *V,
00993                                                    SDValue Val) {
00994   DanglingDebugInfo &DDI = DanglingDebugInfoMap[V];
00995   if (DDI.getDI()) {
00996     const DbgValueInst *DI = DDI.getDI();
00997     DebugLoc dl = DDI.getdl();
00998     unsigned DbgSDNodeOrder = DDI.getSDNodeOrder();
00999     MDNode *Variable = DI->getVariable();
01000     MDNode *Expr = DI->getExpression();
01001     uint64_t Offset = DI->getOffset();
01002     // A dbg.value for an alloca is always indirect.
01003     bool IsIndirect = isa<AllocaInst>(V) || Offset != 0;
01004     SDDbgValue *SDV;
01005     if (Val.getNode()) {
01006       if (!EmitFuncArgumentDbgValue(V, Variable, Expr, Offset, IsIndirect,
01007                                     Val)) {
01008         SDV = DAG.getDbgValue(Variable, Expr, Val.getNode(), Val.getResNo(),
01009                               IsIndirect, Offset, dl, DbgSDNodeOrder);
01010         DAG.AddDbgValue(SDV, Val.getNode(), false);
01011       }
01012     } else
01013       DEBUG(dbgs() << "Dropping debug info for " << *DI << "\n");
01014     DanglingDebugInfoMap[V] = DanglingDebugInfo();
01015   }
01016 }
01017 
01018 /// getValue - Return an SDValue for the given Value.
01019 SDValue SelectionDAGBuilder::getValue(const Value *V) {
01020   // If we already have an SDValue for this value, use it. It's important
01021   // to do this first, so that we don't create a CopyFromReg if we already
01022   // have a regular SDValue.
01023   SDValue &N = NodeMap[V];
01024   if (N.getNode()) return N;
01025 
01026   // If there's a virtual register allocated and initialized for this
01027   // value, use it.
01028   DenseMap<const Value *, unsigned>::iterator It = FuncInfo.ValueMap.find(V);
01029   if (It != FuncInfo.ValueMap.end()) {
01030     unsigned InReg = It->second;
01031     RegsForValue RFV(*DAG.getContext(), DAG.getTargetLoweringInfo(), InReg,
01032                      V->getType());
01033     SDValue Chain = DAG.getEntryNode();
01034     N = RFV.getCopyFromRegs(DAG, FuncInfo, getCurSDLoc(), Chain, nullptr, V);
01035     resolveDanglingDebugInfo(V, N);
01036     return N;
01037   }
01038 
01039   // Otherwise create a new SDValue and remember it.
01040   SDValue Val = getValueImpl(V);
01041   NodeMap[V] = Val;
01042   resolveDanglingDebugInfo(V, Val);
01043   return Val;
01044 }
01045 
01046 /// getNonRegisterValue - Return an SDValue for the given Value, but
01047 /// don't look in FuncInfo.ValueMap for a virtual register.
01048 SDValue SelectionDAGBuilder::getNonRegisterValue(const Value *V) {
01049   // If we already have an SDValue for this value, use it.
01050   SDValue &N = NodeMap[V];
01051   if (N.getNode()) return N;
01052 
01053   // Otherwise create a new SDValue and remember it.
01054   SDValue Val = getValueImpl(V);
01055   NodeMap[V] = Val;
01056   resolveDanglingDebugInfo(V, Val);
01057   return Val;
01058 }
01059 
01060 /// getValueImpl - Helper function for getValue and getNonRegisterValue.
01061 /// Create an SDValue for the given value.
01062 SDValue SelectionDAGBuilder::getValueImpl(const Value *V) {
01063   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
01064 
01065   if (const Constant *C = dyn_cast<Constant>(V)) {
01066     EVT VT = TLI.getValueType(V->getType(), true);
01067 
01068     if (const ConstantInt *CI = dyn_cast<ConstantInt>(C))
01069       return DAG.getConstant(*CI, VT);
01070 
01071     if (const GlobalValue *GV = dyn_cast<GlobalValue>(C))
01072       return DAG.getGlobalAddress(GV, getCurSDLoc(), VT);
01073 
01074     if (isa<ConstantPointerNull>(C)) {
01075       unsigned AS = V->getType()->getPointerAddressSpace();
01076       return DAG.getConstant(0, TLI.getPointerTy(AS));
01077     }
01078 
01079     if (const ConstantFP *CFP = dyn_cast<ConstantFP>(C))
01080       return DAG.getConstantFP(*CFP, VT);
01081 
01082     if (isa<UndefValue>(C) && !V->getType()->isAggregateType())
01083       return DAG.getUNDEF(VT);
01084 
01085     if (const ConstantExpr *CE = dyn_cast<ConstantExpr>(C)) {
01086       visit(CE->getOpcode(), *CE);
01087       SDValue N1 = NodeMap[V];
01088       assert(N1.getNode() && "visit didn't populate the NodeMap!");
01089       return N1;
01090     }
01091 
01092     if (isa<ConstantStruct>(C) || isa<ConstantArray>(C)) {
01093       SmallVector<SDValue, 4> Constants;
01094       for (User::const_op_iterator OI = C->op_begin(), OE = C->op_end();
01095            OI != OE; ++OI) {
01096         SDNode *Val = getValue(*OI).getNode();
01097         // If the operand is an empty aggregate, there are no values.
01098         if (!Val) continue;
01099         // Add each leaf value from the operand to the Constants list
01100         // to form a flattened list of all the values.
01101         for (unsigned i = 0, e = Val->getNumValues(); i != e; ++i)
01102           Constants.push_back(SDValue(Val, i));
01103       }
01104 
01105       return DAG.getMergeValues(Constants, getCurSDLoc());
01106     }
01107 
01108     if (const ConstantDataSequential *CDS =
01109           dyn_cast<ConstantDataSequential>(C)) {
01110       SmallVector<SDValue, 4> Ops;
01111       for (unsigned i = 0, e = CDS->getNumElements(); i != e; ++i) {
01112         SDNode *Val = getValue(CDS->getElementAsConstant(i)).getNode();
01113         // Add each leaf value from the operand to the Constants list
01114         // to form a flattened list of all the values.
01115         for (unsigned i = 0, e = Val->getNumValues(); i != e; ++i)
01116           Ops.push_back(SDValue(Val, i));
01117       }
01118 
01119       if (isa<ArrayType>(CDS->getType()))
01120         return DAG.getMergeValues(Ops, getCurSDLoc());
01121       return NodeMap[V] = DAG.getNode(ISD::BUILD_VECTOR, getCurSDLoc(),
01122                                       VT, Ops);
01123     }
01124 
01125     if (C->getType()->isStructTy() || C->getType()->isArrayTy()) {
01126       assert((isa<ConstantAggregateZero>(C) || isa<UndefValue>(C)) &&
01127              "Unknown struct or array constant!");
01128 
01129       SmallVector<EVT, 4> ValueVTs;
01130       ComputeValueVTs(TLI, C->getType(), ValueVTs);
01131       unsigned NumElts = ValueVTs.size();
01132       if (NumElts == 0)
01133         return SDValue(); // empty struct
01134       SmallVector<SDValue, 4> Constants(NumElts);
01135       for (unsigned i = 0; i != NumElts; ++i) {
01136         EVT EltVT = ValueVTs[i];
01137         if (isa<UndefValue>(C))
01138           Constants[i] = DAG.getUNDEF(EltVT);
01139         else if (EltVT.isFloatingPoint())
01140           Constants[i] = DAG.getConstantFP(0, EltVT);
01141         else
01142           Constants[i] = DAG.getConstant(0, EltVT);
01143       }
01144 
01145       return DAG.getMergeValues(Constants, getCurSDLoc());
01146     }
01147 
01148     if (const BlockAddress *BA = dyn_cast<BlockAddress>(C))
01149       return DAG.getBlockAddress(BA, VT);
01150 
01151     VectorType *VecTy = cast<VectorType>(V->getType());
01152     unsigned NumElements = VecTy->getNumElements();
01153 
01154     // Now that we know the number and type of the elements, get that number of
01155     // elements into the Ops array based on what kind of constant it is.
01156     SmallVector<SDValue, 16> Ops;
01157     if (const ConstantVector *CV = dyn_cast<ConstantVector>(C)) {
01158       for (unsigned i = 0; i != NumElements; ++i)
01159         Ops.push_back(getValue(CV->getOperand(i)));
01160     } else {
01161       assert(isa<ConstantAggregateZero>(C) && "Unknown vector constant!");
01162       EVT EltVT = TLI.getValueType(VecTy->getElementType());
01163 
01164       SDValue Op;
01165       if (EltVT.isFloatingPoint())
01166         Op = DAG.getConstantFP(0, EltVT);
01167       else
01168         Op = DAG.getConstant(0, EltVT);
01169       Ops.assign(NumElements, Op);
01170     }
01171 
01172     // Create a BUILD_VECTOR node.
01173     return NodeMap[V] = DAG.getNode(ISD::BUILD_VECTOR, getCurSDLoc(), VT, Ops);
01174   }
01175 
01176   // If this is a static alloca, generate it as the frameindex instead of
01177   // computation.
01178   if (const AllocaInst *AI = dyn_cast<AllocaInst>(V)) {
01179     DenseMap<const AllocaInst*, int>::iterator SI =
01180       FuncInfo.StaticAllocaMap.find(AI);
01181     if (SI != FuncInfo.StaticAllocaMap.end())
01182       return DAG.getFrameIndex(SI->second, TLI.getPointerTy());
01183   }
01184 
01185   // If this is an instruction which fast-isel has deferred, select it now.
01186   if (const Instruction *Inst = dyn_cast<Instruction>(V)) {
01187     unsigned InReg = FuncInfo.InitializeRegForValue(Inst);
01188     RegsForValue RFV(*DAG.getContext(), TLI, InReg, Inst->getType());
01189     SDValue Chain = DAG.getEntryNode();
01190     return RFV.getCopyFromRegs(DAG, FuncInfo, getCurSDLoc(), Chain, nullptr, V);
01191   }
01192 
01193   llvm_unreachable("Can't get register for value!");
01194 }
01195 
01196 void SelectionDAGBuilder::visitRet(const ReturnInst &I) {
01197   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
01198   SDValue Chain = getControlRoot();
01199   SmallVector<ISD::OutputArg, 8> Outs;
01200   SmallVector<SDValue, 8> OutVals;
01201 
01202   if (!FuncInfo.CanLowerReturn) {
01203     unsigned DemoteReg = FuncInfo.DemoteRegister;
01204     const Function *F = I.getParent()->getParent();
01205 
01206     // Emit a store of the return value through the virtual register.
01207     // Leave Outs empty so that LowerReturn won't try to load return
01208     // registers the usual way.
01209     SmallVector<EVT, 1> PtrValueVTs;
01210     ComputeValueVTs(TLI, PointerType::getUnqual(F->getReturnType()),
01211                     PtrValueVTs);
01212 
01213     SDValue RetPtr = DAG.getRegister(DemoteReg, PtrValueVTs[0]);
01214     SDValue RetOp = getValue(I.getOperand(0));
01215 
01216     SmallVector<EVT, 4> ValueVTs;
01217     SmallVector<uint64_t, 4> Offsets;
01218     ComputeValueVTs(TLI, I.getOperand(0)->getType(), ValueVTs, &Offsets);
01219     unsigned NumValues = ValueVTs.size();
01220 
01221     SmallVector<SDValue, 4> Chains(NumValues);
01222     for (unsigned i = 0; i != NumValues; ++i) {
01223       SDValue Add = DAG.getNode(ISD::ADD, getCurSDLoc(),
01224                                 RetPtr.getValueType(), RetPtr,
01225                                 DAG.getIntPtrConstant(Offsets[i]));
01226       Chains[i] =
01227         DAG.getStore(Chain, getCurSDLoc(),
01228                      SDValue(RetOp.getNode(), RetOp.getResNo() + i),
01229                      // FIXME: better loc info would be nice.
01230                      Add, MachinePointerInfo(), false, false, 0);
01231     }
01232 
01233     Chain = DAG.getNode(ISD::TokenFactor, getCurSDLoc(),
01234                         MVT::Other, Chains);
01235   } else if (I.getNumOperands() != 0) {
01236     SmallVector<EVT, 4> ValueVTs;
01237     ComputeValueVTs(TLI, I.getOperand(0)->getType(), ValueVTs);
01238     unsigned NumValues = ValueVTs.size();
01239     if (NumValues) {
01240       SDValue RetOp = getValue(I.getOperand(0));
01241       for (unsigned j = 0, f = NumValues; j != f; ++j) {
01242         EVT VT = ValueVTs[j];
01243 
01244         ISD::NodeType ExtendKind = ISD::ANY_EXTEND;
01245 
01246         const Function *F = I.getParent()->getParent();
01247         if (F->getAttributes().hasAttribute(AttributeSet::ReturnIndex,
01248                                             Attribute::SExt))
01249           ExtendKind = ISD::SIGN_EXTEND;
01250         else if (F->getAttributes().hasAttribute(AttributeSet::ReturnIndex,
01251                                                  Attribute::ZExt))
01252           ExtendKind = ISD::ZERO_EXTEND;
01253 
01254         if (ExtendKind != ISD::ANY_EXTEND && VT.isInteger())
01255           VT = TLI.getTypeForExtArgOrReturn(*DAG.getContext(), VT, ExtendKind);
01256 
01257         unsigned NumParts = TLI.getNumRegisters(*DAG.getContext(), VT);
01258         MVT PartVT = TLI.getRegisterType(*DAG.getContext(), VT);
01259         SmallVector<SDValue, 4> Parts(NumParts);
01260         getCopyToParts(DAG, getCurSDLoc(),
01261                        SDValue(RetOp.getNode(), RetOp.getResNo() + j),
01262                        &Parts[0], NumParts, PartVT, &I, ExtendKind);
01263 
01264         // 'inreg' on function refers to return value
01265         ISD::ArgFlagsTy Flags = ISD::ArgFlagsTy();
01266         if (F->getAttributes().hasAttribute(AttributeSet::ReturnIndex,
01267                                             Attribute::InReg))
01268           Flags.setInReg();
01269 
01270         // Propagate extension type if any
01271         if (ExtendKind == ISD::SIGN_EXTEND)
01272           Flags.setSExt();
01273         else if (ExtendKind == ISD::ZERO_EXTEND)
01274           Flags.setZExt();
01275 
01276         for (unsigned i = 0; i < NumParts; ++i) {
01277           Outs.push_back(ISD::OutputArg(Flags, Parts[i].getValueType(),
01278                                         VT, /*isfixed=*/true, 0, 0));
01279           OutVals.push_back(Parts[i]);
01280         }
01281       }
01282     }
01283   }
01284 
01285   bool isVarArg = DAG.getMachineFunction().getFunction()->isVarArg();
01286   CallingConv::ID CallConv =
01287     DAG.getMachineFunction().getFunction()->getCallingConv();
01288   Chain = DAG.getTargetLoweringInfo().LowerReturn(
01289       Chain, CallConv, isVarArg, Outs, OutVals, getCurSDLoc(), DAG);
01290 
01291   // Verify that the target's LowerReturn behaved as expected.
01292   assert(Chain.getNode() && Chain.getValueType() == MVT::Other &&
01293          "LowerReturn didn't return a valid chain!");
01294 
01295   // Update the DAG with the new chain value resulting from return lowering.
01296   DAG.setRoot(Chain);
01297 }
01298 
01299 /// CopyToExportRegsIfNeeded - If the given value has virtual registers
01300 /// created for it, emit nodes to copy the value into the virtual
01301 /// registers.
01302 void SelectionDAGBuilder::CopyToExportRegsIfNeeded(const Value *V) {
01303   // Skip empty types
01304   if (V->getType()->isEmptyTy())
01305     return;
01306 
01307   DenseMap<const Value *, unsigned>::iterator VMI = FuncInfo.ValueMap.find(V);
01308   if (VMI != FuncInfo.ValueMap.end()) {
01309     assert(!V->use_empty() && "Unused value assigned virtual registers!");
01310     CopyValueToVirtualRegister(V, VMI->second);
01311   }
01312 }
01313 
01314 /// ExportFromCurrentBlock - If this condition isn't known to be exported from
01315 /// the current basic block, add it to ValueMap now so that we'll get a
01316 /// CopyTo/FromReg.
01317 void SelectionDAGBuilder::ExportFromCurrentBlock(const Value *V) {
01318   // No need to export constants.
01319   if (!isa<Instruction>(V) && !isa<Argument>(V)) return;
01320 
01321   // Already exported?
01322   if (FuncInfo.isExportedInst(V)) return;
01323 
01324   unsigned Reg = FuncInfo.InitializeRegForValue(V);
01325   CopyValueToVirtualRegister(V, Reg);
01326 }
01327 
01328 bool SelectionDAGBuilder::isExportableFromCurrentBlock(const Value *V,
01329                                                      const BasicBlock *FromBB) {
01330   // The operands of the setcc have to be in this block.  We don't know
01331   // how to export them from some other block.
01332   if (const Instruction *VI = dyn_cast<Instruction>(V)) {
01333     // Can export from current BB.
01334     if (VI->getParent() == FromBB)
01335       return true;
01336 
01337     // Is already exported, noop.
01338     return FuncInfo.isExportedInst(V);
01339   }
01340 
01341   // If this is an argument, we can export it if the BB is the entry block or
01342   // if it is already exported.
01343   if (isa<Argument>(V)) {
01344     if (FromBB == &FromBB->getParent()->getEntryBlock())
01345       return true;
01346 
01347     // Otherwise, can only export this if it is already exported.
01348     return FuncInfo.isExportedInst(V);
01349   }
01350 
01351   // Otherwise, constants can always be exported.
01352   return true;
01353 }
01354 
01355 /// Return branch probability calculated by BranchProbabilityInfo for IR blocks.
01356 uint32_t SelectionDAGBuilder::getEdgeWeight(const MachineBasicBlock *Src,
01357                                             const MachineBasicBlock *Dst) const {
01358   BranchProbabilityInfo *BPI = FuncInfo.BPI;
01359   if (!BPI)
01360     return 0;
01361   const BasicBlock *SrcBB = Src->getBasicBlock();
01362   const BasicBlock *DstBB = Dst->getBasicBlock();
01363   return BPI->getEdgeWeight(SrcBB, DstBB);
01364 }
01365 
01366 void SelectionDAGBuilder::
01367 addSuccessorWithWeight(MachineBasicBlock *Src, MachineBasicBlock *Dst,
01368                        uint32_t Weight /* = 0 */) {
01369   if (!Weight)
01370     Weight = getEdgeWeight(Src, Dst);
01371   Src->addSuccessor(Dst, Weight);
01372 }
01373 
01374 
01375 static bool InBlock(const Value *V, const BasicBlock *BB) {
01376   if (const Instruction *I = dyn_cast<Instruction>(V))
01377     return I->getParent() == BB;
01378   return true;
01379 }
01380 
01381 /// EmitBranchForMergedCondition - Helper method for FindMergedConditions.
01382 /// This function emits a branch and is used at the leaves of an OR or an
01383 /// AND operator tree.
01384 ///
01385 void
01386 SelectionDAGBuilder::EmitBranchForMergedCondition(const Value *Cond,
01387                                                   MachineBasicBlock *TBB,
01388                                                   MachineBasicBlock *FBB,
01389                                                   MachineBasicBlock *CurBB,
01390                                                   MachineBasicBlock *SwitchBB,
01391                                                   uint32_t TWeight,
01392                                                   uint32_t FWeight) {
01393   const BasicBlock *BB = CurBB->getBasicBlock();
01394 
01395   // If the leaf of the tree is a comparison, merge the condition into
01396   // the caseblock.
01397   if (const CmpInst *BOp = dyn_cast<CmpInst>(Cond)) {
01398     // The operands of the cmp have to be in this block.  We don't know
01399     // how to export them from some other block.  If this is the first block
01400     // of the sequence, no exporting is needed.
01401     if (CurBB == SwitchBB ||
01402         (isExportableFromCurrentBlock(BOp->getOperand(0), BB) &&
01403          isExportableFromCurrentBlock(BOp->getOperand(1), BB))) {
01404       ISD::CondCode Condition;
01405       if (const ICmpInst *IC = dyn_cast<ICmpInst>(Cond)) {
01406         Condition = getICmpCondCode(IC->getPredicate());
01407       } else if (const FCmpInst *FC = dyn_cast<FCmpInst>(Cond)) {
01408         Condition = getFCmpCondCode(FC->getPredicate());
01409         if (TM.Options.NoNaNsFPMath)
01410           Condition = getFCmpCodeWithoutNaN(Condition);
01411       } else {
01412         (void)Condition; // silence warning.
01413         llvm_unreachable("Unknown compare instruction");
01414       }
01415 
01416       CaseBlock CB(Condition, BOp->getOperand(0), BOp->getOperand(1), nullptr,
01417                    TBB, FBB, CurBB, TWeight, FWeight);
01418       SwitchCases.push_back(CB);
01419       return;
01420     }
01421   }
01422 
01423   // Create a CaseBlock record representing this branch.
01424   CaseBlock CB(ISD::SETEQ, Cond, ConstantInt::getTrue(*DAG.getContext()),
01425                nullptr, TBB, FBB, CurBB, TWeight, FWeight);
01426   SwitchCases.push_back(CB);
01427 }
01428 
01429 /// Scale down both weights to fit into uint32_t.
01430 static void ScaleWeights(uint64_t &NewTrue, uint64_t &NewFalse) {
01431   uint64_t NewMax = (NewTrue > NewFalse) ? NewTrue : NewFalse;
01432   uint32_t Scale = (NewMax / UINT32_MAX) + 1;
01433   NewTrue = NewTrue / Scale;
01434   NewFalse = NewFalse / Scale;
01435 }
01436 
01437 /// FindMergedConditions - If Cond is an expression like
01438 void SelectionDAGBuilder::FindMergedConditions(const Value *Cond,
01439                                                MachineBasicBlock *TBB,
01440                                                MachineBasicBlock *FBB,
01441                                                MachineBasicBlock *CurBB,
01442                                                MachineBasicBlock *SwitchBB,
01443                                                unsigned Opc, uint32_t TWeight,
01444                                                uint32_t FWeight) {
01445   // If this node is not part of the or/and tree, emit it as a branch.
01446   const Instruction *BOp = dyn_cast<Instruction>(Cond);
01447   if (!BOp || !(isa<BinaryOperator>(BOp) || isa<CmpInst>(BOp)) ||
01448       (unsigned)BOp->getOpcode() != Opc || !BOp->hasOneUse() ||
01449       BOp->getParent() != CurBB->getBasicBlock() ||
01450       !InBlock(BOp->getOperand(0), CurBB->getBasicBlock()) ||
01451       !InBlock(BOp->getOperand(1), CurBB->getBasicBlock())) {
01452     EmitBranchForMergedCondition(Cond, TBB, FBB, CurBB, SwitchBB,
01453                                  TWeight, FWeight);
01454     return;
01455   }
01456 
01457   //  Create TmpBB after CurBB.
01458   MachineFunction::iterator BBI = CurBB;
01459   MachineFunction &MF = DAG.getMachineFunction();
01460   MachineBasicBlock *TmpBB = MF.CreateMachineBasicBlock(CurBB->getBasicBlock());
01461   CurBB->getParent()->insert(++BBI, TmpBB);
01462 
01463   if (Opc == Instruction::Or) {
01464     // Codegen X | Y as:
01465     // BB1:
01466     //   jmp_if_X TBB
01467     //   jmp TmpBB
01468     // TmpBB:
01469     //   jmp_if_Y TBB
01470     //   jmp FBB
01471     //
01472 
01473     // We have flexibility in setting Prob for BB1 and Prob for TmpBB.
01474     // The requirement is that
01475     //   TrueProb for BB1 + (FalseProb for BB1 * TrueProb for TmpBB)
01476     //     = TrueProb for orignal BB.
01477     // Assuming the orignal weights are A and B, one choice is to set BB1's
01478     // weights to A and A+2B, and set TmpBB's weights to A and 2B. This choice
01479     // assumes that
01480     //   TrueProb for BB1 == FalseProb for BB1 * TrueProb for TmpBB.
01481     // Another choice is to assume TrueProb for BB1 equals to TrueProb for
01482     // TmpBB, but the math is more complicated.
01483 
01484     uint64_t NewTrueWeight = TWeight;
01485     uint64_t NewFalseWeight = (uint64_t)TWeight + 2 * (uint64_t)FWeight;
01486     ScaleWeights(NewTrueWeight, NewFalseWeight);
01487     // Emit the LHS condition.
01488     FindMergedConditions(BOp->getOperand(0), TBB, TmpBB, CurBB, SwitchBB, Opc,
01489                          NewTrueWeight, NewFalseWeight);
01490 
01491     NewTrueWeight = TWeight;
01492     NewFalseWeight = 2 * (uint64_t)FWeight;
01493     ScaleWeights(NewTrueWeight, NewFalseWeight);
01494     // Emit the RHS condition into TmpBB.
01495     FindMergedConditions(BOp->getOperand(1), TBB, FBB, TmpBB, SwitchBB, Opc,
01496                          NewTrueWeight, NewFalseWeight);
01497   } else {
01498     assert(Opc == Instruction::And && "Unknown merge op!");
01499     // Codegen X & Y as:
01500     // BB1:
01501     //   jmp_if_X TmpBB
01502     //   jmp FBB
01503     // TmpBB:
01504     //   jmp_if_Y TBB
01505     //   jmp FBB
01506     //
01507     //  This requires creation of TmpBB after CurBB.
01508 
01509     // We have flexibility in setting Prob for BB1 and Prob for TmpBB.
01510     // The requirement is that
01511     //   FalseProb for BB1 + (TrueProb for BB1 * FalseProb for TmpBB)
01512     //     = FalseProb for orignal BB.
01513     // Assuming the orignal weights are A and B, one choice is to set BB1's
01514     // weights to 2A+B and B, and set TmpBB's weights to 2A and B. This choice
01515     // assumes that
01516     //   FalseProb for BB1 == TrueProb for BB1 * FalseProb for TmpBB.
01517 
01518     uint64_t NewTrueWeight = 2 * (uint64_t)TWeight + (uint64_t)FWeight;
01519     uint64_t NewFalseWeight = FWeight;
01520     ScaleWeights(NewTrueWeight, NewFalseWeight);
01521     // Emit the LHS condition.
01522     FindMergedConditions(BOp->getOperand(0), TmpBB, FBB, CurBB, SwitchBB, Opc,
01523                          NewTrueWeight, NewFalseWeight);
01524 
01525     NewTrueWeight = 2 * (uint64_t)TWeight;
01526     NewFalseWeight = FWeight;
01527     ScaleWeights(NewTrueWeight, NewFalseWeight);
01528     // Emit the RHS condition into TmpBB.
01529     FindMergedConditions(BOp->getOperand(1), TBB, FBB, TmpBB, SwitchBB, Opc,
01530                          NewTrueWeight, NewFalseWeight);
01531   }
01532 }
01533 
01534 /// If the set of cases should be emitted as a series of branches, return true.
01535 /// If we should emit this as a bunch of and/or'd together conditions, return
01536 /// false.
01537 bool
01538 SelectionDAGBuilder::ShouldEmitAsBranches(const std::vector<CaseBlock> &Cases) {
01539   if (Cases.size() != 2) return true;
01540 
01541   // If this is two comparisons of the same values or'd or and'd together, they
01542   // will get folded into a single comparison, so don't emit two blocks.
01543   if ((Cases[0].CmpLHS == Cases[1].CmpLHS &&
01544        Cases[0].CmpRHS == Cases[1].CmpRHS) ||
01545       (Cases[0].CmpRHS == Cases[1].CmpLHS &&
01546        Cases[0].CmpLHS == Cases[1].CmpRHS)) {
01547     return false;
01548   }
01549 
01550   // Handle: (X != null) | (Y != null) --> (X|Y) != 0
01551   // Handle: (X == null) & (Y == null) --> (X|Y) == 0
01552   if (Cases[0].CmpRHS == Cases[1].CmpRHS &&
01553       Cases[0].CC == Cases[1].CC &&
01554       isa<Constant>(Cases[0].CmpRHS) &&
01555       cast<Constant>(Cases[0].CmpRHS)->isNullValue()) {
01556     if (Cases[0].CC == ISD::SETEQ && Cases[0].TrueBB == Cases[1].ThisBB)
01557       return false;
01558     if (Cases[0].CC == ISD::SETNE && Cases[0].FalseBB == Cases[1].ThisBB)
01559       return false;
01560   }
01561 
01562   return true;
01563 }
01564 
01565 void SelectionDAGBuilder::visitBr(const BranchInst &I) {
01566   MachineBasicBlock *BrMBB = FuncInfo.MBB;
01567 
01568   // Update machine-CFG edges.
01569   MachineBasicBlock *Succ0MBB = FuncInfo.MBBMap[I.getSuccessor(0)];
01570 
01571   // Figure out which block is immediately after the current one.
01572   MachineBasicBlock *NextBlock = nullptr;
01573   MachineFunction::iterator BBI = BrMBB;
01574   if (++BBI != FuncInfo.MF->end())
01575     NextBlock = BBI;
01576 
01577   if (I.isUnconditional()) {
01578     // Update machine-CFG edges.
01579     BrMBB->addSuccessor(Succ0MBB);
01580 
01581     // If this is not a fall-through branch or optimizations are switched off,
01582     // emit the branch.
01583     if (Succ0MBB != NextBlock || TM.getOptLevel() == CodeGenOpt::None)
01584       DAG.setRoot(DAG.getNode(ISD::BR, getCurSDLoc(),
01585                               MVT::Other, getControlRoot(),
01586                               DAG.getBasicBlock(Succ0MBB)));
01587 
01588     return;
01589   }
01590 
01591   // If this condition is one of the special cases we handle, do special stuff
01592   // now.
01593   const Value *CondVal = I.getCondition();
01594   MachineBasicBlock *Succ1MBB = FuncInfo.MBBMap[I.getSuccessor(1)];
01595 
01596   // If this is a series of conditions that are or'd or and'd together, emit
01597   // this as a sequence of branches instead of setcc's with and/or operations.
01598   // As long as jumps are not expensive, this should improve performance.
01599   // For example, instead of something like:
01600   //     cmp A, B
01601   //     C = seteq
01602   //     cmp D, E
01603   //     F = setle
01604   //     or C, F
01605   //     jnz foo
01606   // Emit:
01607   //     cmp A, B
01608   //     je foo
01609   //     cmp D, E
01610   //     jle foo
01611   //
01612   if (const BinaryOperator *BOp = dyn_cast<BinaryOperator>(CondVal)) {
01613     if (!DAG.getTargetLoweringInfo().isJumpExpensive() &&
01614         BOp->hasOneUse() && (BOp->getOpcode() == Instruction::And ||
01615                              BOp->getOpcode() == Instruction::Or)) {
01616       FindMergedConditions(BOp, Succ0MBB, Succ1MBB, BrMBB, BrMBB,
01617                            BOp->getOpcode(), getEdgeWeight(BrMBB, Succ0MBB),
01618                            getEdgeWeight(BrMBB, Succ1MBB));
01619       // If the compares in later blocks need to use values not currently
01620       // exported from this block, export them now.  This block should always
01621       // be the first entry.
01622       assert(SwitchCases[0].ThisBB == BrMBB && "Unexpected lowering!");
01623 
01624       // Allow some cases to be rejected.
01625       if (ShouldEmitAsBranches(SwitchCases)) {
01626         for (unsigned i = 1, e = SwitchCases.size(); i != e; ++i) {
01627           ExportFromCurrentBlock(SwitchCases[i].CmpLHS);
01628           ExportFromCurrentBlock(SwitchCases[i].CmpRHS);
01629         }
01630 
01631         // Emit the branch for this block.
01632         visitSwitchCase(SwitchCases[0], BrMBB);
01633         SwitchCases.erase(SwitchCases.begin());
01634         return;
01635       }
01636 
01637       // Okay, we decided not to do this, remove any inserted MBB's and clear
01638       // SwitchCases.
01639       for (unsigned i = 1, e = SwitchCases.size(); i != e; ++i)
01640         FuncInfo.MF->erase(SwitchCases[i].ThisBB);
01641 
01642       SwitchCases.clear();
01643     }
01644   }
01645 
01646   // Create a CaseBlock record representing this branch.
01647   CaseBlock CB(ISD::SETEQ, CondVal, ConstantInt::getTrue(*DAG.getContext()),
01648                nullptr, Succ0MBB, Succ1MBB, BrMBB);
01649 
01650   // Use visitSwitchCase to actually insert the fast branch sequence for this
01651   // cond branch.
01652   visitSwitchCase(CB, BrMBB);
01653 }
01654 
01655 /// visitSwitchCase - Emits the necessary code to represent a single node in
01656 /// the binary search tree resulting from lowering a switch instruction.
01657 void SelectionDAGBuilder::visitSwitchCase(CaseBlock &CB,
01658                                           MachineBasicBlock *SwitchBB) {
01659   SDValue Cond;
01660   SDValue CondLHS = getValue(CB.CmpLHS);
01661   SDLoc dl = getCurSDLoc();
01662 
01663   // Build the setcc now.
01664   if (!CB.CmpMHS) {
01665     // Fold "(X == true)" to X and "(X == false)" to !X to
01666     // handle common cases produced by branch lowering.
01667     if (CB.CmpRHS == ConstantInt::getTrue(*DAG.getContext()) &&
01668         CB.CC == ISD::SETEQ)
01669       Cond = CondLHS;
01670     else if (CB.CmpRHS == ConstantInt::getFalse(*DAG.getContext()) &&
01671              CB.CC == ISD::SETEQ) {
01672       SDValue True = DAG.getConstant(1, CondLHS.getValueType());
01673       Cond = DAG.getNode(ISD::XOR, dl, CondLHS.getValueType(), CondLHS, True);
01674     } else
01675       Cond = DAG.getSetCC(dl, MVT::i1, CondLHS, getValue(CB.CmpRHS), CB.CC);
01676   } else {
01677     assert(CB.CC == ISD::SETLE && "Can handle only LE ranges now");
01678 
01679     const APInt& Low = cast<ConstantInt>(CB.CmpLHS)->getValue();
01680     const APInt& High  = cast<ConstantInt>(CB.CmpRHS)->getValue();
01681 
01682     SDValue CmpOp = getValue(CB.CmpMHS);
01683     EVT VT = CmpOp.getValueType();
01684 
01685     if (cast<ConstantInt>(CB.CmpLHS)->isMinValue(true)) {
01686       Cond = DAG.getSetCC(dl, MVT::i1, CmpOp, DAG.getConstant(High, VT),
01687                           ISD::SETLE);
01688     } else {
01689       SDValue SUB = DAG.getNode(ISD::SUB, dl,
01690                                 VT, CmpOp, DAG.getConstant(Low, VT));
01691       Cond = DAG.getSetCC(dl, MVT::i1, SUB,
01692                           DAG.getConstant(High-Low, VT), ISD::SETULE);
01693     }
01694   }
01695 
01696   // Update successor info
01697   addSuccessorWithWeight(SwitchBB, CB.TrueBB, CB.TrueWeight);
01698   // TrueBB and FalseBB are always different unless the incoming IR is
01699   // degenerate. This only happens when running llc on weird IR.
01700   if (CB.TrueBB != CB.FalseBB)
01701     addSuccessorWithWeight(SwitchBB, CB.FalseBB, CB.FalseWeight);
01702 
01703   // Set NextBlock to be the MBB immediately after the current one, if any.
01704   // This is used to avoid emitting unnecessary branches to the next block.
01705   MachineBasicBlock *NextBlock = nullptr;
01706   MachineFunction::iterator BBI = SwitchBB;
01707   if (++BBI != FuncInfo.MF->end())
01708     NextBlock = BBI;
01709 
01710   // If the lhs block is the next block, invert the condition so that we can
01711   // fall through to the lhs instead of the rhs block.
01712   if (CB.TrueBB == NextBlock) {
01713     std::swap(CB.TrueBB, CB.FalseBB);
01714     SDValue True = DAG.getConstant(1, Cond.getValueType());
01715     Cond = DAG.getNode(ISD::XOR, dl, Cond.getValueType(), Cond, True);
01716   }
01717 
01718   SDValue BrCond = DAG.getNode(ISD::BRCOND, dl,
01719                                MVT::Other, getControlRoot(), Cond,
01720                                DAG.getBasicBlock(CB.TrueBB));
01721 
01722   // Insert the false branch. Do this even if it's a fall through branch,
01723   // this makes it easier to do DAG optimizations which require inverting
01724   // the branch condition.
01725   BrCond = DAG.getNode(ISD::BR, dl, MVT::Other, BrCond,
01726                        DAG.getBasicBlock(CB.FalseBB));
01727 
01728   DAG.setRoot(BrCond);
01729 }
01730 
01731 /// visitJumpTable - Emit JumpTable node in the current MBB
01732 void SelectionDAGBuilder::visitJumpTable(JumpTable &JT) {
01733   // Emit the code for the jump table
01734   assert(JT.Reg != -1U && "Should lower JT Header first!");
01735   EVT PTy = DAG.getTargetLoweringInfo().getPointerTy();
01736   SDValue Index = DAG.getCopyFromReg(getControlRoot(), getCurSDLoc(),
01737                                      JT.Reg, PTy);
01738   SDValue Table = DAG.getJumpTable(JT.JTI, PTy);
01739   SDValue BrJumpTable = DAG.getNode(ISD::BR_JT, getCurSDLoc(),
01740                                     MVT::Other, Index.getValue(1),
01741                                     Table, Index);
01742   DAG.setRoot(BrJumpTable);
01743 }
01744 
01745 /// visitJumpTableHeader - This function emits necessary code to produce index
01746 /// in the JumpTable from switch case.
01747 void SelectionDAGBuilder::visitJumpTableHeader(JumpTable &JT,
01748                                                JumpTableHeader &JTH,
01749                                                MachineBasicBlock *SwitchBB) {
01750   // Subtract the lowest switch case value from the value being switched on and
01751   // conditional branch to default mbb if the result is greater than the
01752   // difference between smallest and largest cases.
01753   SDValue SwitchOp = getValue(JTH.SValue);
01754   EVT VT = SwitchOp.getValueType();
01755   SDValue Sub = DAG.getNode(ISD::SUB, getCurSDLoc(), VT, SwitchOp,
01756                             DAG.getConstant(JTH.First, VT));
01757 
01758   // The SDNode we just created, which holds the value being switched on minus
01759   // the smallest case value, needs to be copied to a virtual register so it
01760   // can be used as an index into the jump table in a subsequent basic block.
01761   // This value may be smaller or larger than the target's pointer type, and
01762   // therefore require extension or truncating.
01763   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
01764   SwitchOp = DAG.getZExtOrTrunc(Sub, getCurSDLoc(), TLI.getPointerTy());
01765 
01766   unsigned JumpTableReg = FuncInfo.CreateReg(TLI.getPointerTy());
01767   SDValue CopyTo = DAG.getCopyToReg(getControlRoot(), getCurSDLoc(),
01768                                     JumpTableReg, SwitchOp);
01769   JT.Reg = JumpTableReg;
01770 
01771   // Emit the range check for the jump table, and branch to the default block
01772   // for the switch statement if the value being switched on exceeds the largest
01773   // case in the switch.
01774   SDValue CMP =
01775       DAG.getSetCC(getCurSDLoc(), TLI.getSetCCResultType(*DAG.getContext(),
01776                                                          Sub.getValueType()),
01777                    Sub, DAG.getConstant(JTH.Last - JTH.First, VT), ISD::SETUGT);
01778 
01779   // Set NextBlock to be the MBB immediately after the current one, if any.
01780   // This is used to avoid emitting unnecessary branches to the next block.
01781   MachineBasicBlock *NextBlock = nullptr;
01782   MachineFunction::iterator BBI = SwitchBB;
01783 
01784   if (++BBI != FuncInfo.MF->end())
01785     NextBlock = BBI;
01786 
01787   SDValue BrCond = DAG.getNode(ISD::BRCOND, getCurSDLoc(),
01788                                MVT::Other, CopyTo, CMP,
01789                                DAG.getBasicBlock(JT.Default));
01790 
01791   if (JT.MBB != NextBlock)
01792     BrCond = DAG.getNode(ISD::BR, getCurSDLoc(), MVT::Other, BrCond,
01793                          DAG.getBasicBlock(JT.MBB));
01794 
01795   DAG.setRoot(BrCond);
01796 }
01797 
01798 /// Codegen a new tail for a stack protector check ParentMBB which has had its
01799 /// tail spliced into a stack protector check success bb.
01800 ///
01801 /// For a high level explanation of how this fits into the stack protector
01802 /// generation see the comment on the declaration of class
01803 /// StackProtectorDescriptor.
01804 void SelectionDAGBuilder::visitSPDescriptorParent(StackProtectorDescriptor &SPD,
01805                                                   MachineBasicBlock *ParentBB) {
01806 
01807   // First create the loads to the guard/stack slot for the comparison.
01808   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
01809   EVT PtrTy = TLI.getPointerTy();
01810 
01811   MachineFrameInfo *MFI = ParentBB->getParent()->getFrameInfo();
01812   int FI = MFI->getStackProtectorIndex();
01813 
01814   const Value *IRGuard = SPD.getGuard();
01815   SDValue GuardPtr = getValue(IRGuard);
01816   SDValue StackSlotPtr = DAG.getFrameIndex(FI, PtrTy);
01817 
01818   unsigned Align =
01819     TLI.getDataLayout()->getPrefTypeAlignment(IRGuard->getType());
01820 
01821   SDValue Guard;
01822 
01823   // If GuardReg is set and useLoadStackGuardNode returns true, retrieve the
01824   // guard value from the virtual register holding the value. Otherwise, emit a
01825   // volatile load to retrieve the stack guard value.
01826   unsigned GuardReg = SPD.getGuardReg();
01827 
01828   if (GuardReg && TLI.useLoadStackGuardNode())
01829     Guard = DAG.getCopyFromReg(DAG.getEntryNode(), getCurSDLoc(), GuardReg,
01830                                PtrTy);
01831   else
01832     Guard = DAG.getLoad(PtrTy, getCurSDLoc(), DAG.getEntryNode(),
01833                         GuardPtr, MachinePointerInfo(IRGuard, 0),
01834                         true, false, false, Align);
01835 
01836   SDValue StackSlot = DAG.getLoad(PtrTy, getCurSDLoc(), DAG.getEntryNode(),
01837                                   StackSlotPtr,
01838                                   MachinePointerInfo::getFixedStack(FI),
01839                                   true, false, false, Align);
01840 
01841   // Perform the comparison via a subtract/getsetcc.
01842   EVT VT = Guard.getValueType();
01843   SDValue Sub = DAG.getNode(ISD::SUB, getCurSDLoc(), VT, Guard, StackSlot);
01844 
01845   SDValue Cmp =
01846       DAG.getSetCC(getCurSDLoc(), TLI.getSetCCResultType(*DAG.getContext(),
01847                                                          Sub.getValueType()),
01848                    Sub, DAG.getConstant(0, VT), ISD::SETNE);
01849 
01850   // If the sub is not 0, then we know the guard/stackslot do not equal, so
01851   // branch to failure MBB.
01852   SDValue BrCond = DAG.getNode(ISD::BRCOND, getCurSDLoc(),
01853                                MVT::Other, StackSlot.getOperand(0),
01854                                Cmp, DAG.getBasicBlock(SPD.getFailureMBB()));
01855   // Otherwise branch to success MBB.
01856   SDValue Br = DAG.getNode(ISD::BR, getCurSDLoc(),
01857                            MVT::Other, BrCond,
01858                            DAG.getBasicBlock(SPD.getSuccessMBB()));
01859 
01860   DAG.setRoot(Br);
01861 }
01862 
01863 /// Codegen the failure basic block for a stack protector check.
01864 ///
01865 /// A failure stack protector machine basic block consists simply of a call to
01866 /// __stack_chk_fail().
01867 ///
01868 /// For a high level explanation of how this fits into the stack protector
01869 /// generation see the comment on the declaration of class
01870 /// StackProtectorDescriptor.
01871 void
01872 SelectionDAGBuilder::visitSPDescriptorFailure(StackProtectorDescriptor &SPD) {
01873   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
01874   SDValue Chain =
01875       TLI.makeLibCall(DAG, RTLIB::STACKPROTECTOR_CHECK_FAIL, MVT::isVoid,
01876                       nullptr, 0, false, getCurSDLoc(), false, false).second;
01877   DAG.setRoot(Chain);
01878 }
01879 
01880 /// visitBitTestHeader - This function emits necessary code to produce value
01881 /// suitable for "bit tests"
01882 void SelectionDAGBuilder::visitBitTestHeader(BitTestBlock &B,
01883                                              MachineBasicBlock *SwitchBB) {
01884   // Subtract the minimum value
01885   SDValue SwitchOp = getValue(B.SValue);
01886   EVT VT = SwitchOp.getValueType();
01887   SDValue Sub = DAG.getNode(ISD::SUB, getCurSDLoc(), VT, SwitchOp,
01888                             DAG.getConstant(B.First, VT));
01889 
01890   // Check range
01891   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
01892   SDValue RangeCmp =
01893       DAG.getSetCC(getCurSDLoc(), TLI.getSetCCResultType(*DAG.getContext(),
01894                                                          Sub.getValueType()),
01895                    Sub, DAG.getConstant(B.Range, VT), ISD::SETUGT);
01896 
01897   // Determine the type of the test operands.
01898   bool UsePtrType = false;
01899   if (!TLI.isTypeLegal(VT))
01900     UsePtrType = true;
01901   else {
01902     for (unsigned i = 0, e = B.Cases.size(); i != e; ++i)
01903       if (!isUIntN(VT.getSizeInBits(), B.Cases[i].Mask)) {
01904         // Switch table case range are encoded into series of masks.
01905         // Just use pointer type, it's guaranteed to fit.
01906         UsePtrType = true;
01907         break;
01908       }
01909   }
01910   if (UsePtrType) {
01911     VT = TLI.getPointerTy();
01912     Sub = DAG.getZExtOrTrunc(Sub, getCurSDLoc(), VT);
01913   }
01914 
01915   B.RegVT = VT.getSimpleVT();
01916   B.Reg = FuncInfo.CreateReg(B.RegVT);
01917   SDValue CopyTo = DAG.getCopyToReg(getControlRoot(), getCurSDLoc(),
01918                                     B.Reg, Sub);
01919 
01920   // Set NextBlock to be the MBB immediately after the current one, if any.
01921   // This is used to avoid emitting unnecessary branches to the next block.
01922   MachineBasicBlock *NextBlock = nullptr;
01923   MachineFunction::iterator BBI = SwitchBB;
01924   if (++BBI != FuncInfo.MF->end())
01925     NextBlock = BBI;
01926 
01927   MachineBasicBlock* MBB = B.Cases[0].ThisBB;
01928 
01929   addSuccessorWithWeight(SwitchBB, B.Default);
01930   addSuccessorWithWeight(SwitchBB, MBB);
01931 
01932   SDValue BrRange = DAG.getNode(ISD::BRCOND, getCurSDLoc(),
01933                                 MVT::Other, CopyTo, RangeCmp,
01934                                 DAG.getBasicBlock(B.Default));
01935 
01936   if (MBB != NextBlock)
01937     BrRange = DAG.getNode(ISD::BR, getCurSDLoc(), MVT::Other, CopyTo,
01938                           DAG.getBasicBlock(MBB));
01939 
01940   DAG.setRoot(BrRange);
01941 }
01942 
01943 /// visitBitTestCase - this function produces one "bit test"
01944 void SelectionDAGBuilder::visitBitTestCase(BitTestBlock &BB,
01945                                            MachineBasicBlock* NextMBB,
01946                                            uint32_t BranchWeightToNext,
01947                                            unsigned Reg,
01948                                            BitTestCase &B,
01949                                            MachineBasicBlock *SwitchBB) {
01950   MVT VT = BB.RegVT;
01951   SDValue ShiftOp = DAG.getCopyFromReg(getControlRoot(), getCurSDLoc(),
01952                                        Reg, VT);
01953   SDValue Cmp;
01954   unsigned PopCount = CountPopulation_64(B.Mask);
01955   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
01956   if (PopCount == 1) {
01957     // Testing for a single bit; just compare the shift count with what it
01958     // would need to be to shift a 1 bit in that position.
01959     Cmp = DAG.getSetCC(
01960         getCurSDLoc(), TLI.getSetCCResultType(*DAG.getContext(), VT), ShiftOp,
01961         DAG.getConstant(countTrailingZeros(B.Mask), VT), ISD::SETEQ);
01962   } else if (PopCount == BB.Range) {
01963     // There is only one zero bit in the range, test for it directly.
01964     Cmp = DAG.getSetCC(
01965         getCurSDLoc(), TLI.getSetCCResultType(*DAG.getContext(), VT), ShiftOp,
01966         DAG.getConstant(CountTrailingOnes_64(B.Mask), VT), ISD::SETNE);
01967   } else {
01968     // Make desired shift
01969     SDValue SwitchVal = DAG.getNode(ISD::SHL, getCurSDLoc(), VT,
01970                                     DAG.getConstant(1, VT), ShiftOp);
01971 
01972     // Emit bit tests and jumps
01973     SDValue AndOp = DAG.getNode(ISD::AND, getCurSDLoc(),
01974                                 VT, SwitchVal, DAG.getConstant(B.Mask, VT));
01975     Cmp = DAG.getSetCC(getCurSDLoc(),
01976                        TLI.getSetCCResultType(*DAG.getContext(), VT), AndOp,
01977                        DAG.getConstant(0, VT), ISD::SETNE);
01978   }
01979 
01980   // The branch weight from SwitchBB to B.TargetBB is B.ExtraWeight.
01981   addSuccessorWithWeight(SwitchBB, B.TargetBB, B.ExtraWeight);
01982   // The branch weight from SwitchBB to NextMBB is BranchWeightToNext.
01983   addSuccessorWithWeight(SwitchBB, NextMBB, BranchWeightToNext);
01984 
01985   SDValue BrAnd = DAG.getNode(ISD::BRCOND, getCurSDLoc(),
01986                               MVT::Other, getControlRoot(),
01987                               Cmp, DAG.getBasicBlock(B.TargetBB));
01988 
01989   // Set NextBlock to be the MBB immediately after the current one, if any.
01990   // This is used to avoid emitting unnecessary branches to the next block.
01991   MachineBasicBlock *NextBlock = nullptr;
01992   MachineFunction::iterator BBI = SwitchBB;
01993   if (++BBI != FuncInfo.MF->end())
01994     NextBlock = BBI;
01995 
01996   if (NextMBB != NextBlock)
01997     BrAnd = DAG.getNode(ISD::BR, getCurSDLoc(), MVT::Other, BrAnd,
01998                         DAG.getBasicBlock(NextMBB));
01999 
02000   DAG.setRoot(BrAnd);
02001 }
02002 
02003 void SelectionDAGBuilder::visitInvoke(const InvokeInst &I) {
02004   MachineBasicBlock *InvokeMBB = FuncInfo.MBB;
02005 
02006   // Retrieve successors.
02007   MachineBasicBlock *Return = FuncInfo.MBBMap[I.getSuccessor(0)];
02008   MachineBasicBlock *LandingPad = FuncInfo.MBBMap[I.getSuccessor(1)];
02009 
02010   const Value *Callee(I.getCalledValue());
02011   const Function *Fn = dyn_cast<Function>(Callee);
02012   if (isa<InlineAsm>(Callee))
02013     visitInlineAsm(&I);
02014   else if (Fn && Fn->isIntrinsic()) {
02015     switch (Fn->getIntrinsicID()) {
02016     default:
02017       llvm_unreachable("Cannot invoke this intrinsic");
02018     case Intrinsic::donothing:
02019       // Ignore invokes to @llvm.donothing: jump directly to the next BB.
02020       break;
02021     case Intrinsic::experimental_patchpoint_void:
02022     case Intrinsic::experimental_patchpoint_i64:
02023       visitPatchpoint(&I, LandingPad);
02024       break;
02025     }
02026   } else
02027     LowerCallTo(&I, getValue(Callee), false, LandingPad);
02028 
02029   // If the value of the invoke is used outside of its defining block, make it
02030   // available as a virtual register.
02031   CopyToExportRegsIfNeeded(&I);
02032 
02033   // Update successor info
02034   addSuccessorWithWeight(InvokeMBB, Return);
02035   addSuccessorWithWeight(InvokeMBB, LandingPad);
02036 
02037   // Drop into normal successor.
02038   DAG.setRoot(DAG.getNode(ISD::BR, getCurSDLoc(),
02039                           MVT::Other, getControlRoot(),
02040                           DAG.getBasicBlock(Return)));
02041 }
02042 
02043 void SelectionDAGBuilder::visitResume(const ResumeInst &RI) {
02044   llvm_unreachable("SelectionDAGBuilder shouldn't visit resume instructions!");
02045 }
02046 
02047 void SelectionDAGBuilder::visitLandingPad(const LandingPadInst &LP) {
02048   assert(FuncInfo.MBB->isLandingPad() &&
02049          "Call to landingpad not in landing pad!");
02050 
02051   MachineBasicBlock *MBB = FuncInfo.MBB;
02052   MachineModuleInfo &MMI = DAG.getMachineFunction().getMMI();
02053   AddLandingPadInfo(LP, MMI, MBB);
02054 
02055   // If there aren't registers to copy the values into (e.g., during SjLj
02056   // exceptions), then don't bother to create these DAG nodes.
02057   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
02058   if (TLI.getExceptionPointerRegister() == 0 &&
02059       TLI.getExceptionSelectorRegister() == 0)
02060     return;
02061 
02062   SmallVector<EVT, 2> ValueVTs;
02063   ComputeValueVTs(TLI, LP.getType(), ValueVTs);
02064   assert(ValueVTs.size() == 2 && "Only two-valued landingpads are supported");
02065 
02066   // Get the two live-in registers as SDValues. The physregs have already been
02067   // copied into virtual registers.
02068   SDValue Ops[2];
02069   Ops[0] = DAG.getZExtOrTrunc(
02070       DAG.getCopyFromReg(DAG.getEntryNode(), getCurSDLoc(),
02071                          FuncInfo.ExceptionPointerVirtReg, TLI.getPointerTy()),
02072       getCurSDLoc(), ValueVTs[0]);
02073   Ops[1] = DAG.getZExtOrTrunc(
02074       DAG.getCopyFromReg(DAG.getEntryNode(), getCurSDLoc(),
02075                          FuncInfo.ExceptionSelectorVirtReg, TLI.getPointerTy()),
02076       getCurSDLoc(), ValueVTs[1]);
02077 
02078   // Merge into one.
02079   SDValue Res = DAG.getNode(ISD::MERGE_VALUES, getCurSDLoc(),
02080                             DAG.getVTList(ValueVTs), Ops);
02081   setValue(&LP, Res);
02082 }
02083 
02084 /// handleSmallSwitchCaseRange - Emit a series of specific tests (suitable for
02085 /// small case ranges).
02086 bool SelectionDAGBuilder::handleSmallSwitchRange(CaseRec& CR,
02087                                                  CaseRecVector& WorkList,
02088                                                  const Value* SV,
02089                                                  MachineBasicBlock *Default,
02090                                                  MachineBasicBlock *SwitchBB) {
02091   // Size is the number of Cases represented by this range.
02092   size_t Size = CR.Range.second - CR.Range.first;
02093   if (Size > 3)
02094     return false;
02095 
02096   // Get the MachineFunction which holds the current MBB.  This is used when
02097   // inserting any additional MBBs necessary to represent the switch.
02098   MachineFunction *CurMF = FuncInfo.MF;
02099 
02100   // Figure out which block is immediately after the current one.
02101   MachineBasicBlock *NextBlock = nullptr;
02102   MachineFunction::iterator BBI = CR.CaseBB;
02103 
02104   if (++BBI != FuncInfo.MF->end())
02105     NextBlock = BBI;
02106 
02107   BranchProbabilityInfo *BPI = FuncInfo.BPI;
02108   // If any two of the cases has the same destination, and if one value
02109   // is the same as the other, but has one bit unset that the other has set,
02110   // use bit manipulation to do two compares at once.  For example:
02111   // "if (X == 6 || X == 4)" -> "if ((X|2) == 6)"
02112   // TODO: This could be extended to merge any 2 cases in switches with 3 cases.
02113   // TODO: Handle cases where CR.CaseBB != SwitchBB.
02114   if (Size == 2 && CR.CaseBB == SwitchBB) {
02115     Case &Small = *CR.Range.first;
02116     Case &Big = *(CR.Range.second-1);
02117 
02118     if (Small.Low == Small.High && Big.Low == Big.High && Small.BB == Big.BB) {
02119       const APInt& SmallValue = cast<ConstantInt>(Small.Low)->getValue();
02120       const APInt& BigValue = cast<ConstantInt>(Big.Low)->getValue();
02121 
02122       // Check that there is only one bit different.
02123       if (BigValue.countPopulation() == SmallValue.countPopulation() + 1 &&
02124           (SmallValue | BigValue) == BigValue) {
02125         // Isolate the common bit.
02126         APInt CommonBit = BigValue & ~SmallValue;
02127         assert((SmallValue | CommonBit) == BigValue &&
02128                CommonBit.countPopulation() == 1 && "Not a common bit?");
02129 
02130         SDValue CondLHS = getValue(SV);
02131         EVT VT = CondLHS.getValueType();
02132         SDLoc DL = getCurSDLoc();
02133 
02134         SDValue Or = DAG.getNode(ISD::OR, DL, VT, CondLHS,
02135                                  DAG.getConstant(CommonBit, VT));
02136         SDValue Cond = DAG.getSetCC(DL, MVT::i1,
02137                                     Or, DAG.getConstant(BigValue, VT),
02138                                     ISD::SETEQ);
02139 
02140         // Update successor info.
02141         // Both Small and Big will jump to Small.BB, so we sum up the weights.
02142         addSuccessorWithWeight(SwitchBB, Small.BB,
02143                                Small.ExtraWeight + Big.ExtraWeight);
02144         addSuccessorWithWeight(SwitchBB, Default,
02145           // The default destination is the first successor in IR.
02146           BPI ? BPI->getEdgeWeight(SwitchBB->getBasicBlock(), (unsigned)0) : 0);
02147 
02148         // Insert the true branch.
02149         SDValue BrCond = DAG.getNode(ISD::BRCOND, DL, MVT::Other,
02150                                      getControlRoot(), Cond,
02151                                      DAG.getBasicBlock(Small.BB));
02152 
02153         // Insert the false branch.
02154         BrCond = DAG.getNode(ISD::BR, DL, MVT::Other, BrCond,
02155                              DAG.getBasicBlock(Default));
02156 
02157         DAG.setRoot(BrCond);
02158         return true;
02159       }
02160     }
02161   }
02162 
02163   // Order cases by weight so the most likely case will be checked first.
02164   uint32_t UnhandledWeights = 0;
02165   if (BPI) {
02166     for (CaseItr I = CR.Range.first, IE = CR.Range.second; I != IE; ++I) {
02167       uint32_t IWeight = I->ExtraWeight;
02168       UnhandledWeights += IWeight;
02169       for (CaseItr J = CR.Range.first; J < I; ++J) {
02170         uint32_t JWeight = J->ExtraWeight;
02171         if (IWeight > JWeight)
02172           std::swap(*I, *J);
02173       }
02174     }
02175   }
02176   // Rearrange the case blocks so that the last one falls through if possible.
02177   Case &BackCase = *(CR.Range.second-1);
02178   if (Size > 1 &&
02179       NextBlock && Default != NextBlock && BackCase.BB != NextBlock) {
02180     // The last case block won't fall through into 'NextBlock' if we emit the
02181     // branches in this order.  See if rearranging a case value would help.
02182     // We start at the bottom as it's the case with the least weight.
02183     for (Case *I = &*(CR.Range.second-2), *E = &*CR.Range.first-1; I != E; --I)
02184       if (I->BB == NextBlock) {
02185         std::swap(*I, BackCase);
02186         break;
02187       }
02188   }
02189 
02190   // Create a CaseBlock record representing a conditional branch to
02191   // the Case's target mbb if the value being switched on SV is equal
02192   // to C.
02193   MachineBasicBlock *CurBlock = CR.CaseBB;
02194   for (CaseItr I = CR.Range.first, E = CR.Range.second; I != E; ++I) {
02195     MachineBasicBlock *FallThrough;
02196     if (I != E-1) {
02197       FallThrough = CurMF->CreateMachineBasicBlock(CurBlock->getBasicBlock());
02198       CurMF->insert(BBI, FallThrough);
02199 
02200       // Put SV in a virtual register to make it available from the new blocks.
02201       ExportFromCurrentBlock(SV);
02202     } else {
02203       // If the last case doesn't match, go to the default block.
02204       FallThrough = Default;
02205     }
02206 
02207     const Value *RHS, *LHS, *MHS;
02208     ISD::CondCode CC;
02209     if (I->High == I->Low) {
02210       // This is just small small case range :) containing exactly 1 case
02211       CC = ISD::SETEQ;
02212       LHS = SV; RHS = I->High; MHS = nullptr;
02213     } else {
02214       CC = ISD::SETLE;
02215       LHS = I->Low; MHS = SV; RHS = I->High;
02216     }
02217 
02218     // The false weight should be sum of all un-handled cases.
02219     UnhandledWeights -= I->ExtraWeight;
02220     CaseBlock CB(CC, LHS, RHS, MHS, /* truebb */ I->BB, /* falsebb */ FallThrough,
02221                  /* me */ CurBlock,
02222                  /* trueweight */ I->ExtraWeight,
02223                  /* falseweight */ UnhandledWeights);
02224 
02225     // If emitting the first comparison, just call visitSwitchCase to emit the
02226     // code into the current block.  Otherwise, push the CaseBlock onto the
02227     // vector to be later processed by SDISel, and insert the node's MBB
02228     // before the next MBB.
02229     if (CurBlock == SwitchBB)
02230       visitSwitchCase(CB, SwitchBB);
02231     else
02232       SwitchCases.push_back(CB);
02233 
02234     CurBlock = FallThrough;
02235   }
02236 
02237   return true;
02238 }
02239 
02240 static inline bool areJTsAllowed(const TargetLowering &TLI) {
02241   return TLI.isOperationLegalOrCustom(ISD::BR_JT, MVT::Other) ||
02242          TLI.isOperationLegalOrCustom(ISD::BRIND, MVT::Other);
02243 }
02244 
02245 static APInt ComputeRange(const APInt &First, const APInt &Last) {
02246   uint32_t BitWidth = std::max(Last.getBitWidth(), First.getBitWidth()) + 1;
02247   APInt LastExt = Last.sext(BitWidth), FirstExt = First.sext(BitWidth);
02248   return (LastExt - FirstExt + 1ULL);
02249 }
02250 
02251 /// handleJTSwitchCase - Emit jumptable for current switch case range
02252 bool SelectionDAGBuilder::handleJTSwitchCase(CaseRec &CR,
02253                                              CaseRecVector &WorkList,
02254                                              const Value *SV,
02255                                              MachineBasicBlock *Default,
02256                                              MachineBasicBlock *SwitchBB) {
02257   Case& FrontCase = *CR.Range.first;
02258   Case& BackCase  = *(CR.Range.second-1);
02259 
02260   const APInt &First = cast<ConstantInt>(FrontCase.Low)->getValue();
02261   const APInt &Last  = cast<ConstantInt>(BackCase.High)->getValue();
02262 
02263   APInt TSize(First.getBitWidth(), 0);
02264   for (CaseItr I = CR.Range.first, E = CR.Range.second; I != E; ++I)
02265     TSize += I->size();
02266 
02267   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
02268   if (!areJTsAllowed(TLI) || TSize.ult(TLI.getMinimumJumpTableEntries()))
02269     return false;
02270 
02271   APInt Range = ComputeRange(First, Last);
02272   // The density is TSize / Range. Require at least 40%.
02273   // It should not be possible for IntTSize to saturate for sane code, but make
02274   // sure we handle Range saturation correctly.
02275   uint64_t IntRange = Range.getLimitedValue(UINT64_MAX/10);
02276   uint64_t IntTSize = TSize.getLimitedValue(UINT64_MAX/10);
02277   if (IntTSize * 10 < IntRange * 4)
02278     return false;
02279 
02280   DEBUG(dbgs() << "Lowering jump table\n"
02281                << "First entry: " << First << ". Last entry: " << Last << '\n'
02282                << "Range: " << Range << ". Size: " << TSize << ".\n\n");
02283 
02284   // Get the MachineFunction which holds the current MBB.  This is used when
02285   // inserting any additional MBBs necessary to represent the switch.
02286   MachineFunction *CurMF = FuncInfo.MF;
02287 
02288   // Figure out which block is immediately after the current one.
02289   MachineFunction::iterator BBI = CR.CaseBB;
02290   ++BBI;
02291 
02292   const BasicBlock *LLVMBB = CR.CaseBB->getBasicBlock();
02293 
02294   // Create a new basic block to hold the code for loading the address
02295   // of the jump table, and jumping to it.  Update successor information;
02296   // we will either branch to the default case for the switch, or the jump
02297   // table.
02298   MachineBasicBlock *JumpTableBB = CurMF->CreateMachineBasicBlock(LLVMBB);
02299   CurMF->insert(BBI, JumpTableBB);
02300 
02301   addSuccessorWithWeight(CR.CaseBB, Default);
02302   addSuccessorWithWeight(CR.CaseBB, JumpTableBB);
02303 
02304   // Build a vector of destination BBs, corresponding to each target
02305   // of the jump table. If the value of the jump table slot corresponds to
02306   // a case statement, push the case's BB onto the vector, otherwise, push
02307   // the default BB.
02308   std::vector<MachineBasicBlock*> DestBBs;
02309   APInt TEI = First;
02310   for (CaseItr I = CR.Range.first, E = CR.Range.second; I != E; ++TEI) {
02311     const APInt &Low = cast<ConstantInt>(I->Low)->getValue();
02312     const APInt &High = cast<ConstantInt>(I->High)->getValue();
02313 
02314     if (Low.sle(TEI) && TEI.sle(High)) {
02315       DestBBs.push_back(I->BB);
02316       if (TEI==High)
02317         ++I;
02318     } else {
02319       DestBBs.push_back(Default);
02320     }
02321   }
02322 
02323   // Calculate weight for each unique destination in CR.
02324   DenseMap<MachineBasicBlock*, uint32_t> DestWeights;
02325   if (FuncInfo.BPI)
02326     for (CaseItr I = CR.Range.first, E = CR.Range.second; I != E; ++I) {
02327       DenseMap<MachineBasicBlock*, uint32_t>::iterator Itr =
02328           DestWeights.find(I->BB);
02329       if (Itr != DestWeights.end())
02330         Itr->second += I->ExtraWeight;
02331       else
02332         DestWeights[I->BB] = I->ExtraWeight;
02333     }
02334 
02335   // Update successor info. Add one edge to each unique successor.
02336   BitVector SuccsHandled(CR.CaseBB->getParent()->getNumBlockIDs());
02337   for (std::vector<MachineBasicBlock*>::iterator I = DestBBs.begin(),
02338          E = DestBBs.end(); I != E; ++I) {
02339     if (!SuccsHandled[(*I)->getNumber()]) {
02340       SuccsHandled[(*I)->getNumber()] = true;
02341       DenseMap<MachineBasicBlock*, uint32_t>::iterator Itr =
02342           DestWeights.find(*I);
02343       addSuccessorWithWeight(JumpTableBB, *I,
02344                              Itr != DestWeights.end() ? Itr->second : 0);
02345     }
02346   }
02347 
02348   // Create a jump table index for this jump table.
02349   unsigned JTEncoding = TLI.getJumpTableEncoding();
02350   unsigned JTI = CurMF->getOrCreateJumpTableInfo(JTEncoding)
02351                        ->createJumpTableIndex(DestBBs);
02352 
02353   // Set the jump table information so that we can codegen it as a second
02354   // MachineBasicBlock
02355   JumpTable JT(-1U, JTI, JumpTableBB, Default);
02356   JumpTableHeader JTH(First, Last, SV, CR.CaseBB, (CR.CaseBB == SwitchBB));
02357   if (CR.CaseBB == SwitchBB)
02358     visitJumpTableHeader(JT, JTH, SwitchBB);
02359 
02360   JTCases.push_back(JumpTableBlock(JTH, JT));
02361   return true;
02362 }
02363 
02364 /// handleBTSplitSwitchCase - emit comparison and split binary search tree into
02365 /// 2 subtrees.
02366 bool SelectionDAGBuilder::handleBTSplitSwitchCase(CaseRec& CR,
02367                                                   CaseRecVector& WorkList,
02368                                                   const Value* SV,
02369                                                   MachineBasicBlock* SwitchBB) {
02370   // Get the MachineFunction which holds the current MBB.  This is used when
02371   // inserting any additional MBBs necessary to represent the switch.
02372   MachineFunction *CurMF = FuncInfo.MF;
02373 
02374   // Figure out which block is immediately after the current one.
02375   MachineFunction::iterator BBI = CR.CaseBB;
02376   ++BBI;
02377 
02378   Case& FrontCase = *CR.Range.first;
02379   Case& BackCase  = *(CR.Range.second-1);
02380   const BasicBlock *LLVMBB = CR.CaseBB->getBasicBlock();
02381 
02382   // Size is the number of Cases represented by this range.
02383   unsigned Size = CR.Range.second - CR.Range.first;
02384 
02385   const APInt &First = cast<ConstantInt>(FrontCase.Low)->getValue();
02386   const APInt &Last  = cast<ConstantInt>(BackCase.High)->getValue();
02387   double FMetric = 0;
02388   CaseItr Pivot = CR.Range.first + Size/2;
02389 
02390   // Select optimal pivot, maximizing sum density of LHS and RHS. This will
02391   // (heuristically) allow us to emit JumpTable's later.
02392   APInt TSize(First.getBitWidth(), 0);
02393   for (CaseItr I = CR.Range.first, E = CR.Range.second;
02394        I!=E; ++I)
02395     TSize += I->size();
02396 
02397   APInt LSize = FrontCase.size();
02398   APInt RSize = TSize-LSize;
02399   DEBUG(dbgs() << "Selecting best pivot: \n"
02400                << "First: " << First << ", Last: " << Last <<'\n'
02401                << "LSize: " << LSize << ", RSize: " << RSize << '\n');
02402   for (CaseItr I = CR.Range.first, J=I+1, E = CR.Range.second;
02403        J!=E; ++I, ++J) {
02404     const APInt &LEnd = cast<ConstantInt>(I->High)->getValue();
02405     const APInt &RBegin = cast<ConstantInt>(J->Low)->getValue();
02406     APInt Range = ComputeRange(LEnd, RBegin);
02407     assert((Range - 2ULL).isNonNegative() &&
02408            "Invalid case distance");
02409     // Use volatile double here to avoid excess precision issues on some hosts,
02410     // e.g. that use 80-bit X87 registers.
02411     volatile double LDensity =
02412        (double)LSize.roundToDouble() /
02413                            (LEnd - First + 1ULL).roundToDouble();
02414     volatile double RDensity =
02415       (double)RSize.roundToDouble() /
02416                            (Last - RBegin + 1ULL).roundToDouble();
02417     volatile double Metric = Range.logBase2()*(LDensity+RDensity);
02418     // Should always split in some non-trivial place
02419     DEBUG(dbgs() <<"=>Step\n"
02420                  << "LEnd: " << LEnd << ", RBegin: " << RBegin << '\n'
02421                  << "LDensity: " << LDensity
02422                  << ", RDensity: " << RDensity << '\n'
02423                  << "Metric: " << Metric << '\n');
02424     if (FMetric < Metric) {
02425       Pivot = J;
02426       FMetric = Metric;
02427       DEBUG(dbgs() << "Current metric set to: " << FMetric << '\n');
02428     }
02429 
02430     LSize += J->size();
02431     RSize -= J->size();
02432   }
02433 
02434   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
02435   if (areJTsAllowed(TLI)) {
02436     // If our case is dense we *really* should handle it earlier!
02437     assert((FMetric > 0) && "Should handle dense range earlier!");
02438   } else {
02439     Pivot = CR.Range.first + Size/2;
02440   }
02441 
02442   CaseRange LHSR(CR.Range.first, Pivot);
02443   CaseRange RHSR(Pivot, CR.Range.second);
02444   const Constant *C = Pivot->Low;
02445   MachineBasicBlock *FalseBB = nullptr, *TrueBB = nullptr;
02446 
02447   // We know that we branch to the LHS if the Value being switched on is
02448   // less than the Pivot value, C.  We use this to optimize our binary
02449   // tree a bit, by recognizing that if SV is greater than or equal to the
02450   // LHS's Case Value, and that Case Value is exactly one less than the
02451   // Pivot's Value, then we can branch directly to the LHS's Target,
02452   // rather than creating a leaf node for it.
02453   if ((LHSR.second - LHSR.first) == 1 &&
02454       LHSR.first->High == CR.GE &&
02455       cast<ConstantInt>(C)->getValue() ==
02456       (cast<ConstantInt>(CR.GE)->getValue() + 1LL)) {
02457     TrueBB = LHSR.first->BB;
02458   } else {
02459     TrueBB = CurMF->CreateMachineBasicBlock(LLVMBB);
02460     CurMF->insert(BBI, TrueBB);
02461     WorkList.push_back(CaseRec(TrueBB, C, CR.GE, LHSR));
02462 
02463     // Put SV in a virtual register to make it available from the new blocks.
02464     ExportFromCurrentBlock(SV);
02465   }
02466 
02467   // Similar to the optimization above, if the Value being switched on is
02468   // known to be less than the Constant CR.LT, and the current Case Value
02469   // is CR.LT - 1, then we can branch directly to the target block for
02470   // the current Case Value, rather than emitting a RHS leaf node for it.
02471   if ((RHSR.second - RHSR.first) == 1 && CR.LT &&
02472       cast<ConstantInt>(RHSR.first->Low)->getValue() ==
02473       (cast<ConstantInt>(CR.LT)->getValue() - 1LL)) {
02474     FalseBB = RHSR.first->BB;
02475   } else {
02476     FalseBB = CurMF->CreateMachineBasicBlock(LLVMBB);
02477     CurMF->insert(BBI, FalseBB);
02478     WorkList.push_back(CaseRec(FalseBB,CR.LT,C,RHSR));
02479 
02480     // Put SV in a virtual register to make it available from the new blocks.
02481     ExportFromCurrentBlock(SV);
02482   }
02483 
02484   // Create a CaseBlock record representing a conditional branch to
02485   // the LHS node if the value being switched on SV is less than C.
02486   // Otherwise, branch to LHS.
02487   CaseBlock CB(ISD::SETLT, SV, C, nullptr, TrueBB, FalseBB, CR.CaseBB);
02488 
02489   if (CR.CaseBB == SwitchBB)
02490     visitSwitchCase(CB, SwitchBB);
02491   else
02492     SwitchCases.push_back(CB);
02493 
02494   return true;
02495 }
02496 
02497 /// handleBitTestsSwitchCase - if current case range has few destination and
02498 /// range span less, than machine word bitwidth, encode case range into series
02499 /// of masks and emit bit tests with these masks.
02500 bool SelectionDAGBuilder::handleBitTestsSwitchCase(CaseRec& CR,
02501                                                    CaseRecVector& WorkList,
02502                                                    const Value* SV,
02503                                                    MachineBasicBlock* Default,
02504                                                    MachineBasicBlock* SwitchBB) {
02505   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
02506   EVT PTy = TLI.getPointerTy();
02507   unsigned IntPtrBits = PTy.getSizeInBits();
02508 
02509   Case& FrontCase = *CR.Range.first;
02510   Case& BackCase  = *(CR.Range.second-1);
02511 
02512   // Get the MachineFunction which holds the current MBB.  This is used when
02513   // inserting any additional MBBs necessary to represent the switch.
02514   MachineFunction *CurMF = FuncInfo.MF;
02515 
02516   // If target does not have legal shift left, do not emit bit tests at all.
02517   if (!TLI.isOperationLegal(ISD::SHL, PTy))
02518     return false;
02519 
02520   size_t numCmps = 0;
02521   for (CaseItr I = CR.Range.first, E = CR.Range.second; I != E; ++I) {
02522     // Single case counts one, case range - two.
02523     numCmps += (I->Low == I->High ? 1 : 2);
02524   }
02525 
02526   // Count unique destinations
02527   SmallSet<MachineBasicBlock*, 4> Dests;
02528   for (CaseItr I = CR.Range.first, E = CR.Range.second; I != E; ++I) {
02529     Dests.insert(I->BB);
02530     if (Dests.size() > 3)
02531       // Don't bother the code below, if there are too much unique destinations
02532       return false;
02533   }
02534   DEBUG(dbgs() << "Total number of unique destinations: "
02535         << Dests.size() << '\n'
02536         << "Total number of comparisons: " << numCmps << '\n');
02537 
02538   // Compute span of values.
02539   const APInt& minValue = cast<ConstantInt>(FrontCase.Low)->getValue();
02540   const APInt& maxValue = cast<ConstantInt>(BackCase.High)->getValue();
02541   APInt cmpRange = maxValue - minValue;
02542 
02543   DEBUG(dbgs() << "Compare range: " << cmpRange << '\n'
02544                << "Low bound: " << minValue << '\n'
02545                << "High bound: " << maxValue << '\n');
02546 
02547   if (cmpRange.uge(IntPtrBits) ||
02548       (!(Dests.size() == 1 && numCmps >= 3) &&
02549        !(Dests.size() == 2 && numCmps >= 5) &&
02550        !(Dests.size() >= 3 && numCmps >= 6)))
02551     return false;
02552 
02553   DEBUG(dbgs() << "Emitting bit tests\n");
02554   APInt lowBound = APInt::getNullValue(cmpRange.getBitWidth());
02555 
02556   // Optimize the case where all the case values fit in a
02557   // word without having to subtract minValue. In this case,
02558   // we can optimize away the subtraction.
02559   if (minValue.isNonNegative() && maxValue.slt(IntPtrBits)) {
02560     cmpRange = maxValue;
02561   } else {
02562     lowBound = minValue;
02563   }
02564 
02565   CaseBitsVector CasesBits;
02566   unsigned i, count = 0;
02567 
02568   for (CaseItr I = CR.Range.first, E = CR.Range.second; I!=E; ++I) {
02569     MachineBasicBlock* Dest = I->BB;
02570     for (i = 0; i < count; ++i)
02571       if (Dest == CasesBits[i].BB)
02572         break;
02573 
02574     if (i == count) {
02575       assert((count < 3) && "Too much destinations to test!");
02576       CasesBits.push_back(CaseBits(0, Dest, 0, 0/*Weight*/));
02577       count++;
02578     }
02579 
02580     const APInt& lowValue = cast<ConstantInt>(I->Low)->getValue();
02581     const APInt& highValue = cast<ConstantInt>(I->High)->getValue();
02582 
02583     uint64_t lo = (lowValue - lowBound).getZExtValue();
02584     uint64_t hi = (highValue - lowBound).getZExtValue();
02585     CasesBits[i].ExtraWeight += I->ExtraWeight;
02586 
02587     for (uint64_t j = lo; j <= hi; j++) {
02588       CasesBits[i].Mask |=  1ULL << j;
02589       CasesBits[i].Bits++;
02590     }
02591 
02592   }
02593   std::sort(CasesBits.begin(), CasesBits.end(), CaseBitsCmp());
02594 
02595   BitTestInfo BTC;
02596 
02597   // Figure out which block is immediately after the current one.
02598   MachineFunction::iterator BBI = CR.CaseBB;
02599   ++BBI;
02600 
02601   const BasicBlock *LLVMBB = CR.CaseBB->getBasicBlock();
02602 
02603   DEBUG(dbgs() << "Cases:\n");
02604   for (unsigned i = 0, e = CasesBits.size(); i!=e; ++i) {
02605     DEBUG(dbgs() << "Mask: " << CasesBits[i].Mask
02606                  << ", Bits: " << CasesBits[i].Bits
02607                  << ", BB: " << CasesBits[i].BB << '\n');
02608 
02609     MachineBasicBlock *CaseBB = CurMF->CreateMachineBasicBlock(LLVMBB);
02610     CurMF->insert(BBI, CaseBB);
02611     BTC.push_back(BitTestCase(CasesBits[i].Mask,
02612                               CaseBB,
02613                               CasesBits[i].BB, CasesBits[i].ExtraWeight));
02614 
02615     // Put SV in a virtual register to make it available from the new blocks.
02616     ExportFromCurrentBlock(SV);
02617   }
02618 
02619   BitTestBlock BTB(lowBound, cmpRange, SV,
02620                    -1U, MVT::Other, (CR.CaseBB == SwitchBB),
02621                    CR.CaseBB, Default, std::move(BTC));
02622 
02623   if (CR.CaseBB == SwitchBB)
02624     visitBitTestHeader(BTB, SwitchBB);
02625 
02626   BitTestCases.push_back(std::move(BTB));
02627 
02628   return true;
02629 }
02630 
02631 /// Clusterify - Transform simple list of Cases into list of CaseRange's
02632 void SelectionDAGBuilder::Clusterify(CaseVector& Cases,
02633                                      const SwitchInst& SI) {
02634   BranchProbabilityInfo *BPI = FuncInfo.BPI;
02635   // Start with "simple" cases.
02636   for (SwitchInst::ConstCaseIt i : SI.cases()) {
02637     const BasicBlock *SuccBB = i.getCaseSuccessor();
02638     MachineBasicBlock *SMBB = FuncInfo.MBBMap[SuccBB];
02639 
02640     uint32_t ExtraWeight =
02641       BPI ? BPI->getEdgeWeight(SI.getParent(), i.getSuccessorIndex()) : 0;
02642 
02643     Cases.push_back(Case(i.getCaseValue(), i.getCaseValue(),
02644                          SMBB, ExtraWeight));
02645   }
02646   std::sort(Cases.begin(), Cases.end(), CaseCmp());
02647 
02648   // Merge case into clusters
02649   if (Cases.size() >= 2)
02650     // Must recompute end() each iteration because it may be
02651     // invalidated by erase if we hold on to it
02652     for (CaseItr I = Cases.begin(), J = std::next(Cases.begin());
02653          J != Cases.end(); ) {
02654       const APInt& nextValue = cast<ConstantInt>(J->Low)->getValue();
02655       const APInt& currentValue = cast<ConstantInt>(I->High)->getValue();
02656       MachineBasicBlock* nextBB = J->BB;
02657       MachineBasicBlock* currentBB = I->BB;
02658 
02659       // If the two neighboring cases go to the same destination, merge them
02660       // into a single case.
02661       if ((nextValue - currentValue == 1) && (currentBB == nextBB)) {
02662         I->High = J->High;
02663         I->ExtraWeight += J->ExtraWeight;
02664         J = Cases.erase(J);
02665       } else {
02666         I = J++;
02667       }
02668     }
02669 
02670   DEBUG({
02671       size_t numCmps = 0;
02672       for (auto &I : Cases)
02673         // A range counts double, since it requires two compares.
02674         numCmps += I.Low != I.High ? 2 : 1;
02675 
02676       dbgs() << "Clusterify finished. Total clusters: " << Cases.size()
02677              << ". Total compares: " << numCmps << '\n';
02678     });
02679 }
02680 
02681 void SelectionDAGBuilder::UpdateSplitBlock(MachineBasicBlock *First,
02682                                            MachineBasicBlock *Last) {
02683   // Update JTCases.
02684   for (unsigned i = 0, e = JTCases.size(); i != e; ++i)
02685     if (JTCases[i].first.HeaderBB == First)
02686       JTCases[i].first.HeaderBB = Last;
02687 
02688   // Update BitTestCases.
02689   for (unsigned i = 0, e = BitTestCases.size(); i != e; ++i)
02690     if (BitTestCases[i].Parent == First)
02691       BitTestCases[i].Parent = Last;
02692 }
02693 
02694 void SelectionDAGBuilder::visitSwitch(const SwitchInst &SI) {
02695   MachineBasicBlock *SwitchMBB = FuncInfo.MBB;
02696 
02697   // Figure out which block is immediately after the current one.
02698   MachineBasicBlock *NextBlock = nullptr;
02699   if (SwitchMBB + 1 != FuncInfo.MF->end())
02700     NextBlock = SwitchMBB + 1;
02701 
02702 
02703   // Create a vector of Cases, sorted so that we can efficiently create a binary
02704   // search tree from them.
02705   CaseVector Cases;
02706   Clusterify(Cases, SI);
02707 
02708   // Get the default destination MBB.
02709   MachineBasicBlock *Default = FuncInfo.MBBMap[SI.getDefaultDest()];
02710 
02711   if (isa<UnreachableInst>(SI.getDefaultDest()->getFirstNonPHIOrDbg()) &&
02712       !Cases.empty()) {
02713     // Replace an unreachable default destination with the most popular case
02714     // destination.
02715     DenseMap<const BasicBlock *, unsigned> Popularity;
02716     unsigned MaxPop = 0;
02717     const BasicBlock *MaxBB = nullptr;
02718     for (auto I : SI.cases()) {
02719       const BasicBlock *BB = I.getCaseSuccessor();
02720       if (++Popularity[BB] > MaxPop) {
02721         MaxPop = Popularity[BB];
02722         MaxBB = BB;
02723       }
02724     }
02725 
02726     // Set new default.
02727     assert(MaxPop > 0);
02728     assert(MaxBB);
02729     Default = FuncInfo.MBBMap[MaxBB];
02730 
02731     // Remove cases that were pointing to the destination that is now the default.
02732     Cases.erase(std::remove_if(Cases.begin(), Cases.end(),
02733                                [&](const Case &C) { return C.BB == Default; }),
02734                 Cases.end());
02735   }
02736 
02737   // If there is only the default destination, go there directly.
02738   if (Cases.empty()) {
02739     // Update machine-CFG edges.
02740     SwitchMBB->addSuccessor(Default);
02741 
02742     // If this is not a fall-through branch, emit the branch.
02743     if (Default != NextBlock) {
02744       DAG.setRoot(DAG.getNode(ISD::BR, getCurSDLoc(), MVT::Other,
02745                               getControlRoot(), DAG.getBasicBlock(Default)));
02746     }
02747     return;
02748   }
02749 
02750   // Get the Value to be switched on.
02751   const Value *SV = SI.getCondition();
02752 
02753   // Push the initial CaseRec onto the worklist
02754   CaseRecVector WorkList;
02755   WorkList.push_back(CaseRec(SwitchMBB,nullptr,nullptr,
02756                              CaseRange(Cases.begin(),Cases.end())));
02757 
02758   while (!WorkList.empty()) {
02759     // Grab a record representing a case range to process off the worklist
02760     CaseRec CR = WorkList.back();
02761     WorkList.pop_back();
02762 
02763     if (handleBitTestsSwitchCase(CR, WorkList, SV, Default, SwitchMBB))
02764       continue;
02765 
02766     // If the range has few cases (two or less) emit a series of specific
02767     // tests.
02768     if (handleSmallSwitchRange(CR, WorkList, SV, Default, SwitchMBB))
02769       continue;
02770 
02771     // If the switch has more than N blocks, and is at least 40% dense, and the
02772     // target supports indirect branches, then emit a jump table rather than
02773     // lowering the switch to a binary tree of conditional branches.
02774     // N defaults to 4 and is controlled via TLS.getMinimumJumpTableEntries().
02775     if (handleJTSwitchCase(CR, WorkList, SV, Default, SwitchMBB))
02776       continue;
02777 
02778     // Emit binary tree. We need to pick a pivot, and push left and right ranges
02779     // onto the worklist. Leafs are handled via handleSmallSwitchRange() call.
02780     handleBTSplitSwitchCase(CR, WorkList, SV, SwitchMBB);
02781   }
02782 }
02783 
02784 void SelectionDAGBuilder::visitIndirectBr(const IndirectBrInst &I) {
02785   MachineBasicBlock *IndirectBrMBB = FuncInfo.MBB;
02786 
02787   // Update machine-CFG edges with unique successors.
02788   SmallSet<BasicBlock*, 32> Done;
02789   for (unsigned i = 0, e = I.getNumSuccessors(); i != e; ++i) {
02790     BasicBlock *BB = I.getSuccessor(i);
02791     bool Inserted = Done.insert(BB).second;
02792     if (!Inserted)
02793         continue;
02794 
02795     MachineBasicBlock *Succ = FuncInfo.MBBMap[BB];
02796     addSuccessorWithWeight(IndirectBrMBB, Succ);
02797   }
02798 
02799   DAG.setRoot(DAG.getNode(ISD::BRIND, getCurSDLoc(),
02800                           MVT::Other, getControlRoot(),
02801                           getValue(I.getAddress())));
02802 }
02803 
02804 void SelectionDAGBuilder::visitUnreachable(const UnreachableInst &I) {
02805   if (DAG.getTarget().Options.TrapUnreachable)
02806     DAG.setRoot(DAG.getNode(ISD::TRAP, getCurSDLoc(), MVT::Other, DAG.getRoot()));
02807 }
02808 
02809 void SelectionDAGBuilder::visitFSub(const User &I) {
02810   // -0.0 - X --> fneg
02811   Type *Ty = I.getType();
02812   if (isa<Constant>(I.getOperand(0)) &&
02813       I.getOperand(0) == ConstantFP::getZeroValueForNegation(Ty)) {
02814     SDValue Op2 = getValue(I.getOperand(1));
02815     setValue(&I, DAG.getNode(ISD::FNEG, getCurSDLoc(),
02816                              Op2.getValueType(), Op2));
02817     return;
02818   }
02819 
02820   visitBinary(I, ISD::FSUB);
02821 }
02822 
02823 void SelectionDAGBuilder::visitBinary(const User &I, unsigned OpCode) {
02824   SDValue Op1 = getValue(I.getOperand(0));
02825   SDValue Op2 = getValue(I.getOperand(1));
02826 
02827   bool nuw = false;
02828   bool nsw = false;
02829   bool exact = false;
02830   if (const OverflowingBinaryOperator *OFBinOp =
02831           dyn_cast<const OverflowingBinaryOperator>(&I)) {
02832     nuw = OFBinOp->hasNoUnsignedWrap();
02833     nsw = OFBinOp->hasNoSignedWrap();
02834   }
02835   if (const PossiblyExactOperator *ExactOp =
02836           dyn_cast<const PossiblyExactOperator>(&I))
02837     exact = ExactOp->isExact();
02838 
02839   SDValue BinNodeValue = DAG.getNode(OpCode, getCurSDLoc(), Op1.getValueType(),
02840                                      Op1, Op2, nuw, nsw, exact);
02841   setValue(&I, BinNodeValue);
02842 }
02843 
02844 void SelectionDAGBuilder::visitShift(const User &I, unsigned Opcode) {
02845   SDValue Op1 = getValue(I.getOperand(0));
02846   SDValue Op2 = getValue(I.getOperand(1));
02847 
02848   EVT ShiftTy =
02849       DAG.getTargetLoweringInfo().getShiftAmountTy(Op2.getValueType());
02850 
02851   // Coerce the shift amount to the right type if we can.
02852   if (!I.getType()->isVectorTy() && Op2.getValueType() != ShiftTy) {
02853     unsigned ShiftSize = ShiftTy.getSizeInBits();
02854     unsigned Op2Size = Op2.getValueType().getSizeInBits();
02855     SDLoc DL = getCurSDLoc();
02856 
02857     // If the operand is smaller than the shift count type, promote it.
02858     if (ShiftSize > Op2Size)
02859       Op2 = DAG.getNode(ISD::ZERO_EXTEND, DL, ShiftTy, Op2);
02860 
02861     // If the operand is larger than the shift count type but the shift
02862     // count type has enough bits to represent any shift value, truncate
02863     // it now. This is a common case and it exposes the truncate to
02864     // optimization early.
02865     else if (ShiftSize >= Log2_32_Ceil(Op2.getValueType().getSizeInBits()))
02866       Op2 = DAG.getNode(ISD::TRUNCATE, DL, ShiftTy, Op2);
02867     // Otherwise we'll need to temporarily settle for some other convenient
02868     // type.  Type legalization will make adjustments once the shiftee is split.
02869     else
02870       Op2 = DAG.getZExtOrTrunc(Op2, DL, MVT::i32);
02871   }
02872 
02873   bool nuw = false;
02874   bool nsw = false;
02875   bool exact = false;
02876 
02877   if (Opcode == ISD::SRL || Opcode == ISD::SRA || Opcode == ISD::SHL) {
02878 
02879     if (const OverflowingBinaryOperator *OFBinOp =
02880             dyn_cast<const OverflowingBinaryOperator>(&I)) {
02881       nuw = OFBinOp->hasNoUnsignedWrap();
02882       nsw = OFBinOp->hasNoSignedWrap();
02883     }
02884     if (const PossiblyExactOperator *ExactOp =
02885             dyn_cast<const PossiblyExactOperator>(&I))
02886       exact = ExactOp->isExact();
02887   }
02888 
02889   SDValue Res = DAG.getNode(Opcode, getCurSDLoc(), Op1.getValueType(), Op1, Op2,
02890                             nuw, nsw, exact);
02891   setValue(&I, Res);
02892 }
02893 
02894 void SelectionDAGBuilder::visitSDiv(const User &I) {
02895   SDValue Op1 = getValue(I.getOperand(0));
02896   SDValue Op2 = getValue(I.getOperand(1));
02897 
02898   // Turn exact SDivs into multiplications.
02899   // FIXME: This should be in DAGCombiner, but it doesn't have access to the
02900   // exact bit.
02901   if (isa<BinaryOperator>(&I) && cast<BinaryOperator>(&I)->isExact() &&
02902       !isa<ConstantSDNode>(Op1) &&
02903       isa<ConstantSDNode>(Op2) && !cast<ConstantSDNode>(Op2)->isNullValue())
02904     setValue(&I, DAG.getTargetLoweringInfo()
02905                      .BuildExactSDIV(Op1, Op2, getCurSDLoc(), DAG));
02906   else
02907     setValue(&I, DAG.getNode(ISD::SDIV, getCurSDLoc(), Op1.getValueType(),
02908                              Op1, Op2));
02909 }
02910 
02911 void SelectionDAGBuilder::visitICmp(const User &I) {
02912   ICmpInst::Predicate predicate = ICmpInst::BAD_ICMP_PREDICATE;
02913   if (const ICmpInst *IC = dyn_cast<ICmpInst>(&I))
02914     predicate = IC->getPredicate();
02915   else if (const ConstantExpr *IC = dyn_cast<ConstantExpr>(&I))
02916     predicate = ICmpInst::Predicate(IC->getPredicate());
02917   SDValue Op1 = getValue(I.getOperand(0));
02918   SDValue Op2 = getValue(I.getOperand(1));
02919   ISD::CondCode Opcode = getICmpCondCode(predicate);
02920 
02921   EVT DestVT = DAG.getTargetLoweringInfo().getValueType(I.getType());
02922   setValue(&I, DAG.getSetCC(getCurSDLoc(), DestVT, Op1, Op2, Opcode));
02923 }
02924 
02925 void SelectionDAGBuilder::visitFCmp(const User &I) {
02926   FCmpInst::Predicate predicate = FCmpInst::BAD_FCMP_PREDICATE;
02927   if (const FCmpInst *FC = dyn_cast<FCmpInst>(&I))
02928     predicate = FC->getPredicate();
02929   else if (const ConstantExpr *FC = dyn_cast<ConstantExpr>(&I))
02930     predicate = FCmpInst::Predicate(FC->getPredicate());
02931   SDValue Op1 = getValue(I.getOperand(0));
02932   SDValue Op2 = getValue(I.getOperand(1));
02933   ISD::CondCode Condition = getFCmpCondCode(predicate);
02934   if (TM.Options.NoNaNsFPMath)
02935     Condition = getFCmpCodeWithoutNaN(Condition);
02936   EVT DestVT = DAG.getTargetLoweringInfo().getValueType(I.getType());
02937   setValue(&I, DAG.getSetCC(getCurSDLoc(), DestVT, Op1, Op2, Condition));
02938 }
02939 
02940 void SelectionDAGBuilder::visitSelect(const User &I) {
02941   SmallVector<EVT, 4> ValueVTs;
02942   ComputeValueVTs(DAG.getTargetLoweringInfo(), I.getType(), ValueVTs);
02943   unsigned NumValues = ValueVTs.size();
02944   if (NumValues == 0) return;
02945 
02946   SmallVector<SDValue, 4> Values(NumValues);
02947   SDValue Cond     = getValue(I.getOperand(0));
02948   SDValue TrueVal  = getValue(I.getOperand(1));
02949   SDValue FalseVal = getValue(I.getOperand(2));
02950   ISD::NodeType OpCode = Cond.getValueType().isVector() ?
02951     ISD::VSELECT : ISD::SELECT;
02952 
02953   for (unsigned i = 0; i != NumValues; ++i)
02954     Values[i] = DAG.getNode(OpCode, getCurSDLoc(),
02955                             TrueVal.getNode()->getValueType(TrueVal.getResNo()+i),
02956                             Cond,
02957                             SDValue(TrueVal.getNode(),
02958                                     TrueVal.getResNo() + i),
02959                             SDValue(FalseVal.getNode(),
02960                                     FalseVal.getResNo() + i));
02961 
02962   setValue(&I, DAG.getNode(ISD::MERGE_VALUES, getCurSDLoc(),
02963                            DAG.getVTList(ValueVTs), Values));
02964 }
02965 
02966 void SelectionDAGBuilder::visitTrunc(const User &I) {
02967   // TruncInst cannot be a no-op cast because sizeof(src) > sizeof(dest).
02968   SDValue N = getValue(I.getOperand(0));
02969   EVT DestVT = DAG.getTargetLoweringInfo().getValueType(I.getType());
02970   setValue(&I, DAG.getNode(ISD::TRUNCATE, getCurSDLoc(), DestVT, N));
02971 }
02972 
02973 void SelectionDAGBuilder::visitZExt(const User &I) {
02974   // ZExt cannot be a no-op cast because sizeof(src) < sizeof(dest).
02975   // ZExt also can't be a cast to bool for same reason. So, nothing much to do
02976   SDValue N = getValue(I.getOperand(0));
02977   EVT DestVT = DAG.getTargetLoweringInfo().getValueType(I.getType());
02978   setValue(&I, DAG.getNode(ISD::ZERO_EXTEND, getCurSDLoc(), DestVT, N));
02979 }
02980 
02981 void SelectionDAGBuilder::visitSExt(const User &I) {
02982   // SExt cannot be a no-op cast because sizeof(src) < sizeof(dest).
02983   // SExt also can't be a cast to bool for same reason. So, nothing much to do
02984   SDValue N = getValue(I.getOperand(0));
02985   EVT DestVT = DAG.getTargetLoweringInfo().getValueType(I.getType());
02986   setValue(&I, DAG.getNode(ISD::SIGN_EXTEND, getCurSDLoc(), DestVT, N));
02987 }
02988 
02989 void SelectionDAGBuilder::visitFPTrunc(const User &I) {
02990   // FPTrunc is never a no-op cast, no need to check
02991   SDValue N = getValue(I.getOperand(0));
02992   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
02993   EVT DestVT = TLI.getValueType(I.getType());
02994   setValue(&I, DAG.getNode(ISD::FP_ROUND, getCurSDLoc(), DestVT, N,
02995                            DAG.getTargetConstant(0, TLI.getPointerTy())));
02996 }
02997 
02998 void SelectionDAGBuilder::visitFPExt(const User &I) {
02999   // FPExt is never a no-op cast, no need to check
03000   SDValue N = getValue(I.getOperand(0));
03001   EVT DestVT = DAG.getTargetLoweringInfo().getValueType(I.getType());
03002   setValue(&I, DAG.getNode(ISD::FP_EXTEND, getCurSDLoc(), DestVT, N));
03003 }
03004 
03005 void SelectionDAGBuilder::visitFPToUI(const User &I) {
03006   // FPToUI is never a no-op cast, no need to check
03007   SDValue N = getValue(I.getOperand(0));
03008   EVT DestVT = DAG.getTargetLoweringInfo().getValueType(I.getType());
03009   setValue(&I, DAG.getNode(ISD::FP_TO_UINT, getCurSDLoc(), DestVT, N));
03010 }
03011 
03012 void SelectionDAGBuilder::visitFPToSI(const User &I) {
03013   // FPToSI is never a no-op cast, no need to check
03014   SDValue N = getValue(I.getOperand(0));
03015   EVT DestVT = DAG.getTargetLoweringInfo().getValueType(I.getType());
03016   setValue(&I, DAG.getNode(ISD::FP_TO_SINT, getCurSDLoc(), DestVT, N));
03017 }
03018 
03019 void SelectionDAGBuilder::visitUIToFP(const User &I) {
03020   // UIToFP is never a no-op cast, no need to check
03021   SDValue N = getValue(I.getOperand(0));
03022   EVT DestVT = DAG.getTargetLoweringInfo().getValueType(I.getType());
03023   setValue(&I, DAG.getNode(ISD::UINT_TO_FP, getCurSDLoc(), DestVT, N));
03024 }
03025 
03026 void SelectionDAGBuilder::visitSIToFP(const User &I) {
03027   // SIToFP is never a no-op cast, no need to check
03028   SDValue N = getValue(I.getOperand(0));
03029   EVT DestVT = DAG.getTargetLoweringInfo().getValueType(I.getType());
03030   setValue(&I, DAG.getNode(ISD::SINT_TO_FP, getCurSDLoc(), DestVT, N));
03031 }
03032 
03033 void SelectionDAGBuilder::visitPtrToInt(const User &I) {
03034   // What to do depends on the size of the integer and the size of the pointer.
03035   // We can either truncate, zero extend, or no-op, accordingly.
03036   SDValue N = getValue(I.getOperand(0));
03037   EVT DestVT = DAG.getTargetLoweringInfo().getValueType(I.getType());
03038   setValue(&I, DAG.getZExtOrTrunc(N, getCurSDLoc(), DestVT));
03039 }
03040 
03041 void SelectionDAGBuilder::visitIntToPtr(const User &I) {
03042   // What to do depends on the size of the integer and the size of the pointer.
03043   // We can either truncate, zero extend, or no-op, accordingly.
03044   SDValue N = getValue(I.getOperand(0));
03045   EVT DestVT = DAG.getTargetLoweringInfo().getValueType(I.getType());
03046   setValue(&I, DAG.getZExtOrTrunc(N, getCurSDLoc(), DestVT));
03047 }
03048 
03049 void SelectionDAGBuilder::visitBitCast(const User &I) {
03050   SDValue N = getValue(I.getOperand(0));
03051   EVT DestVT = DAG.getTargetLoweringInfo().getValueType(I.getType());
03052 
03053   // BitCast assures us that source and destination are the same size so this is
03054   // either a BITCAST or a no-op.
03055   if (DestVT != N.getValueType())
03056     setValue(&I, DAG.getNode(ISD::BITCAST, getCurSDLoc(),
03057                              DestVT, N)); // convert types.
03058   // Check if the original LLVM IR Operand was a ConstantInt, because getValue()
03059   // might fold any kind of constant expression to an integer constant and that
03060   // is not what we are looking for. Only regcognize a bitcast of a genuine
03061   // constant integer as an opaque constant.
03062   else if(ConstantInt *C = dyn_cast<ConstantInt>(I.getOperand(0)))
03063     setValue(&I, DAG.getConstant(C->getValue(), DestVT, /*isTarget=*/false,
03064                                  /*isOpaque*/true));
03065   else
03066     setValue(&I, N);            // noop cast.
03067 }
03068 
03069 void SelectionDAGBuilder::visitAddrSpaceCast(const User &I) {
03070   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
03071   const Value *SV = I.getOperand(0);
03072   SDValue N = getValue(SV);
03073   EVT DestVT = TLI.getValueType(I.getType());
03074 
03075   unsigned SrcAS = SV->getType()->getPointerAddressSpace();
03076   unsigned DestAS = I.getType()->getPointerAddressSpace();
03077 
03078   if (!TLI.isNoopAddrSpaceCast(SrcAS, DestAS))
03079     N = DAG.getAddrSpaceCast(getCurSDLoc(), DestVT, N, SrcAS, DestAS);
03080 
03081   setValue(&I, N);
03082 }
03083 
03084 void SelectionDAGBuilder::visitInsertElement(const User &I) {
03085   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
03086   SDValue InVec = getValue(I.getOperand(0));
03087   SDValue InVal = getValue(I.getOperand(1));
03088   SDValue InIdx = DAG.getSExtOrTrunc(getValue(I.getOperand(2)),
03089                                      getCurSDLoc(), TLI.getVectorIdxTy());
03090   setValue(&I, DAG.getNode(ISD::INSERT_VECTOR_ELT, getCurSDLoc(),
03091                            TLI.getValueType(I.getType()), InVec, InVal, InIdx));
03092 }
03093 
03094 void SelectionDAGBuilder::visitExtractElement(const User &I) {
03095   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
03096   SDValue InVec = getValue(I.getOperand(0));
03097   SDValue InIdx = DAG.getSExtOrTrunc(getValue(I.getOperand(1)),
03098                                      getCurSDLoc(), TLI.getVectorIdxTy());
03099   setValue(&I, DAG.getNode(ISD::EXTRACT_VECTOR_ELT, getCurSDLoc(),
03100                            TLI.getValueType(I.getType()), InVec, InIdx));
03101 }
03102 
03103 // Utility for visitShuffleVector - Return true if every element in Mask,
03104 // beginning from position Pos and ending in Pos+Size, falls within the
03105 // specified sequential range [L, L+Pos). or is undef.
03106 static bool isSequentialInRange(const SmallVectorImpl<int> &Mask,
03107                                 unsigned Pos, unsigned Size, int Low) {
03108   for (unsigned i = Pos, e = Pos+Size; i != e; ++i, ++Low)
03109     if (Mask[i] >= 0 && Mask[i] != Low)
03110       return false;
03111   return true;
03112 }
03113 
03114 void SelectionDAGBuilder::visitShuffleVector(const User &I) {
03115   SDValue Src1 = getValue(I.getOperand(0));
03116   SDValue Src2 = getValue(I.getOperand(1));
03117 
03118   SmallVector<int, 8> Mask;
03119   ShuffleVectorInst::getShuffleMask(cast<Constant>(I.getOperand(2)), Mask);
03120   unsigned MaskNumElts = Mask.size();
03121 
03122   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
03123   EVT VT = TLI.getValueType(I.getType());
03124   EVT SrcVT = Src1.getValueType();
03125   unsigned SrcNumElts = SrcVT.getVectorNumElements();
03126 
03127   if (SrcNumElts == MaskNumElts) {
03128     setValue(&I, DAG.getVectorShuffle(VT, getCurSDLoc(), Src1, Src2,
03129                                       &Mask[0]));
03130     return;
03131   }
03132 
03133   // Normalize the shuffle vector since mask and vector length don't match.
03134   if (SrcNumElts < MaskNumElts && MaskNumElts % SrcNumElts == 0) {
03135     // Mask is longer than the source vectors and is a multiple of the source
03136     // vectors.  We can use concatenate vector to make the mask and vectors
03137     // lengths match.
03138     if (SrcNumElts*2 == MaskNumElts) {
03139       // First check for Src1 in low and Src2 in high
03140       if (isSequentialInRange(Mask, 0, SrcNumElts, 0) &&
03141           isSequentialInRange(Mask, SrcNumElts, SrcNumElts, SrcNumElts)) {
03142         // The shuffle is concatenating two vectors together.
03143         setValue(&I, DAG.getNode(ISD::CONCAT_VECTORS, getCurSDLoc(),
03144                                  VT, Src1, Src2));
03145         return;
03146       }
03147       // Then check for Src2 in low and Src1 in high
03148       if (isSequentialInRange(Mask, 0, SrcNumElts, SrcNumElts) &&
03149           isSequentialInRange(Mask, SrcNumElts, SrcNumElts, 0)) {
03150         // The shuffle is concatenating two vectors together.
03151         setValue(&I, DAG.getNode(ISD::CONCAT_VECTORS, getCurSDLoc(),
03152                                  VT, Src2, Src1));
03153         return;
03154       }
03155     }
03156 
03157     // Pad both vectors with undefs to make them the same length as the mask.
03158     unsigned NumConcat = MaskNumElts / SrcNumElts;
03159     bool Src1U = Src1.getOpcode() == ISD::UNDEF;
03160     bool Src2U = Src2.getOpcode() == ISD::UNDEF;
03161     SDValue UndefVal = DAG.getUNDEF(SrcVT);
03162 
03163     SmallVector<SDValue, 8> MOps1(NumConcat, UndefVal);
03164     SmallVector<SDValue, 8> MOps2(NumConcat, UndefVal);
03165     MOps1[0] = Src1;
03166     MOps2[0] = Src2;
03167 
03168     Src1 = Src1U ? DAG.getUNDEF(VT) : DAG.getNode(ISD::CONCAT_VECTORS,
03169                                                   getCurSDLoc(), VT, MOps1);
03170     Src2 = Src2U ? DAG.getUNDEF(VT) : DAG.getNode(ISD::CONCAT_VECTORS,
03171                                                   getCurSDLoc(), VT, MOps2);
03172 
03173     // Readjust mask for new input vector length.
03174     SmallVector<int, 8> MappedOps;
03175     for (unsigned i = 0; i != MaskNumElts; ++i) {
03176       int Idx = Mask[i];
03177       if (Idx >= (int)SrcNumElts)
03178         Idx -= SrcNumElts - MaskNumElts;
03179       MappedOps.push_back(Idx);
03180     }
03181 
03182     setValue(&I, DAG.getVectorShuffle(VT, getCurSDLoc(), Src1, Src2,
03183                                       &MappedOps[0]));
03184     return;
03185   }
03186 
03187   if (SrcNumElts > MaskNumElts) {
03188     // Analyze the access pattern of the vector to see if we can extract
03189     // two subvectors and do the shuffle. The analysis is done by calculating
03190     // the range of elements the mask access on both vectors.
03191     int MinRange[2] = { static_cast<int>(SrcNumElts),
03192                         static_cast<int>(SrcNumElts)};
03193     int MaxRange[2] = {-1, -1};
03194 
03195     for (unsigned i = 0; i != MaskNumElts; ++i) {
03196       int Idx = Mask[i];
03197       unsigned Input = 0;
03198       if (Idx < 0)
03199         continue;
03200 
03201       if (Idx >= (int)SrcNumElts) {
03202         Input = 1;
03203         Idx -= SrcNumElts;
03204       }
03205       if (Idx > MaxRange[Input])
03206         MaxRange[Input] = Idx;
03207       if (Idx < MinRange[Input])
03208         MinRange[Input] = Idx;
03209     }
03210 
03211     // Check if the access is smaller than the vector size and can we find
03212     // a reasonable extract index.
03213     int RangeUse[2] = { -1, -1 };  // 0 = Unused, 1 = Extract, -1 = Can not
03214                                    // Extract.
03215     int StartIdx[2];  // StartIdx to extract from
03216     for (unsigned Input = 0; Input < 2; ++Input) {
03217       if (MinRange[Input] >= (int)SrcNumElts && MaxRange[Input] < 0) {
03218         RangeUse[Input] = 0; // Unused
03219         StartIdx[Input] = 0;
03220         continue;
03221       }
03222 
03223       // Find a good start index that is a multiple of the mask length. Then
03224       // see if the rest of the elements are in range.
03225       StartIdx[Input] = (MinRange[Input]/MaskNumElts)*MaskNumElts;
03226       if (MaxRange[Input] - StartIdx[Input] < (int)MaskNumElts &&
03227           StartIdx[Input] + MaskNumElts <= SrcNumElts)
03228         RangeUse[Input] = 1; // Extract from a multiple of the mask length.
03229     }
03230 
03231     if (RangeUse[0] == 0 && RangeUse[1] == 0) {
03232       setValue(&I, DAG.getUNDEF(VT)); // Vectors are not used.
03233       return;
03234     }
03235     if (RangeUse[0] >= 0 && RangeUse[1] >= 0) {
03236       // Extract appropriate subvector and generate a vector shuffle
03237       for (unsigned Input = 0; Input < 2; ++Input) {
03238         SDValue &Src = Input == 0 ? Src1 : Src2;
03239         if (RangeUse[Input] == 0)
03240           Src = DAG.getUNDEF(VT);
03241         else
03242           Src = DAG.getNode(
03243               ISD::EXTRACT_SUBVECTOR, getCurSDLoc(), VT, Src,
03244               DAG.getConstant(StartIdx[Input], TLI.getVectorIdxTy()));
03245       }
03246 
03247       // Calculate new mask.
03248       SmallVector<int, 8> MappedOps;
03249       for (unsigned i = 0; i != MaskNumElts; ++i) {
03250         int Idx = Mask[i];
03251         if (Idx >= 0) {
03252           if (Idx < (int)SrcNumElts)
03253             Idx -= StartIdx[0];
03254           else
03255             Idx -= SrcNumElts + StartIdx[1] - MaskNumElts;
03256         }
03257         MappedOps.push_back(Idx);
03258       }
03259 
03260       setValue(&I, DAG.getVectorShuffle(VT, getCurSDLoc(), Src1, Src2,
03261                                         &MappedOps[0]));
03262       return;
03263     }
03264   }
03265 
03266   // We can't use either concat vectors or extract subvectors so fall back to
03267   // replacing the shuffle with extract and build vector.
03268   // to insert and build vector.
03269   EVT EltVT = VT.getVectorElementType();
03270   EVT IdxVT = TLI.getVectorIdxTy();
03271   SmallVector<SDValue,8> Ops;
03272   for (unsigned i = 0; i != MaskNumElts; ++i) {
03273     int Idx = Mask[i];
03274     SDValue Res;
03275 
03276     if (Idx < 0) {
03277       Res = DAG.getUNDEF(EltVT);
03278     } else {
03279       SDValue &Src = Idx < (int)SrcNumElts ? Src1 : Src2;
03280       if (Idx >= (int)SrcNumElts) Idx -= SrcNumElts;
03281 
03282       Res = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, getCurSDLoc(),
03283                         EltVT, Src, DAG.getConstant(Idx, IdxVT));
03284     }
03285 
03286     Ops.push_back(Res);
03287   }
03288 
03289   setValue(&I, DAG.getNode(ISD::BUILD_VECTOR, getCurSDLoc(), VT, Ops));
03290 }
03291 
03292 void SelectionDAGBuilder::visitInsertValue(const InsertValueInst &I) {
03293   const Value *Op0 = I.getOperand(0);
03294   const Value *Op1 = I.getOperand(1);
03295   Type *AggTy = I.getType();
03296   Type *ValTy = Op1->getType();
03297   bool IntoUndef = isa<UndefValue>(Op0);
03298   bool FromUndef = isa<UndefValue>(Op1);
03299 
03300   unsigned LinearIndex = ComputeLinearIndex(AggTy, I.getIndices());
03301 
03302   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
03303   SmallVector<EVT, 4> AggValueVTs;
03304   ComputeValueVTs(TLI, AggTy, AggValueVTs);
03305   SmallVector<EVT, 4> ValValueVTs;
03306   ComputeValueVTs(TLI, ValTy, ValValueVTs);
03307 
03308   unsigned NumAggValues = AggValueVTs.size();
03309   unsigned NumValValues = ValValueVTs.size();
03310   SmallVector<SDValue, 4> Values(NumAggValues);
03311 
03312   // Ignore an insertvalue that produces an empty object
03313   if (!NumAggValues) {
03314     setValue(&I, DAG.getUNDEF(MVT(MVT::Other)));
03315     return;
03316   }
03317 
03318   SDValue Agg = getValue(Op0);
03319   unsigned i = 0;
03320   // Copy the beginning value(s) from the original aggregate.
03321   for (; i != LinearIndex; ++i)
03322     Values[i] = IntoUndef ? DAG.getUNDEF(AggValueVTs[i]) :
03323                 SDValue(Agg.getNode(), Agg.getResNo() + i);
03324   // Copy values from the inserted value(s).
03325   if (NumValValues) {
03326     SDValue Val = getValue(Op1);
03327     for (; i != LinearIndex + NumValValues; ++i)
03328       Values[i] = FromUndef ? DAG.getUNDEF(AggValueVTs[i]) :
03329                   SDValue(Val.getNode(), Val.getResNo() + i - LinearIndex);
03330   }
03331   // Copy remaining value(s) from the original aggregate.
03332   for (; i != NumAggValues; ++i)
03333     Values[i] = IntoUndef ? DAG.getUNDEF(AggValueVTs[i]) :
03334                 SDValue(Agg.getNode(), Agg.getResNo() + i);
03335 
03336   setValue(&I, DAG.getNode(ISD::MERGE_VALUES, getCurSDLoc(),
03337                            DAG.getVTList(AggValueVTs), Values));
03338 }
03339 
03340 void SelectionDAGBuilder::visitExtractValue(const ExtractValueInst &I) {
03341   const Value *Op0 = I.getOperand(0);
03342   Type *AggTy = Op0->getType();
03343   Type *ValTy = I.getType();
03344   bool OutOfUndef = isa<UndefValue>(Op0);
03345 
03346   unsigned LinearIndex = ComputeLinearIndex(AggTy, I.getIndices());
03347 
03348   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
03349   SmallVector<EVT, 4> ValValueVTs;
03350   ComputeValueVTs(TLI, ValTy, ValValueVTs);
03351 
03352   unsigned NumValValues = ValValueVTs.size();
03353 
03354   // Ignore a extractvalue that produces an empty object
03355   if (!NumValValues) {
03356     setValue(&I, DAG.getUNDEF(MVT(MVT::Other)));
03357     return;
03358   }
03359 
03360   SmallVector<SDValue, 4> Values(NumValValues);
03361 
03362   SDValue Agg = getValue(Op0);
03363   // Copy out the selected value(s).
03364   for (unsigned i = LinearIndex; i != LinearIndex + NumValValues; ++i)
03365     Values[i - LinearIndex] =
03366       OutOfUndef ?
03367         DAG.getUNDEF(Agg.getNode()->getValueType(Agg.getResNo() + i)) :
03368         SDValue(Agg.getNode(), Agg.getResNo() + i);
03369 
03370   setValue(&I, DAG.getNode(ISD::MERGE_VALUES, getCurSDLoc(),
03371                            DAG.getVTList(ValValueVTs), Values));
03372 }
03373 
03374 void SelectionDAGBuilder::visitGetElementPtr(const User &I) {
03375   Value *Op0 = I.getOperand(0);
03376   // Note that the pointer operand may be a vector of pointers. Take the scalar
03377   // element which holds a pointer.
03378   Type *Ty = Op0->getType()->getScalarType();
03379   unsigned AS = Ty->getPointerAddressSpace();
03380   SDValue N = getValue(Op0);
03381 
03382   for (GetElementPtrInst::const_op_iterator OI = I.op_begin()+1, E = I.op_end();
03383        OI != E; ++OI) {
03384     const Value *Idx = *OI;
03385     if (StructType *StTy = dyn_cast<StructType>(Ty)) {
03386       unsigned Field = cast<Constant>(Idx)->getUniqueInteger().getZExtValue();
03387       if (Field) {
03388         // N = N + Offset
03389         uint64_t Offset = DL->getStructLayout(StTy)->getElementOffset(Field);
03390         N = DAG.getNode(ISD::ADD, getCurSDLoc(), N.getValueType(), N,
03391                         DAG.getConstant(Offset, N.getValueType()));
03392       }
03393 
03394       Ty = StTy->getElementType(Field);
03395     } else {
03396       Ty = cast<SequentialType>(Ty)->getElementType();
03397 
03398       // If this is a constant subscript, handle it quickly.
03399       const TargetLowering &TLI = DAG.getTargetLoweringInfo();
03400       if (const ConstantInt *CI = dyn_cast<ConstantInt>(Idx)) {
03401         if (CI->isZero()) continue;
03402         uint64_t Offs =
03403             DL->getTypeAllocSize(Ty)*cast<ConstantInt>(CI)->getSExtValue();
03404         SDValue OffsVal;
03405         EVT PTy = TLI.getPointerTy(AS);
03406         unsigned PtrBits = PTy.getSizeInBits();
03407         if (PtrBits < 64)
03408           OffsVal = DAG.getNode(ISD::TRUNCATE, getCurSDLoc(), PTy,
03409                                 DAG.getConstant(Offs, MVT::i64));
03410         else
03411           OffsVal = DAG.getConstant(Offs, PTy);
03412 
03413         N = DAG.getNode(ISD::ADD, getCurSDLoc(), N.getValueType(), N,
03414                         OffsVal);
03415         continue;
03416       }
03417 
03418       // N = N + Idx * ElementSize;
03419       APInt ElementSize =
03420           APInt(TLI.getPointerSizeInBits(AS), DL->getTypeAllocSize(Ty));
03421       SDValue IdxN = getValue(Idx);
03422 
03423       // If the index is smaller or larger than intptr_t, truncate or extend
03424       // it.
03425       IdxN = DAG.getSExtOrTrunc(IdxN, getCurSDLoc(), N.getValueType());
03426 
03427       // If this is a multiply by a power of two, turn it into a shl
03428       // immediately.  This is a very common case.
03429       if (ElementSize != 1) {
03430         if (ElementSize.isPowerOf2()) {
03431           unsigned Amt = ElementSize.logBase2();
03432           IdxN = DAG.getNode(ISD::SHL, getCurSDLoc(),
03433                              N.getValueType(), IdxN,
03434                              DAG.getConstant(Amt, IdxN.getValueType()));
03435         } else {
03436           SDValue Scale = DAG.getConstant(ElementSize, IdxN.getValueType());
03437           IdxN = DAG.getNode(ISD::MUL, getCurSDLoc(),
03438                              N.getValueType(), IdxN, Scale);
03439         }
03440       }
03441 
03442       N = DAG.getNode(ISD::ADD, getCurSDLoc(),
03443                       N.getValueType(), N, IdxN);
03444     }
03445   }
03446 
03447   setValue(&I, N);
03448 }
03449 
03450 void SelectionDAGBuilder::visitAlloca(const AllocaInst &I) {
03451   // If this is a fixed sized alloca in the entry block of the function,
03452   // allocate it statically on the stack.
03453   if (FuncInfo.StaticAllocaMap.count(&I))
03454     return;   // getValue will auto-populate this.
03455 
03456   Type *Ty = I.getAllocatedType();
03457   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
03458   uint64_t TySize = TLI.getDataLayout()->getTypeAllocSize(Ty);
03459   unsigned Align =
03460       std::max((unsigned)TLI.getDataLayout()->getPrefTypeAlignment(Ty),
03461                I.getAlignment());
03462 
03463   SDValue AllocSize = getValue(I.getArraySize());
03464 
03465   EVT IntPtr = TLI.getPointerTy();
03466   if (AllocSize.getValueType() != IntPtr)
03467     AllocSize = DAG.getZExtOrTrunc(AllocSize, getCurSDLoc(), IntPtr);
03468 
03469   AllocSize = DAG.getNode(ISD::MUL, getCurSDLoc(), IntPtr,
03470                           AllocSize,
03471                           DAG.getConstant(TySize, IntPtr));
03472 
03473   // Handle alignment.  If the requested alignment is less than or equal to
03474   // the stack alignment, ignore it.  If the size is greater than or equal to
03475   // the stack alignment, we note this in the DYNAMIC_STACKALLOC node.
03476   unsigned StackAlign =
03477       DAG.getSubtarget().getFrameLowering()->getStackAlignment();
03478   if (Align <= StackAlign)
03479     Align = 0;
03480 
03481   // Round the size of the allocation up to the stack alignment size
03482   // by add SA-1 to the size.
03483   AllocSize = DAG.getNode(ISD::ADD, getCurSDLoc(),
03484                           AllocSize.getValueType(), AllocSize,
03485                           DAG.getIntPtrConstant(StackAlign-1));
03486 
03487   // Mask out the low bits for alignment purposes.
03488   AllocSize = DAG.getNode(ISD::AND, getCurSDLoc(),
03489                           AllocSize.getValueType(), AllocSize,
03490                           DAG.getIntPtrConstant(~(uint64_t)(StackAlign-1)));
03491 
03492   SDValue Ops[] = { getRoot(), AllocSize, DAG.getIntPtrConstant(Align) };
03493   SDVTList VTs = DAG.getVTList(AllocSize.getValueType(), MVT::Other);
03494   SDValue DSA = DAG.getNode(ISD::DYNAMIC_STACKALLOC, getCurSDLoc(), VTs, Ops);
03495   setValue(&I, DSA);
03496   DAG.setRoot(DSA.getValue(1));
03497 
03498   assert(FuncInfo.MF->getFrameInfo()->hasVarSizedObjects());
03499 }
03500 
03501 void SelectionDAGBuilder::visitLoad(const LoadInst &I) {
03502   if (I.isAtomic())
03503     return visitAtomicLoad(I);
03504 
03505   const Value *SV = I.getOperand(0);
03506   SDValue Ptr = getValue(SV);
03507 
03508   Type *Ty = I.getType();
03509 
03510   bool isVolatile = I.isVolatile();
03511   bool isNonTemporal = I.getMetadata(LLVMContext::MD_nontemporal) != nullptr;
03512   bool isInvariant = I.getMetadata(LLVMContext::MD_invariant_load) != nullptr;
03513   unsigned Alignment = I.getAlignment();
03514 
03515   AAMDNodes AAInfo;
03516   I.getAAMetadata(AAInfo);
03517   const MDNode *Ranges = I.getMetadata(LLVMContext::MD_range);
03518 
03519   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
03520   SmallVector<EVT, 4> ValueVTs;
03521   SmallVector<uint64_t, 4> Offsets;
03522   ComputeValueVTs(TLI, Ty, ValueVTs, &Offsets);
03523   unsigned NumValues = ValueVTs.size();
03524   if (NumValues == 0)
03525     return;
03526 
03527   SDValue Root;
03528   bool ConstantMemory = false;
03529   if (isVolatile || NumValues > MaxParallelChains)
03530     // Serialize volatile loads with other side effects.
03531     Root = getRoot();
03532   else if (AA->pointsToConstantMemory(
03533              AliasAnalysis::Location(SV, AA->getTypeStoreSize(Ty), AAInfo))) {
03534     // Do not serialize (non-volatile) loads of constant memory with anything.
03535     Root = DAG.getEntryNode();
03536     ConstantMemory = true;
03537   } else {
03538     // Do not serialize non-volatile loads against each other.
03539     Root = DAG.getRoot();
03540   }
03541 
03542   if (isVolatile)
03543     Root = TLI.prepareVolatileOrAtomicLoad(Root, getCurSDLoc(), DAG);
03544 
03545   SmallVector<SDValue, 4> Values(NumValues);
03546   SmallVector<SDValue, 4> Chains(std::min(unsigned(MaxParallelChains),
03547                                           NumValues));
03548   EVT PtrVT = Ptr.getValueType();
03549   unsigned ChainI = 0;
03550   for (unsigned i = 0; i != NumValues; ++i, ++ChainI) {
03551     // Serializing loads here may result in excessive register pressure, and
03552     // TokenFactor places arbitrary choke points on the scheduler. SD scheduling
03553     // could recover a bit by hoisting nodes upward in the chain by recognizing
03554     // they are side-effect free or do not alias. The optimizer should really
03555     // avoid this case by converting large object/array copies to llvm.memcpy
03556     // (MaxParallelChains should always remain as failsafe).
03557     if (ChainI == MaxParallelChains) {
03558       assert(PendingLoads.empty() && "PendingLoads must be serialized first");
03559       SDValue Chain = DAG.getNode(ISD::TokenFactor, getCurSDLoc(), MVT::Other,
03560                                   makeArrayRef(Chains.data(), ChainI));
03561       Root = Chain;
03562       ChainI = 0;
03563     }
03564     SDValue A = DAG.getNode(ISD::ADD, getCurSDLoc(),
03565                             PtrVT, Ptr,
03566                             DAG.getConstant(Offsets[i], PtrVT));
03567     SDValue L = DAG.getLoad(ValueVTs[i], getCurSDLoc(), Root,
03568                             A, MachinePointerInfo(SV, Offsets[i]), isVolatile,
03569                             isNonTemporal, isInvariant, Alignment, AAInfo,
03570                             Ranges);
03571 
03572     Values[i] = L;
03573     Chains[ChainI] = L.getValue(1);
03574   }
03575 
03576   if (!ConstantMemory) {
03577     SDValue Chain = DAG.getNode(ISD::TokenFactor, getCurSDLoc(), MVT::Other,
03578                                 makeArrayRef(Chains.data(), ChainI));
03579     if (isVolatile)
03580       DAG.setRoot(Chain);
03581     else
03582       PendingLoads.push_back(Chain);
03583   }
03584 
03585   setValue(&I, DAG.getNode(ISD::MERGE_VALUES, getCurSDLoc(),
03586                            DAG.getVTList(ValueVTs), Values));
03587 }
03588 
03589 void SelectionDAGBuilder::visitStore(const StoreInst &I) {
03590   if (I.isAtomic())
03591     return visitAtomicStore(I);
03592 
03593   const Value *SrcV = I.getOperand(0);
03594   const Value *PtrV = I.getOperand(1);
03595 
03596   SmallVector<EVT, 4> ValueVTs;
03597   SmallVector<uint64_t, 4> Offsets;
03598   ComputeValueVTs(DAG.getTargetLoweringInfo(), SrcV->getType(),
03599                   ValueVTs, &Offsets);
03600   unsigned NumValues = ValueVTs.size();
03601   if (NumValues == 0)
03602     return;
03603 
03604   // Get the lowered operands. Note that we do this after
03605   // checking if NumResults is zero, because with zero results
03606   // the operands won't have values in the map.
03607   SDValue Src = getValue(SrcV);
03608   SDValue Ptr = getValue(PtrV);
03609 
03610   SDValue Root = getRoot();
03611   SmallVector<SDValue, 4> Chains(std::min(unsigned(MaxParallelChains),
03612                                           NumValues));
03613   EVT PtrVT = Ptr.getValueType();
03614   bool isVolatile = I.isVolatile();
03615   bool isNonTemporal = I.getMetadata(LLVMContext::MD_nontemporal) != nullptr;
03616   unsigned Alignment = I.getAlignment();
03617 
03618   AAMDNodes AAInfo;
03619   I.getAAMetadata(AAInfo);
03620 
03621   unsigned ChainI = 0;
03622   for (unsigned i = 0; i != NumValues; ++i, ++ChainI) {
03623     // See visitLoad comments.
03624     if (ChainI == MaxParallelChains) {
03625       SDValue Chain = DAG.getNode(ISD::TokenFactor, getCurSDLoc(), MVT::Other,
03626                                   makeArrayRef(Chains.data(), ChainI));
03627       Root = Chain;
03628       ChainI = 0;
03629     }
03630     SDValue Add = DAG.getNode(ISD::ADD, getCurSDLoc(), PtrVT, Ptr,
03631                               DAG.getConstant(Offsets[i], PtrVT));
03632     SDValue St = DAG.getStore(Root, getCurSDLoc(),
03633                               SDValue(Src.getNode(), Src.getResNo() + i),
03634                               Add, MachinePointerInfo(PtrV, Offsets[i]),
03635                               isVolatile, isNonTemporal, Alignment, AAInfo);
03636     Chains[ChainI] = St;
03637   }
03638 
03639   SDValue StoreNode = DAG.getNode(ISD::TokenFactor, getCurSDLoc(), MVT::Other,
03640                                   makeArrayRef(Chains.data(), ChainI));
03641   DAG.setRoot(StoreNode);
03642 }
03643 
03644 void SelectionDAGBuilder::visitMaskedStore(const CallInst &I) {
03645   SDLoc sdl = getCurSDLoc();
03646 
03647   Value  *PtrOperand = I.getArgOperand(0);
03648   SDValue Ptr = getValue(PtrOperand);
03649   SDValue Src0 = getValue(I.getArgOperand(1));
03650   SDValue Mask = getValue(I.getArgOperand(3));
03651   EVT VT = Src0.getValueType();
03652   unsigned Alignment = (cast<ConstantInt>(I.getArgOperand(2)))->getZExtValue();
03653   if (!Alignment)
03654     Alignment = DAG.getEVTAlignment(VT);
03655 
03656   AAMDNodes AAInfo;
03657   I.getAAMetadata(AAInfo);
03658 
03659   MachineMemOperand *MMO =
03660     DAG.getMachineFunction().
03661     getMachineMemOperand(MachinePointerInfo(PtrOperand),
03662                           MachineMemOperand::MOStore,  VT.getStoreSize(),
03663                           Alignment, AAInfo);
03664   SDValue StoreNode = DAG.getMaskedStore(getRoot(), sdl, Src0, Ptr, Mask, MMO);
03665   DAG.setRoot(StoreNode);
03666   setValue(&I, StoreNode);
03667 }
03668 
03669 void SelectionDAGBuilder::visitMaskedLoad(const CallInst &I) {
03670   SDLoc sdl = getCurSDLoc();
03671 
03672   Value  *PtrOperand = I.getArgOperand(0);
03673   SDValue Ptr = getValue(PtrOperand);
03674   SDValue Src0 = getValue(I.getArgOperand(1));
03675   SDValue Mask = getValue(I.getArgOperand(3));
03676 
03677   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
03678   EVT VT = TLI.getValueType(I.getType());
03679   unsigned Alignment = (cast<ConstantInt>(I.getArgOperand(2)))->getZExtValue();
03680   if (!Alignment)
03681     Alignment = DAG.getEVTAlignment(VT);
03682 
03683   AAMDNodes AAInfo;
03684   I.getAAMetadata(AAInfo);
03685   const MDNode *Ranges = I.getMetadata(LLVMContext::MD_range);
03686 
03687   SDValue InChain = DAG.getRoot();
03688   if (AA->pointsToConstantMemory(
03689       AliasAnalysis::Location(PtrOperand,
03690                               AA->getTypeStoreSize(I.getType()),
03691                               AAInfo))) {
03692     // Do not serialize (non-volatile) loads of constant memory with anything.
03693     InChain = DAG.getEntryNode();
03694   }
03695 
03696   MachineMemOperand *MMO =
03697     DAG.getMachineFunction().
03698     getMachineMemOperand(MachinePointerInfo(PtrOperand),
03699                           MachineMemOperand::MOLoad,  VT.getStoreSize(),
03700                           Alignment, AAInfo, Ranges);
03701 
03702   SDValue Load = DAG.getMaskedLoad(VT, sdl, InChain, Ptr, Mask, Src0, MMO);
03703   SDValue OutChain = Load.getValue(1);
03704   DAG.setRoot(OutChain);
03705   setValue(&I, Load);
03706 }
03707 
03708 void SelectionDAGBuilder::visitAtomicCmpXchg(const AtomicCmpXchgInst &I) {
03709   SDLoc dl = getCurSDLoc();
03710   AtomicOrdering SuccessOrder = I.getSuccessOrdering();
03711   AtomicOrdering FailureOrder = I.getFailureOrdering();
03712   SynchronizationScope Scope = I.getSynchScope();
03713 
03714   SDValue InChain = getRoot();
03715 
03716   MVT MemVT = getValue(I.getCompareOperand()).getSimpleValueType();
03717   SDVTList VTs = DAG.getVTList(MemVT, MVT::i1, MVT::Other);
03718   SDValue L = DAG.getAtomicCmpSwap(
03719       ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS, dl, MemVT, VTs, InChain,
03720       getValue(I.getPointerOperand()), getValue(I.getCompareOperand()),
03721       getValue(I.getNewValOperand()), MachinePointerInfo(I.getPointerOperand()),
03722       /*Alignment=*/ 0, SuccessOrder, FailureOrder, Scope);
03723 
03724   SDValue OutChain = L.getValue(2);
03725 
03726   setValue(&I, L);
03727   DAG.setRoot(OutChain);
03728 }
03729 
03730 void SelectionDAGBuilder::visitAtomicRMW(const AtomicRMWInst &I) {
03731   SDLoc dl = getCurSDLoc();
03732   ISD::NodeType NT;
03733   switch (I.getOperation()) {
03734   default: llvm_unreachable("Unknown atomicrmw operation");
03735   case AtomicRMWInst::Xchg: NT = ISD::ATOMIC_SWAP; break;
03736   case AtomicRMWInst::Add:  NT = ISD::ATOMIC_LOAD_ADD; break;
03737   case AtomicRMWInst::Sub:  NT = ISD::ATOMIC_LOAD_SUB; break;
03738   case AtomicRMWInst::And:  NT = ISD::ATOMIC_LOAD_AND; break;
03739   case AtomicRMWInst::Nand: NT = ISD::ATOMIC_LOAD_NAND; break;
03740   case AtomicRMWInst::Or:   NT = ISD::ATOMIC_LOAD_OR; break;
03741   case AtomicRMWInst::Xor:  NT = ISD::ATOMIC_LOAD_XOR; break;
03742   case AtomicRMWInst::Max:  NT = ISD::ATOMIC_LOAD_MAX; break;
03743   case AtomicRMWInst::Min:  NT = ISD::ATOMIC_LOAD_MIN; break;
03744   case AtomicRMWInst::UMax: NT = ISD::ATOMIC_LOAD_UMAX; break;
03745   case AtomicRMWInst::UMin: NT = ISD::ATOMIC_LOAD_UMIN; break;
03746   }
03747   AtomicOrdering Order = I.getOrdering();
03748   SynchronizationScope Scope = I.getSynchScope();
03749 
03750   SDValue InChain = getRoot();
03751 
03752   SDValue L =
03753     DAG.getAtomic(NT, dl,
03754                   getValue(I.getValOperand()).getSimpleValueType(),
03755                   InChain,
03756                   getValue(I.getPointerOperand()),
03757                   getValue(I.getValOperand()),
03758                   I.getPointerOperand(),
03759                   /* Alignment=*/ 0, Order, Scope);
03760 
03761   SDValue OutChain = L.getValue(1);
03762 
03763   setValue(&I, L);
03764   DAG.setRoot(OutChain);
03765 }
03766 
03767 void SelectionDAGBuilder::visitFence(const FenceInst &I) {
03768   SDLoc dl = getCurSDLoc();
03769   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
03770   SDValue Ops[3];
03771   Ops[0] = getRoot();
03772   Ops[1] = DAG.getConstant(I.getOrdering(), TLI.getPointerTy());
03773   Ops[2] = DAG.getConstant(I.getSynchScope(), TLI.getPointerTy());
03774   DAG.setRoot(DAG.getNode(ISD::ATOMIC_FENCE, dl, MVT::Other, Ops));
03775 }
03776 
03777 void SelectionDAGBuilder::visitAtomicLoad(const LoadInst &I) {
03778   SDLoc dl = getCurSDLoc();
03779   AtomicOrdering Order = I.getOrdering();
03780   SynchronizationScope Scope = I.getSynchScope();
03781 
03782   SDValue InChain = getRoot();
03783 
03784   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
03785   EVT VT = TLI.getValueType(I.getType());
03786 
03787   if (I.getAlignment() < VT.getSizeInBits() / 8)
03788     report_fatal_error("Cannot generate unaligned atomic load");
03789 
03790   MachineMemOperand *MMO =
03791       DAG.getMachineFunction().
03792       getMachineMemOperand(MachinePointerInfo(I.getPointerOperand()),
03793                            MachineMemOperand::MOVolatile |
03794                            MachineMemOperand::MOLoad,
03795                            VT.getStoreSize(),
03796                            I.getAlignment() ? I.getAlignment() :
03797                                               DAG.getEVTAlignment(VT));
03798 
03799   InChain = TLI.prepareVolatileOrAtomicLoad(InChain, dl, DAG);
03800   SDValue L =
03801       DAG.getAtomic(ISD::ATOMIC_LOAD, dl, VT, VT, InChain,
03802                     getValue(I.getPointerOperand()), MMO,
03803                     Order, Scope);
03804 
03805   SDValue OutChain = L.getValue(1);
03806 
03807   setValue(&I, L);
03808   DAG.setRoot(OutChain);
03809 }
03810 
03811 void SelectionDAGBuilder::visitAtomicStore(const StoreInst &I) {
03812   SDLoc dl = getCurSDLoc();
03813 
03814   AtomicOrdering Order = I.getOrdering();
03815   SynchronizationScope Scope = I.getSynchScope();
03816 
03817   SDValue InChain = getRoot();
03818 
03819   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
03820   EVT VT = TLI.getValueType(I.getValueOperand()->getType());
03821 
03822   if (I.getAlignment() < VT.getSizeInBits() / 8)
03823     report_fatal_error("Cannot generate unaligned atomic store");
03824 
03825   SDValue OutChain =
03826     DAG.getAtomic(ISD::ATOMIC_STORE, dl, VT,
03827                   InChain,
03828                   getValue(I.getPointerOperand()),
03829                   getValue(I.getValueOperand()),
03830                   I.getPointerOperand(), I.getAlignment(),
03831                   Order, Scope);
03832 
03833   DAG.setRoot(OutChain);
03834 }
03835 
03836 /// visitTargetIntrinsic - Lower a call of a target intrinsic to an INTRINSIC
03837 /// node.
03838 void SelectionDAGBuilder::visitTargetIntrinsic(const CallInst &I,
03839                                                unsigned Intrinsic) {
03840   bool HasChain = !I.doesNotAccessMemory();
03841   bool OnlyLoad = HasChain && I.onlyReadsMemory();
03842 
03843   // Build the operand list.
03844   SmallVector<SDValue, 8> Ops;
03845   if (HasChain) {  // If this intrinsic has side-effects, chainify it.
03846     if (OnlyLoad) {
03847       // We don't need to serialize loads against other loads.
03848       Ops.push_back(DAG.getRoot());
03849     } else {
03850       Ops.push_back(getRoot());
03851     }
03852   }
03853 
03854   // Info is set by getTgtMemInstrinsic
03855   TargetLowering::IntrinsicInfo Info;
03856   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
03857   bool IsTgtIntrinsic = TLI.getTgtMemIntrinsic(Info, I, Intrinsic);
03858 
03859   // Add the intrinsic ID as an integer operand if it's not a target intrinsic.
03860   if (!IsTgtIntrinsic || Info.opc == ISD::INTRINSIC_VOID ||
03861       Info.opc == ISD::INTRINSIC_W_CHAIN)
03862     Ops.push_back(DAG.getTargetConstant(Intrinsic, TLI.getPointerTy()));
03863 
03864   // Add all operands of the call to the operand list.
03865   for (unsigned i = 0, e = I.getNumArgOperands(); i != e; ++i) {
03866     SDValue Op = getValue(I.getArgOperand(i));
03867     Ops.push_back(Op);
03868   }
03869 
03870   SmallVector<EVT, 4> ValueVTs;
03871   ComputeValueVTs(TLI, I.getType(), ValueVTs);
03872 
03873   if (HasChain)
03874     ValueVTs.push_back(MVT::Other);
03875 
03876   SDVTList VTs = DAG.getVTList(ValueVTs);
03877 
03878   // Create the node.
03879   SDValue Result;
03880   if (IsTgtIntrinsic) {
03881     // This is target intrinsic that touches memory
03882     Result = DAG.getMemIntrinsicNode(Info.opc, getCurSDLoc(),
03883                                      VTs, Ops, Info.memVT,
03884                                    MachinePointerInfo(Info.ptrVal, Info.offset),
03885                                      Info.align, Info.vol,
03886                                      Info.readMem, Info.writeMem, Info.size);
03887   } else if (!HasChain) {
03888     Result = DAG.getNode(ISD::INTRINSIC_WO_CHAIN, getCurSDLoc(), VTs, Ops);
03889   } else if (!I.getType()->isVoidTy()) {
03890     Result = DAG.getNode(ISD::INTRINSIC_W_CHAIN, getCurSDLoc(), VTs, Ops);
03891   } else {
03892     Result = DAG.getNode(ISD::INTRINSIC_VOID, getCurSDLoc(), VTs, Ops);
03893   }
03894 
03895   if (HasChain) {
03896     SDValue Chain = Result.getValue(Result.getNode()->getNumValues()-1);
03897     if (OnlyLoad)
03898       PendingLoads.push_back(Chain);
03899     else
03900       DAG.setRoot(Chain);
03901   }
03902 
03903   if (!I.getType()->isVoidTy()) {
03904     if (VectorType *PTy = dyn_cast<VectorType>(I.getType())) {
03905       EVT VT = TLI.getValueType(PTy);
03906       Result = DAG.getNode(ISD::BITCAST, getCurSDLoc(), VT, Result);
03907     }
03908 
03909     setValue(&I, Result);
03910   }
03911 }
03912 
03913 /// GetSignificand - Get the significand and build it into a floating-point
03914 /// number with exponent of 1:
03915 ///
03916 ///   Op = (Op & 0x007fffff) | 0x3f800000;
03917 ///
03918 /// where Op is the hexadecimal representation of floating point value.
03919 static SDValue
03920 GetSignificand(SelectionDAG &DAG, SDValue Op, SDLoc dl) {
03921   SDValue t1 = DAG.getNode(ISD::AND, dl, MVT::i32, Op,
03922                            DAG.getConstant(0x007fffff, MVT::i32));
03923   SDValue t2 = DAG.getNode(ISD::OR, dl, MVT::i32, t1,
03924                            DAG.getConstant(0x3f800000, MVT::i32));
03925   return DAG.getNode(ISD::BITCAST, dl, MVT::f32, t2);
03926 }
03927 
03928 /// GetExponent - Get the exponent:
03929 ///
03930 ///   (float)(int)(((Op & 0x7f800000) >> 23) - 127);
03931 ///
03932 /// where Op is the hexadecimal representation of floating point value.
03933 static SDValue
03934 GetExponent(SelectionDAG &DAG, SDValue Op, const TargetLowering &TLI,
03935             SDLoc dl) {
03936   SDValue t0 = DAG.getNode(ISD::AND, dl, MVT::i32, Op,
03937                            DAG.getConstant(0x7f800000, MVT::i32));
03938   SDValue t1 = DAG.getNode(ISD::SRL, dl, MVT::i32, t0,
03939                            DAG.getConstant(23, TLI.getPointerTy()));
03940   SDValue t2 = DAG.getNode(ISD::SUB, dl, MVT::i32, t1,
03941                            DAG.getConstant(127, MVT::i32));
03942   return DAG.getNode(ISD::SINT_TO_FP, dl, MVT::f32, t2);
03943 }
03944 
03945 /// getF32Constant - Get 32-bit floating point constant.
03946 static SDValue
03947 getF32Constant(SelectionDAG &DAG, unsigned Flt) {
03948   return DAG.getConstantFP(APFloat(APFloat::IEEEsingle, APInt(32, Flt)),
03949                            MVT::f32);
03950 }
03951 
03952 /// expandExp - Lower an exp intrinsic. Handles the special sequences for
03953 /// limited-precision mode.
03954 static SDValue expandExp(SDLoc dl, SDValue Op, SelectionDAG &DAG,
03955                          const TargetLowering &TLI) {
03956   if (Op.getValueType() == MVT::f32 &&
03957       LimitFloatPrecision > 0 && LimitFloatPrecision <= 18) {
03958 
03959     // Put the exponent in the right bit position for later addition to the
03960     // final result:
03961     //
03962     //   #define LOG2OFe 1.4426950f
03963     //   IntegerPartOfX = ((int32_t)(X * LOG2OFe));
03964     SDValue t0 = DAG.getNode(ISD::FMUL, dl, MVT::f32, Op,
03965                              getF32Constant(DAG, 0x3fb8aa3b));
03966     SDValue IntegerPartOfX = DAG.getNode(ISD::FP_TO_SINT, dl, MVT::i32, t0);
03967 
03968     //   FractionalPartOfX = (X * LOG2OFe) - (float)IntegerPartOfX;
03969     SDValue t1 = DAG.getNode(ISD::SINT_TO_FP, dl, MVT::f32, IntegerPartOfX);
03970     SDValue X = DAG.getNode(ISD::FSUB, dl, MVT::f32, t0, t1);
03971 
03972     //   IntegerPartOfX <<= 23;
03973     IntegerPartOfX = DAG.getNode(ISD::SHL, dl, MVT::i32, IntegerPartOfX,
03974                                  DAG.getConstant(23, TLI.getPointerTy()));
03975 
03976     SDValue TwoToFracPartOfX;
03977     if (LimitFloatPrecision <= 6) {
03978       // For floating-point precision of 6:
03979       //
03980       //   TwoToFractionalPartOfX =
03981       //     0.997535578f +
03982       //       (0.735607626f + 0.252464424f * x) * x;
03983       //
03984       // error 0.0144103317, which is 6 bits
03985       SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X,
03986                                getF32Constant(DAG, 0x3e814304));
03987       SDValue t3 = DAG.getNode(ISD::FADD, dl, MVT::f32, t2,
03988                                getF32Constant(DAG, 0x3f3c50c8));
03989       SDValue t4 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t3, X);
03990       TwoToFracPartOfX = DAG.getNode(ISD::FADD, dl, MVT::f32, t4,
03991                                      getF32Constant(DAG, 0x3f7f5e7e));
03992     } else if (LimitFloatPrecision <= 12) {
03993       // For floating-point precision of 12:
03994       //
03995       //   TwoToFractionalPartOfX =
03996       //     0.999892986f +
03997       //       (0.696457318f +
03998       //         (0.224338339f + 0.792043434e-1f * x) * x) * x;
03999       //
04000       // 0.000107046256 error, which is 13 to 14 bits
04001       SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X,
04002                                getF32Constant(DAG, 0x3da235e3));
04003       SDValue t3 = DAG.getNode(ISD::FADD, dl, MVT::f32, t2,
04004                                getF32Constant(DAG, 0x3e65b8f3));
04005       SDValue t4 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t3, X);
04006       SDValue t5 = DAG.getNode(ISD::FADD, dl, MVT::f32, t4,
04007                                getF32Constant(DAG, 0x3f324b07));
04008       SDValue t6 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t5, X);
04009       TwoToFracPartOfX = DAG.getNode(ISD::FADD, dl, MVT::f32, t6,
04010                                      getF32Constant(DAG, 0x3f7ff8fd));
04011     } else { // LimitFloatPrecision <= 18
04012       // For floating-point precision of 18:
04013       //
04014       //   TwoToFractionalPartOfX =
04015       //     0.999999982f +
04016       //       (0.693148872f +
04017       //         (0.240227044f +
04018       //           (0.554906021e-1f +
04019       //             (0.961591928e-2f +
04020       //               (0.136028312e-2f + 0.157059148e-3f *x)*x)*x)*x)*x)*x;
04021       //
04022       // error 2.47208000*10^(-7), which is better than 18 bits
04023       SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X,
04024                                getF32Constant(DAG, 0x3924b03e));
04025       SDValue t3 = DAG.getNode(ISD::FADD, dl, MVT::f32, t2,
04026                                getF32Constant(DAG, 0x3ab24b87));
04027       SDValue t4 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t3, X);
04028       SDValue t5 = DAG.getNode(ISD::FADD, dl, MVT::f32, t4,
04029                                getF32Constant(DAG, 0x3c1d8c17));
04030       SDValue t6 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t5, X);
04031       SDValue t7 = DAG.getNode(ISD::FADD, dl, MVT::f32, t6,
04032                                getF32Constant(DAG, 0x3d634a1d));
04033       SDValue t8 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t7, X);
04034       SDValue t9 = DAG.getNode(ISD::FADD, dl, MVT::f32, t8,
04035                                getF32Constant(DAG, 0x3e75fe14));
04036       SDValue t10 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t9, X);
04037       SDValue t11 = DAG.getNode(ISD::FADD, dl, MVT::f32, t10,
04038                                 getF32Constant(DAG, 0x3f317234));
04039       SDValue t12 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t11, X);
04040       TwoToFracPartOfX = DAG.getNode(ISD::FADD, dl, MVT::f32, t12,
04041                                      getF32Constant(DAG, 0x3f800000));
04042     }
04043 
04044     // Add the exponent into the result in integer domain.
04045     SDValue t13 = DAG.getNode(ISD::BITCAST, dl, MVT::i32, TwoToFracPartOfX);
04046     return DAG.getNode(ISD::BITCAST, dl, MVT::f32,
04047                        DAG.getNode(ISD::ADD, dl, MVT::i32,
04048                                    t13, IntegerPartOfX));
04049   }
04050 
04051   // No special expansion.
04052   return DAG.getNode(ISD::FEXP, dl, Op.getValueType(), Op);
04053 }
04054 
04055 /// expandLog - Lower a log intrinsic. Handles the special sequences for
04056 /// limited-precision mode.
04057 static SDValue expandLog(SDLoc dl, SDValue Op, SelectionDAG &DAG,
04058                          const TargetLowering &TLI) {
04059   if (Op.getValueType() == MVT::f32 &&
04060       LimitFloatPrecision > 0 && LimitFloatPrecision <= 18) {
04061     SDValue Op1 = DAG.getNode(ISD::BITCAST, dl, MVT::i32, Op);
04062 
04063     // Scale the exponent by log(2) [0.69314718f].
04064     SDValue Exp = GetExponent(DAG, Op1, TLI, dl);
04065     SDValue LogOfExponent = DAG.getNode(ISD::FMUL, dl, MVT::f32, Exp,
04066                                         getF32Constant(DAG, 0x3f317218));
04067 
04068     // Get the significand and build it into a floating-point number with
04069     // exponent of 1.
04070     SDValue X = GetSignificand(DAG, Op1, dl);
04071 
04072     SDValue LogOfMantissa;
04073     if (LimitFloatPrecision <= 6) {
04074       // For floating-point precision of 6:
04075       //
04076       //   LogofMantissa =
04077       //     -1.1609546f +
04078       //       (1.4034025f - 0.23903021f * x) * x;
04079       //
04080       // error 0.0034276066, which is better than 8 bits
04081       SDValue t0 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X,
04082                                getF32Constant(DAG, 0xbe74c456));
04083       SDValue t1 = DAG.getNode(ISD::FADD, dl, MVT::f32, t0,
04084                                getF32Constant(DAG, 0x3fb3a2b1));
04085       SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t1, X);
04086       LogOfMantissa = DAG.getNode(ISD::FSUB, dl, MVT::f32, t2,
04087                                   getF32Constant(DAG, 0x3f949a29));
04088     } else if (LimitFloatPrecision <= 12) {
04089       // For floating-point precision of 12:
04090       //
04091       //   LogOfMantissa =
04092       //     -1.7417939f +
04093       //       (2.8212026f +
04094       //         (-1.4699568f +
04095       //           (0.44717955f - 0.56570851e-1f * x) * x) * x) * x;
04096       //
04097       // error 0.000061011436, which is 14 bits
04098       SDValue t0 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X,
04099                                getF32Constant(DAG, 0xbd67b6d6));
04100       SDValue t1 = DAG.getNode(ISD::FADD, dl, MVT::f32, t0,
04101                                getF32Constant(DAG, 0x3ee4f4b8));
04102       SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t1, X);
04103       SDValue t3 = DAG.getNode(ISD::FSUB, dl, MVT::f32, t2,
04104                                getF32Constant(DAG, 0x3fbc278b));
04105       SDValue t4 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t3, X);
04106       SDValue t5 = DAG.getNode(ISD::FADD, dl, MVT::f32, t4,
04107                                getF32Constant(DAG, 0x40348e95));
04108       SDValue t6 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t5, X);
04109       LogOfMantissa = DAG.getNode(ISD::FSUB, dl, MVT::f32, t6,
04110                                   getF32Constant(DAG, 0x3fdef31a));
04111     } else { // LimitFloatPrecision <= 18
04112       // For floating-point precision of 18:
04113       //
04114       //   LogOfMantissa =
04115       //     -2.1072184f +
04116       //       (4.2372794f +
04117       //         (-3.7029485f +
04118       //           (2.2781945f +
04119       //             (-0.87823314f +
04120       //               (0.19073739f - 0.17809712e-1f * x) * x) * x) * x) * x)*x;
04121       //
04122       // error 0.0000023660568, which is better than 18 bits
04123       SDValue t0 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X,
04124                                getF32Constant(DAG, 0xbc91e5ac));
04125       SDValue t1 = DAG.getNode(ISD::FADD, dl, MVT::f32, t0,
04126                                getF32Constant(DAG, 0x3e4350aa));
04127       SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t1, X);
04128       SDValue t3 = DAG.getNode(ISD::FSUB, dl, MVT::f32, t2,
04129                                getF32Constant(DAG, 0x3f60d3e3));
04130       SDValue t4 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t3, X);
04131       SDValue t5 = DAG.getNode(ISD::FADD, dl, MVT::f32, t4,
04132                                getF32Constant(DAG, 0x4011cdf0));
04133       SDValue t6 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t5, X);
04134       SDValue t7 = DAG.getNode(ISD::FSUB, dl, MVT::f32, t6,
04135                                getF32Constant(DAG, 0x406cfd1c));
04136       SDValue t8 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t7, X);
04137       SDValue t9 = DAG.getNode(ISD::FADD, dl, MVT::f32, t8,
04138                                getF32Constant(DAG, 0x408797cb));
04139       SDValue t10 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t9, X);
04140       LogOfMantissa = DAG.getNode(ISD::FSUB, dl, MVT::f32, t10,
04141                                   getF32Constant(DAG, 0x4006dcab));
04142     }
04143 
04144     return DAG.getNode(ISD::FADD, dl, MVT::f32, LogOfExponent, LogOfMantissa);
04145   }
04146 
04147   // No special expansion.
04148   return DAG.getNode(ISD::FLOG, dl, Op.getValueType(), Op);
04149 }
04150 
04151 /// expandLog2 - Lower a log2 intrinsic. Handles the special sequences for
04152 /// limited-precision mode.
04153 static SDValue expandLog2(SDLoc dl, SDValue Op, SelectionDAG &DAG,
04154                           const TargetLowering &TLI) {
04155   if (Op.getValueType() == MVT::f32 &&
04156       LimitFloatPrecision > 0 && LimitFloatPrecision <= 18) {
04157     SDValue Op1 = DAG.getNode(ISD::BITCAST, dl, MVT::i32, Op);
04158 
04159     // Get the exponent.
04160     SDValue LogOfExponent = GetExponent(DAG, Op1, TLI, dl);
04161 
04162     // Get the significand and build it into a floating-point number with
04163     // exponent of 1.
04164     SDValue X = GetSignificand(DAG, Op1, dl);
04165 
04166     // Different possible minimax approximations of significand in
04167     // floating-point for various degrees of accuracy over [1,2].
04168     SDValue Log2ofMantissa;
04169     if (LimitFloatPrecision <= 6) {
04170       // For floating-point precision of 6:
04171       //
04172       //   Log2ofMantissa = -1.6749035f + (2.0246817f - .34484768f * x) * x;
04173       //
04174       // error 0.0049451742, which is more than 7 bits
04175       SDValue t0 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X,
04176                                getF32Constant(DAG, 0xbeb08fe0));
04177       SDValue t1 = DAG.getNode(ISD::FADD, dl, MVT::f32, t0,
04178                                getF32Constant(DAG, 0x40019463));
04179       SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t1, X);
04180       Log2ofMantissa = DAG.getNode(ISD::FSUB, dl, MVT::f32, t2,
04181                                    getF32Constant(DAG, 0x3fd6633d));
04182     } else if (LimitFloatPrecision <= 12) {
04183       // For floating-point precision of 12:
04184       //
04185       //   Log2ofMantissa =
04186       //     -2.51285454f +
04187       //       (4.07009056f +
04188       //         (-2.12067489f +
04189       //           (.645142248f - 0.816157886e-1f * x) * x) * x) * x;
04190       //
04191       // error 0.0000876136000, which is better than 13 bits
04192       SDValue t0 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X,
04193                                getF32Constant(DAG, 0xbda7262e));
04194       SDValue t1 = DAG.getNode(ISD::FADD, dl, MVT::f32, t0,
04195                                getF32Constant(DAG, 0x3f25280b));
04196       SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t1, X);
04197       SDValue t3 = DAG.getNode(ISD::FSUB, dl, MVT::f32, t2,
04198                                getF32Constant(DAG, 0x4007b923));
04199       SDValue t4 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t3, X);
04200       SDValue t5 = DAG.getNode(ISD::FADD, dl, MVT::f32, t4,
04201                                getF32Constant(DAG, 0x40823e2f));
04202       SDValue t6 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t5, X);
04203       Log2ofMantissa = DAG.getNode(ISD::FSUB, dl, MVT::f32, t6,
04204                                    getF32Constant(DAG, 0x4020d29c));
04205     } else { // LimitFloatPrecision <= 18
04206       // For floating-point precision of 18:
04207       //
04208       //   Log2ofMantissa =
04209       //     -3.0400495f +
04210       //       (6.1129976f +
04211       //         (-5.3420409f +
04212       //           (3.2865683f +
04213       //             (-1.2669343f +
04214       //               (0.27515199f -
04215       //                 0.25691327e-1f * x) * x) * x) * x) * x) * x;
04216       //
04217       // error 0.0000018516, which is better than 18 bits
04218       SDValue t0 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X,
04219                                getF32Constant(DAG, 0xbcd2769e));
04220       SDValue t1 = DAG.getNode(ISD::FADD, dl, MVT::f32, t0,
04221                                getF32Constant(DAG, 0x3e8ce0b9));
04222       SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t1, X);
04223       SDValue t3 = DAG.getNode(ISD::FSUB, dl, MVT::f32, t2,
04224                                getF32Constant(DAG, 0x3fa22ae7));
04225       SDValue t4 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t3, X);
04226       SDValue t5 = DAG.getNode(ISD::FADD, dl, MVT::f32, t4,
04227                                getF32Constant(DAG, 0x40525723));
04228       SDValue t6 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t5, X);
04229       SDValue t7 = DAG.getNode(ISD::FSUB, dl, MVT::f32, t6,
04230                                getF32Constant(DAG, 0x40aaf200));
04231       SDValue t8 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t7, X);
04232       SDValue t9 = DAG.getNode(ISD::FADD, dl, MVT::f32, t8,
04233                                getF32Constant(DAG, 0x40c39dad));
04234       SDValue t10 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t9, X);
04235       Log2ofMantissa = DAG.getNode(ISD::FSUB, dl, MVT::f32, t10,
04236                                    getF32Constant(DAG, 0x4042902c));
04237     }
04238 
04239     return DAG.getNode(ISD::FADD, dl, MVT::f32, LogOfExponent, Log2ofMantissa);
04240   }
04241 
04242   // No special expansion.
04243   return DAG.getNode(ISD::FLOG2, dl, Op.getValueType(), Op);
04244 }
04245 
04246 /// expandLog10 - Lower a log10 intrinsic. Handles the special sequences for
04247 /// limited-precision mode.
04248 static SDValue expandLog10(SDLoc dl, SDValue Op, SelectionDAG &DAG,
04249                            const TargetLowering &TLI) {
04250   if (Op.getValueType() == MVT::f32 &&
04251       LimitFloatPrecision > 0 && LimitFloatPrecision <= 18) {
04252     SDValue Op1 = DAG.getNode(ISD::BITCAST, dl, MVT::i32, Op);
04253 
04254     // Scale the exponent by log10(2) [0.30102999f].
04255     SDValue Exp = GetExponent(DAG, Op1, TLI, dl);
04256     SDValue LogOfExponent = DAG.getNode(ISD::FMUL, dl, MVT::f32, Exp,
04257                                         getF32Constant(DAG, 0x3e9a209a));
04258 
04259     // Get the significand and build it into a floating-point number with
04260     // exponent of 1.
04261     SDValue X = GetSignificand(DAG, Op1, dl);
04262 
04263     SDValue Log10ofMantissa;
04264     if (LimitFloatPrecision <= 6) {
04265       // For floating-point precision of 6:
04266       //
04267       //   Log10ofMantissa =
04268       //     -0.50419619f +
04269       //       (0.60948995f - 0.10380950f * x) * x;
04270       //
04271       // error 0.0014886165, which is 6 bits
04272       SDValue t0 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X,
04273                                getF32Constant(DAG, 0xbdd49a13));
04274       SDValue t1 = DAG.getNode(ISD::FADD, dl, MVT::f32, t0,
04275                                getF32Constant(DAG, 0x3f1c0789));
04276       SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t1, X);
04277       Log10ofMantissa = DAG.getNode(ISD::FSUB, dl, MVT::f32, t2,
04278                                     getF32Constant(DAG, 0x3f011300));
04279     } else if (LimitFloatPrecision <= 12) {
04280       // For floating-point precision of 12:
04281       //
04282       //   Log10ofMantissa =
04283       //     -0.64831180f +
04284       //       (0.91751397f +
04285       //         (-0.31664806f + 0.47637168e-1f * x) * x) * x;
04286       //
04287       // error 0.00019228036, which is better than 12 bits
04288       SDValue t0 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X,
04289                                getF32Constant(DAG, 0x3d431f31));
04290       SDValue t1 = DAG.getNode(ISD::FSUB, dl, MVT::f32, t0,
04291                                getF32Constant(DAG, 0x3ea21fb2));
04292       SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t1, X);
04293       SDValue t3 = DAG.getNode(ISD::FADD, dl, MVT::f32, t2,
04294                                getF32Constant(DAG, 0x3f6ae232));
04295       SDValue t4 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t3, X);
04296       Log10ofMantissa = DAG.getNode(ISD::FSUB, dl, MVT::f32, t4,
04297                                     getF32Constant(DAG, 0x3f25f7c3));
04298     } else { // LimitFloatPrecision <= 18
04299       // For floating-point precision of 18:
04300       //
04301       //   Log10ofMantissa =
04302       //     -0.84299375f +
04303       //       (1.5327582f +
04304       //         (-1.0688956f +
04305       //           (0.49102474f +
04306       //             (-0.12539807f + 0.13508273e-1f * x) * x) * x) * x) * x;
04307       //
04308       // error 0.0000037995730, which is better than 18 bits
04309       SDValue t0 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X,
04310                                getF32Constant(DAG, 0x3c5d51ce));
04311       SDValue t1 = DAG.getNode(ISD::FSUB, dl, MVT::f32, t0,
04312                                getF32Constant(DAG, 0x3e00685a));
04313       SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t1, X);
04314       SDValue t3 = DAG.getNode(ISD::FADD, dl, MVT::f32, t2,
04315                                getF32Constant(DAG, 0x3efb6798));
04316       SDValue t4 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t3, X);
04317       SDValue t5 = DAG.getNode(ISD::FSUB, dl, MVT::f32, t4,
04318                                getF32Constant(DAG, 0x3f88d192));
04319       SDValue t6 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t5, X);
04320       SDValue t7 = DAG.getNode(ISD::FADD, dl, MVT::f32, t6,
04321                                getF32Constant(DAG, 0x3fc4316c));
04322       SDValue t8 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t7, X);
04323       Log10ofMantissa = DAG.getNode(ISD::FSUB, dl, MVT::f32, t8,
04324                                     getF32Constant(DAG, 0x3f57ce70));
04325     }
04326 
04327     return DAG.getNode(ISD::FADD, dl, MVT::f32, LogOfExponent, Log10ofMantissa);
04328   }
04329 
04330   // No special expansion.
04331   return DAG.getNode(ISD::FLOG10, dl, Op.getValueType(), Op);
04332 }
04333 
04334 /// expandExp2 - Lower an exp2 intrinsic. Handles the special sequences for
04335 /// limited-precision mode.
04336 static SDValue expandExp2(SDLoc dl, SDValue Op, SelectionDAG &DAG,
04337                           const TargetLowering &TLI) {
04338   if (Op.getValueType() == MVT::f32 &&
04339       LimitFloatPrecision > 0 && LimitFloatPrecision <= 18) {
04340     SDValue IntegerPartOfX = DAG.getNode(ISD::FP_TO_SINT, dl, MVT::i32, Op);
04341 
04342     //   FractionalPartOfX = x - (float)IntegerPartOfX;
04343     SDValue t1 = DAG.getNode(ISD::SINT_TO_FP, dl, MVT::f32, IntegerPartOfX);
04344     SDValue X = DAG.getNode(ISD::FSUB, dl, MVT::f32, Op, t1);
04345 
04346     //   IntegerPartOfX <<= 23;
04347     IntegerPartOfX = DAG.getNode(ISD::SHL, dl, MVT::i32, IntegerPartOfX,
04348                                  DAG.getConstant(23, TLI.getPointerTy()));
04349 
04350     SDValue TwoToFractionalPartOfX;
04351     if (LimitFloatPrecision <= 6) {
04352       // For floating-point precision of 6:
04353       //
04354       //   TwoToFractionalPartOfX =
04355       //     0.997535578f +
04356       //       (0.735607626f + 0.252464424f * x) * x;
04357       //
04358       // error 0.0144103317, which is 6 bits
04359       SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X,
04360                                getF32Constant(DAG, 0x3e814304));
04361       SDValue t3 = DAG.getNode(ISD::FADD, dl, MVT::f32, t2,
04362                                getF32Constant(DAG, 0x3f3c50c8));
04363       SDValue t4 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t3, X);
04364       TwoToFractionalPartOfX = DAG.getNode(ISD::FADD, dl, MVT::f32, t4,
04365                                            getF32Constant(DAG, 0x3f7f5e7e));
04366     } else if (LimitFloatPrecision <= 12) {
04367       // For floating-point precision of 12:
04368       //
04369       //   TwoToFractionalPartOfX =
04370       //     0.999892986f +
04371       //       (0.696457318f +
04372       //         (0.224338339f + 0.792043434e-1f * x) * x) * x;
04373       //
04374       // error 0.000107046256, which is 13 to 14 bits
04375       SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X,
04376                                getF32Constant(DAG, 0x3da235e3));
04377       SDValue t3 = DAG.getNode(ISD::FADD, dl, MVT::f32, t2,
04378                                getF32Constant(DAG, 0x3e65b8f3));
04379       SDValue t4 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t3, X);
04380       SDValue t5 = DAG.getNode(ISD::FADD, dl, MVT::f32, t4,
04381                                getF32Constant(DAG, 0x3f324b07));
04382       SDValue t6 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t5, X);
04383       TwoToFractionalPartOfX = DAG.getNode(ISD::FADD, dl, MVT::f32, t6,
04384                                            getF32Constant(DAG, 0x3f7ff8fd));
04385     } else { // LimitFloatPrecision <= 18
04386       // For floating-point precision of 18:
04387       //
04388       //   TwoToFractionalPartOfX =
04389       //     0.999999982f +
04390       //       (0.693148872f +
04391       //         (0.240227044f +
04392       //           (0.554906021e-1f +
04393       //             (0.961591928e-2f +
04394       //               (0.136028312e-2f + 0.157059148e-3f *x)*x)*x)*x)*x)*x;
04395       // error 2.47208000*10^(-7), which is better than 18 bits
04396       SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X,
04397                                getF32Constant(DAG, 0x3924b03e));
04398       SDValue t3 = DAG.getNode(ISD::FADD, dl, MVT::f32, t2,
04399                                getF32Constant(DAG, 0x3ab24b87));
04400       SDValue t4 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t3, X);
04401       SDValue t5 = DAG.getNode(ISD::FADD, dl, MVT::f32, t4,
04402                                getF32Constant(DAG, 0x3c1d8c17));
04403       SDValue t6 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t5, X);
04404       SDValue t7 = DAG.getNode(ISD::FADD, dl, MVT::f32, t6,
04405                                getF32Constant(DAG, 0x3d634a1d));
04406       SDValue t8 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t7, X);
04407       SDValue t9 = DAG.getNode(ISD::FADD, dl, MVT::f32, t8,
04408                                getF32Constant(DAG, 0x3e75fe14));
04409       SDValue t10 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t9, X);
04410       SDValue t11 = DAG.getNode(ISD::FADD, dl, MVT::f32, t10,
04411                                 getF32Constant(DAG, 0x3f317234));
04412       SDValue t12 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t11, X);
04413       TwoToFractionalPartOfX = DAG.getNode(ISD::FADD, dl, MVT::f32, t12,
04414                                            getF32Constant(DAG, 0x3f800000));
04415     }
04416 
04417     // Add the exponent into the result in integer domain.
04418     SDValue t13 = DAG.getNode(ISD::BITCAST, dl, MVT::i32,
04419                               TwoToFractionalPartOfX);
04420     return DAG.getNode(ISD::BITCAST, dl, MVT::f32,
04421                        DAG.getNode(ISD::ADD, dl, MVT::i32,
04422                                    t13, IntegerPartOfX));
04423   }
04424 
04425   // No special expansion.
04426   return DAG.getNode(ISD::FEXP2, dl, Op.getValueType(), Op);
04427 }
04428 
04429 /// visitPow - Lower a pow intrinsic. Handles the special sequences for
04430 /// limited-precision mode with x == 10.0f.
04431 static SDValue expandPow(SDLoc dl, SDValue LHS, SDValue RHS,
04432                          SelectionDAG &DAG, const TargetLowering &TLI) {
04433   bool IsExp10 = false;
04434   if (LHS.getValueType() == MVT::f32 && RHS.getValueType() == MVT::f32 &&
04435       LimitFloatPrecision > 0 && LimitFloatPrecision <= 18) {
04436     if (ConstantFPSDNode *LHSC = dyn_cast<ConstantFPSDNode>(LHS)) {
04437       APFloat Ten(10.0f);
04438       IsExp10 = LHSC->isExactlyValue(Ten);
04439     }
04440   }
04441 
04442   if (IsExp10) {
04443     // Put the exponent in the right bit position for later addition to the
04444     // final result:
04445     //
04446     //   #define LOG2OF10 3.3219281f
04447     //   IntegerPartOfX = (int32_t)(x * LOG2OF10);
04448     SDValue t0 = DAG.getNode(ISD::FMUL, dl, MVT::f32, RHS,
04449                              getF32Constant(DAG, 0x40549a78));
04450     SDValue IntegerPartOfX = DAG.getNode(ISD::FP_TO_SINT, dl, MVT::i32, t0);
04451 
04452     //   FractionalPartOfX = x - (float)IntegerPartOfX;
04453     SDValue t1 = DAG.getNode(ISD::SINT_TO_FP, dl, MVT::f32, IntegerPartOfX);
04454     SDValue X = DAG.getNode(ISD::FSUB, dl, MVT::f32, t0, t1);
04455 
04456     //   IntegerPartOfX <<= 23;
04457     IntegerPartOfX = DAG.getNode(ISD::SHL, dl, MVT::i32, IntegerPartOfX,
04458                                  DAG.getConstant(23, TLI.getPointerTy()));
04459 
04460     SDValue TwoToFractionalPartOfX;
04461     if (LimitFloatPrecision <= 6) {
04462       // For floating-point precision of 6:
04463       //
04464       //   twoToFractionalPartOfX =
04465       //     0.997535578f +
04466       //       (0.735607626f + 0.252464424f * x) * x;
04467       //
04468       // error 0.0144103317, which is 6 bits
04469       SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X,
04470                                getF32Constant(DAG, 0x3e814304));
04471       SDValue t3 = DAG.getNode(ISD::FADD, dl, MVT::f32, t2,
04472                                getF32Constant(DAG, 0x3f3c50c8));
04473       SDValue t4 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t3, X);
04474       TwoToFractionalPartOfX = DAG.getNode(ISD::FADD, dl, MVT::f32, t4,
04475                                            getF32Constant(DAG, 0x3f7f5e7e));
04476     } else if (LimitFloatPrecision <= 12) {
04477       // For floating-point precision of 12:
04478       //
04479       //   TwoToFractionalPartOfX =
04480       //     0.999892986f +
04481       //       (0.696457318f +
04482       //         (0.224338339f + 0.792043434e-1f * x) * x) * x;
04483       //
04484       // error 0.000107046256, which is 13 to 14 bits
04485       SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X,
04486                                getF32Constant(DAG, 0x3da235e3));
04487       SDValue t3 = DAG.getNode(ISD::FADD, dl, MVT::f32, t2,
04488                                getF32Constant(DAG, 0x3e65b8f3));
04489       SDValue t4 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t3, X);
04490       SDValue t5 = DAG.getNode(ISD::FADD, dl, MVT::f32, t4,
04491                                getF32Constant(DAG, 0x3f324b07));
04492       SDValue t6 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t5, X);
04493       TwoToFractionalPartOfX = DAG.getNode(ISD::FADD, dl, MVT::f32, t6,
04494                                            getF32Constant(DAG, 0x3f7ff8fd));
04495     } else { // LimitFloatPrecision <= 18
04496       // For floating-point precision of 18:
04497       //
04498       //   TwoToFractionalPartOfX =
04499       //     0.999999982f +
04500       //       (0.693148872f +
04501       //         (0.240227044f +
04502       //           (0.554906021e-1f +
04503       //             (0.961591928e-2f +
04504       //               (0.136028312e-2f + 0.157059148e-3f *x)*x)*x)*x)*x)*x;
04505       // error 2.47208000*10^(-7), which is better than 18 bits
04506       SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X,
04507                                getF32Constant(DAG, 0x3924b03e));
04508       SDValue t3 = DAG.getNode(ISD::FADD, dl, MVT::f32, t2,
04509                                getF32Constant(DAG, 0x3ab24b87));
04510       SDValue t4 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t3, X);
04511       SDValue t5 = DAG.getNode(ISD::FADD, dl, MVT::f32, t4,
04512                                getF32Constant(DAG, 0x3c1d8c17));
04513       SDValue t6 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t5, X);
04514       SDValue t7 = DAG.getNode(ISD::FADD, dl, MVT::f32, t6,
04515                                getF32Constant(DAG, 0x3d634a1d));
04516       SDValue t8 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t7, X);
04517       SDValue t9 = DAG.getNode(ISD::FADD, dl, MVT::f32, t8,
04518                                getF32Constant(DAG, 0x3e75fe14));
04519       SDValue t10 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t9, X);
04520       SDValue t11 = DAG.getNode(ISD::FADD, dl, MVT::f32, t10,
04521                                 getF32Constant(DAG, 0x3f317234));
04522       SDValue t12 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t11, X);
04523       TwoToFractionalPartOfX = DAG.getNode(ISD::FADD, dl, MVT::f32, t12,
04524                                            getF32Constant(DAG, 0x3f800000));
04525     }
04526 
04527     SDValue t13 = DAG.getNode(ISD::BITCAST, dl,MVT::i32,TwoToFractionalPartOfX);
04528     return DAG.getNode(ISD::BITCAST, dl, MVT::f32,
04529                        DAG.getNode(ISD::ADD, dl, MVT::i32,
04530                                    t13, IntegerPartOfX));
04531   }
04532 
04533   // No special expansion.
04534   return DAG.getNode(ISD::FPOW, dl, LHS.getValueType(), LHS, RHS);
04535 }
04536 
04537 
04538 /// ExpandPowI - Expand a llvm.powi intrinsic.
04539 static SDValue ExpandPowI(SDLoc DL, SDValue LHS, SDValue RHS,
04540                           SelectionDAG &DAG) {
04541   // If RHS is a constant, we can expand this out to a multiplication tree,
04542   // otherwise we end up lowering to a call to __powidf2 (for example).  When
04543   // optimizing for size, we only want to do this if the expansion would produce
04544   // a small number of multiplies, otherwise we do the full expansion.
04545   if (ConstantSDNode *RHSC = dyn_cast<ConstantSDNode>(RHS)) {
04546     // Get the exponent as a positive value.
04547     unsigned Val = RHSC->getSExtValue();
04548     if ((int)Val < 0) Val = -Val;
04549 
04550     // powi(x, 0) -> 1.0
04551     if (Val == 0)
04552       return DAG.getConstantFP(1.0, LHS.getValueType());
04553 
04554     const Function *F = DAG.getMachineFunction().getFunction();
04555     if (!F->getAttributes().hasAttribute(AttributeSet::FunctionIndex,
04556                                          Attribute::OptimizeForSize) ||
04557         // If optimizing for size, don't insert too many multiplies.  This
04558         // inserts up to 5 multiplies.
04559         CountPopulation_32(Val)+Log2_32(Val) < 7) {
04560       // We use the simple binary decomposition method to generate the multiply
04561       // sequence.  There are more optimal ways to do this (for example,
04562       // powi(x,15) generates one more multiply than it should), but this has
04563       // the benefit of being both really simple and much better than a libcall.
04564       SDValue Res;  // Logically starts equal to 1.0
04565       SDValue CurSquare = LHS;
04566       while (Val) {
04567         if (Val & 1) {
04568           if (Res.getNode())
04569             Res = DAG.getNode(ISD::FMUL, DL,Res.getValueType(), Res, CurSquare);
04570           else
04571             Res = CurSquare;  // 1.0*CurSquare.
04572         }
04573 
04574         CurSquare = DAG.getNode(ISD::FMUL, DL, CurSquare.getValueType(),
04575                                 CurSquare, CurSquare);
04576         Val >>= 1;
04577       }
04578 
04579       // If the original was negative, invert the result, producing 1/(x*x*x).
04580       if (RHSC->getSExtValue() < 0)
04581         Res = DAG.getNode(ISD::FDIV, DL, LHS.getValueType(),
04582                           DAG.getConstantFP(1.0, LHS.getValueType()), Res);
04583       return Res;
04584     }
04585   }
04586 
04587   // Otherwise, expand to a libcall.
04588   return DAG.getNode(ISD::FPOWI, DL, LHS.getValueType(), LHS, RHS);
04589 }
04590 
04591 // getTruncatedArgReg - Find underlying register used for an truncated
04592 // argument.
04593 static unsigned getTruncatedArgReg(const SDValue &N) {
04594   if (N.getOpcode() != ISD::TRUNCATE)
04595     return 0;
04596 
04597   const SDValue &Ext = N.getOperand(0);
04598   if (Ext.getOpcode() == ISD::AssertZext ||
04599       Ext.getOpcode() == ISD::AssertSext) {
04600     const SDValue &CFR = Ext.getOperand(0);
04601     if (CFR.getOpcode() == ISD::CopyFromReg)
04602       return cast<RegisterSDNode>(CFR.getOperand(1))->getReg();
04603     if (CFR.getOpcode() == ISD::TRUNCATE)
04604       return getTruncatedArgReg(CFR);
04605   }
04606   return 0;
04607 }
04608 
04609 /// EmitFuncArgumentDbgValue - If the DbgValueInst is a dbg_value of a function
04610 /// argument, create the corresponding DBG_VALUE machine instruction for it now.
04611 /// At the end of instruction selection, they will be inserted to the entry BB.
04612 bool SelectionDAGBuilder::EmitFuncArgumentDbgValue(const Value *V,
04613                                                    MDNode *Variable,
04614                                                    MDNode *Expr, int64_t Offset,
04615                                                    bool IsIndirect,
04616                                                    const SDValue &N) {
04617   const Argument *Arg = dyn_cast<Argument>(V);
04618   if (!Arg)
04619     return false;
04620 
04621   MachineFunction &MF = DAG.getMachineFunction();
04622   const TargetInstrInfo *TII = DAG.getSubtarget().getInstrInfo();
04623 
04624   // Ignore inlined function arguments here.
04625   DIVariable DV(Variable);
04626   if (DV.isInlinedFnArgument(MF.getFunction()))
04627     return false;
04628 
04629   Optional<MachineOperand> Op;
04630   // Some arguments' frame index is recorded during argument lowering.
04631   if (int FI = FuncInfo.getArgumentFrameIndex(Arg))
04632     Op = MachineOperand::CreateFI(FI);
04633 
04634   if (!Op && N.getNode()) {
04635     unsigned Reg;
04636     if (N.getOpcode() == ISD::CopyFromReg)
04637       Reg = cast<RegisterSDNode>(N.getOperand(1))->getReg();
04638     else
04639       Reg = getTruncatedArgReg(N);
04640     if (Reg && TargetRegisterInfo::isVirtualRegister(Reg)) {
04641       MachineRegisterInfo &RegInfo = MF.getRegInfo();
04642       unsigned PR = RegInfo.getLiveInPhysReg(Reg);
04643       if (PR)
04644         Reg = PR;
04645     }
04646     if (Reg)
04647       Op = MachineOperand::CreateReg(Reg, false);
04648   }
04649 
04650   if (!Op) {
04651     // Check if ValueMap has reg number.
04652     DenseMap<const Value *, unsigned>::iterator VMI = FuncInfo.ValueMap.find(V);
04653     if (VMI != FuncInfo.ValueMap.end())
04654       Op = MachineOperand::CreateReg(VMI->second, false);
04655   }
04656 
04657   if (!Op && N.getNode())
04658     // Check if frame index is available.
04659     if (LoadSDNode *LNode = dyn_cast<LoadSDNode>(N.getNode()))
04660       if (FrameIndexSDNode *FINode =
04661           dyn_cast<FrameIndexSDNode>(LNode->getBasePtr().getNode()))
04662         Op = MachineOperand::CreateFI(FINode->getIndex());
04663 
04664   if (!Op)
04665     return false;
04666 
04667   if (Op->isReg())
04668     FuncInfo.ArgDbgValues.push_back(
04669         BuildMI(MF, getCurDebugLoc(), TII->get(TargetOpcode::DBG_VALUE),
04670                 IsIndirect, Op->getReg(), Offset, Variable, Expr));
04671   else
04672     FuncInfo.ArgDbgValues.push_back(
04673         BuildMI(MF, getCurDebugLoc(), TII->get(TargetOpcode::DBG_VALUE))
04674             .addOperand(*Op)
04675             .addImm(Offset)
04676             .addMetadata(Variable)
04677             .addMetadata(Expr));
04678 
04679   return true;
04680 }
04681 
04682 // VisualStudio defines setjmp as _setjmp
04683 #if defined(_MSC_VER) && defined(setjmp) && \
04684                          !defined(setjmp_undefined_for_msvc)
04685 #  pragma push_macro("setjmp")
04686 #  undef setjmp
04687 #  define setjmp_undefined_for_msvc
04688 #endif
04689 
04690 /// visitIntrinsicCall - Lower the call to the specified intrinsic function.  If
04691 /// we want to emit this as a call to a named external function, return the name
04692 /// otherwise lower it and return null.
04693 const char *
04694 SelectionDAGBuilder::visitIntrinsicCall(const CallInst &I, unsigned Intrinsic) {
04695   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
04696   SDLoc sdl = getCurSDLoc();
04697   DebugLoc dl = getCurDebugLoc();
04698   SDValue Res;
04699 
04700   switch (Intrinsic) {
04701   default:
04702     // By default, turn this into a target intrinsic node.
04703     visitTargetIntrinsic(I, Intrinsic);
04704     return nullptr;
04705   case Intrinsic::vastart:  visitVAStart(I); return nullptr;
04706   case Intrinsic::vaend:    visitVAEnd(I); return nullptr;
04707   case Intrinsic::vacopy:   visitVACopy(I); return nullptr;
04708   case Intrinsic::returnaddress:
04709     setValue(&I, DAG.getNode(ISD::RETURNADDR, sdl, TLI.getPointerTy(),
04710                              getValue(I.getArgOperand(0))));
04711     return nullptr;
04712   case Intrinsic::frameaddress:
04713     setValue(&I, DAG.getNode(ISD::FRAMEADDR, sdl, TLI.getPointerTy(),
04714                              getValue(I.getArgOperand(0))));
04715     return nullptr;
04716   case Intrinsic::read_register: {
04717     Value *Reg = I.getArgOperand(0);
04718     SDValue RegName =
04719         DAG.getMDNode(cast<MDNode>(cast<MetadataAsValue>(Reg)->getMetadata()));
04720     EVT VT = TLI.getValueType(I.getType());
04721     setValue(&I, DAG.getNode(ISD::READ_REGISTER, sdl, VT, RegName));
04722     return nullptr;
04723   }
04724   case Intrinsic::write_register: {
04725     Value *Reg = I.getArgOperand(0);
04726     Value *RegValue = I.getArgOperand(1);
04727     SDValue Chain = getValue(RegValue).getOperand(0);
04728     SDValue RegName =
04729         DAG.getMDNode(cast<MDNode>(cast<MetadataAsValue>(Reg)->getMetadata()));
04730     DAG.setRoot(DAG.getNode(ISD::WRITE_REGISTER, sdl, MVT::Other, Chain,
04731                             RegName, getValue(RegValue)));
04732     return nullptr;
04733   }
04734   case Intrinsic::setjmp:
04735     return &"_setjmp"[!TLI.usesUnderscoreSetJmp()];
04736   case Intrinsic::longjmp:
04737     return &"_longjmp"[!TLI.usesUnderscoreLongJmp()];
04738   case Intrinsic::memcpy: {
04739     // Assert for address < 256 since we support only user defined address
04740     // spaces.
04741     assert(cast<PointerType>(I.getArgOperand(0)->getType())->getAddressSpace()
04742            < 256 &&
04743            cast<PointerType>(I.getArgOperand(1)->getType())->getAddressSpace()
04744            < 256 &&
04745            "Unknown address space");
04746     SDValue Op1 = getValue(I.getArgOperand(0));
04747     SDValue Op2 = getValue(I.getArgOperand(1));
04748     SDValue Op3 = getValue(I.getArgOperand(2));
04749     unsigned Align = cast<ConstantInt>(I.getArgOperand(3))->getZExtValue();
04750     if (!Align)
04751       Align = 1; // @llvm.memcpy defines 0 and 1 to both mean no alignment.
04752     bool isVol = cast<ConstantInt>(I.getArgOperand(4))->getZExtValue();
04753     DAG.setRoot(DAG.getMemcpy(getRoot(), sdl, Op1, Op2, Op3, Align, isVol, false,
04754                               MachinePointerInfo(I.getArgOperand(0)),
04755                               MachinePointerInfo(I.getArgOperand(1))));
04756     return nullptr;
04757   }
04758   case Intrinsic::memset: {
04759     // Assert for address < 256 since we support only user defined address
04760     // spaces.
04761     assert(cast<PointerType>(I.getArgOperand(0)->getType())->getAddressSpace()
04762            < 256 &&
04763            "Unknown address space");
04764     SDValue Op1 = getValue(I.getArgOperand(0));
04765     SDValue Op2 = getValue(I.getArgOperand(1));
04766     SDValue Op3 = getValue(I.getArgOperand(2));
04767     unsigned Align = cast<ConstantInt>(I.getArgOperand(3))->getZExtValue();
04768     if (!Align)
04769       Align = 1; // @llvm.memset defines 0 and 1 to both mean no alignment.
04770     bool isVol = cast<ConstantInt>(I.getArgOperand(4))->getZExtValue();
04771     DAG.setRoot(DAG.getMemset(getRoot(), sdl, Op1, Op2, Op3, Align, isVol,
04772                               MachinePointerInfo(I.getArgOperand(0))));
04773     return nullptr;
04774   }
04775   case Intrinsic::memmove: {
04776     // Assert for address < 256 since we support only user defined address
04777     // spaces.
04778     assert(cast<PointerType>(I.getArgOperand(0)->getType())->getAddressSpace()
04779            < 256 &&
04780            cast<PointerType>(I.getArgOperand(1)->getType())->getAddressSpace()
04781            < 256 &&
04782            "Unknown address space");
04783     SDValue Op1 = getValue(I.getArgOperand(0));
04784     SDValue Op2 = getValue(I.getArgOperand(1));
04785     SDValue Op3 = getValue(I.getArgOperand(2));
04786     unsigned Align = cast<ConstantInt>(I.getArgOperand(3))->getZExtValue();
04787     if (!Align)
04788       Align = 1; // @llvm.memmove defines 0 and 1 to both mean no alignment.
04789     bool isVol = cast<ConstantInt>(I.getArgOperand(4))->getZExtValue();
04790     DAG.setRoot(DAG.getMemmove(getRoot(), sdl, Op1, Op2, Op3, Align, isVol,
04791                                MachinePointerInfo(I.getArgOperand(0)),
04792                                MachinePointerInfo(I.getArgOperand(1))));
04793     return nullptr;
04794   }
04795   case Intrinsic::dbg_declare: {
04796     const DbgDeclareInst &DI = cast<DbgDeclareInst>(I);
04797     MDNode *Variable = DI.getVariable();
04798     MDNode *Expression = DI.getExpression();
04799     const Value *Address = DI.getAddress();
04800     DIVariable DIVar(Variable);
04801     assert((!DIVar || DIVar.isVariable()) &&
04802       "Variable in DbgDeclareInst should be either null or a DIVariable.");
04803     if (!Address || !DIVar) {
04804       DEBUG(dbgs() << "Dropping debug info for " << DI << "\n");
04805       return nullptr;
04806     }
04807 
04808     // Check if address has undef value.
04809     if (isa<UndefValue>(Address) ||
04810         (Address->use_empty() && !isa<Argument>(Address))) {
04811       DEBUG(dbgs() << "Dropping debug info for " << DI << "\n");
04812       return nullptr;
04813     }
04814 
04815     SDValue &N = NodeMap[Address];
04816     if (!N.getNode() && isa<Argument>(Address))
04817       // Check unused arguments map.
04818       N = UnusedArgNodeMap[Address];
04819     SDDbgValue *SDV;
04820     if (N.getNode()) {
04821       if (const BitCastInst *BCI = dyn_cast<BitCastInst>(Address))
04822         Address = BCI->getOperand(0);
04823       // Parameters are handled specially.
04824       bool isParameter =
04825         (DIVariable(Variable).getTag() == dwarf::DW_TAG_arg_variable ||
04826          isa<Argument>(Address));
04827 
04828       const AllocaInst *AI = dyn_cast<AllocaInst>(Address);
04829 
04830       if (isParameter && !AI) {
04831         FrameIndexSDNode *FINode = dyn_cast<FrameIndexSDNode>(N.getNode());
04832         if (FINode)
04833           // Byval parameter.  We have a frame index at this point.
04834           SDV = DAG.getFrameIndexDbgValue(
04835               Variable, Expression, FINode->getIndex(), 0, dl, SDNodeOrder);
04836         else {
04837           // Address is an argument, so try to emit its dbg value using
04838           // virtual register info from the FuncInfo.ValueMap.
04839           EmitFuncArgumentDbgValue(Address, Variable, Expression, 0, false, N);
04840           return nullptr;
04841         }
04842       } else if (AI)
04843         SDV = DAG.getDbgValue(Variable, Expression, N.getNode(), N.getResNo(),
04844                               true, 0, dl, SDNodeOrder);
04845       else {
04846         // Can't do anything with other non-AI cases yet.
04847         DEBUG(dbgs() << "Dropping debug info for " << DI << "\n");
04848         DEBUG(dbgs() << "non-AllocaInst issue for Address: \n\t");
04849         DEBUG(Address->dump());
04850         return nullptr;
04851       }
04852       DAG.AddDbgValue(SDV, N.getNode(), isParameter);
04853     } else {
04854       // If Address is an argument then try to emit its dbg value using
04855       // virtual register info from the FuncInfo.ValueMap.
04856       if (!EmitFuncArgumentDbgValue(Address, Variable, Expression, 0, false,
04857                                     N)) {
04858         // If variable is pinned by a alloca in dominating bb then
04859         // use StaticAllocaMap.
04860         if (const AllocaInst *AI = dyn_cast<AllocaInst>(Address)) {
04861           if (AI->getParent() != DI.getParent()) {
04862             DenseMap<const AllocaInst*, int>::iterator SI =
04863               FuncInfo.StaticAllocaMap.find(AI);
04864             if (SI != FuncInfo.StaticAllocaMap.end()) {
04865               SDV = DAG.getFrameIndexDbgValue(Variable, Expression, SI->second,
04866                                               0, dl, SDNodeOrder);
04867               DAG.AddDbgValue(SDV, nullptr, false);
04868               return nullptr;
04869             }
04870           }
04871         }
04872         DEBUG(dbgs() << "Dropping debug info for " << DI << "\n");
04873       }
04874     }
04875     return nullptr;
04876   }
04877   case Intrinsic::dbg_value: {
04878     const DbgValueInst &DI = cast<DbgValueInst>(I);
04879     DIVariable DIVar(DI.getVariable());
04880     assert((!DIVar || DIVar.isVariable()) &&
04881       "Variable in DbgValueInst should be either null or a DIVariable.");
04882     if (!DIVar)
04883       return nullptr;
04884 
04885     MDNode *Variable = DI.getVariable();
04886     MDNode *Expression = DI.getExpression();
04887     uint64_t Offset = DI.getOffset();
04888     const Value *V = DI.getValue();
04889     if (!V)
04890       return nullptr;
04891 
04892     SDDbgValue *SDV;
04893     if (isa<ConstantInt>(V) || isa<ConstantFP>(V) || isa<UndefValue>(V)) {
04894       SDV = DAG.getConstantDbgValue(Variable, Expression, V, Offset, dl,
04895                                     SDNodeOrder);
04896       DAG.AddDbgValue(SDV, nullptr, false);
04897     } else {
04898       // Do not use getValue() in here; we don't want to generate code at
04899       // this point if it hasn't been done yet.
04900       SDValue N = NodeMap[V];
04901       if (!N.getNode() && isa<Argument>(V))
04902         // Check unused arguments map.
04903         N = UnusedArgNodeMap[V];
04904       if (N.getNode()) {
04905         // A dbg.value for an alloca is always indirect.
04906         bool IsIndirect = isa<AllocaInst>(V) || Offset != 0;
04907         if (!EmitFuncArgumentDbgValue(V, Variable, Expression, Offset,
04908                                       IsIndirect, N)) {
04909           SDV = DAG.getDbgValue(Variable, Expression, N.getNode(), N.getResNo(),
04910                                 IsIndirect, Offset, dl, SDNodeOrder);
04911           DAG.AddDbgValue(SDV, N.getNode(), false);
04912         }
04913       } else if (!V->use_empty() ) {
04914         // Do not call getValue(V) yet, as we don't want to generate code.
04915         // Remember it for later.
04916         DanglingDebugInfo DDI(&DI, dl, SDNodeOrder);
04917         DanglingDebugInfoMap[V] = DDI;
04918       } else {
04919         // We may expand this to cover more cases.  One case where we have no
04920         // data available is an unreferenced parameter.
04921         DEBUG(dbgs() << "Dropping debug info for " << DI << "\n");
04922       }
04923     }
04924 
04925     // Build a debug info table entry.
04926     if (const BitCastInst *BCI = dyn_cast<BitCastInst>(V))
04927       V = BCI->getOperand(0);
04928     const AllocaInst *AI = dyn_cast<AllocaInst>(V);
04929     // Don't handle byval struct arguments or VLAs, for example.
04930     if (!AI) {
04931       DEBUG(dbgs() << "Dropping debug location info for:\n  " << DI << "\n");
04932       DEBUG(dbgs() << "  Last seen at:\n    " << *V << "\n");
04933       return nullptr;
04934     }
04935     DenseMap<const AllocaInst*, int>::iterator SI =
04936       FuncInfo.StaticAllocaMap.find(AI);
04937     if (SI == FuncInfo.StaticAllocaMap.end())
04938       return nullptr; // VLAs.
04939     return nullptr;
04940   }
04941 
04942   case Intrinsic::eh_typeid_for: {
04943     // Find the type id for the given typeinfo.
04944     GlobalValue *GV = ExtractTypeInfo(I.getArgOperand(0));
04945     unsigned TypeID = DAG.getMachineFunction().getMMI().getTypeIDFor(GV);
04946     Res = DAG.getConstant(TypeID, MVT::i32);
04947     setValue(&I, Res);
04948     return nullptr;
04949   }
04950 
04951   case Intrinsic::eh_return_i32:
04952   case Intrinsic::eh_return_i64:
04953     DAG.getMachineFunction().getMMI().setCallsEHReturn(true);
04954     DAG.setRoot(DAG.getNode(ISD::EH_RETURN, sdl,
04955                             MVT::Other,
04956                             getControlRoot(),
04957                             getValue(I.getArgOperand(0)),
04958                             getValue(I.getArgOperand(1))));
04959     return nullptr;
04960   case Intrinsic::eh_unwind_init:
04961     DAG.getMachineFunction().getMMI().setCallsUnwindInit(true);
04962     return nullptr;
04963   case Intrinsic::eh_dwarf_cfa: {
04964     SDValue CfaArg = DAG.getSExtOrTrunc(getValue(I.getArgOperand(0)), sdl,
04965                                         TLI.getPointerTy());
04966     SDValue Offset = DAG.getNode(ISD::ADD, sdl,
04967                                  CfaArg.getValueType(),
04968                                  DAG.getNode(ISD::FRAME_TO_ARGS_OFFSET, sdl,
04969                                              CfaArg.getValueType()),
04970                                  CfaArg);
04971     SDValue FA = DAG.getNode(ISD::FRAMEADDR, sdl, TLI.getPointerTy(),
04972                              DAG.getConstant(0, TLI.getPointerTy()));
04973     setValue(&I, DAG.getNode(ISD::ADD, sdl, FA.getValueType(),
04974                              FA, Offset));
04975     return nullptr;
04976   }
04977   case Intrinsic::eh_sjlj_callsite: {
04978     MachineModuleInfo &MMI = DAG.getMachineFunction().getMMI();
04979     ConstantInt *CI = dyn_cast<ConstantInt>(I.getArgOperand(0));
04980     assert(CI && "Non-constant call site value in eh.sjlj.callsite!");
04981     assert(MMI.getCurrentCallSite() == 0 && "Overlapping call sites!");
04982 
04983     MMI.setCurrentCallSite(CI->getZExtValue());
04984     return nullptr;
04985   }
04986   case Intrinsic::eh_sjlj_functioncontext: {
04987     // Get and store the index of the function context.
04988     MachineFrameInfo *MFI = DAG.getMachineFunction().getFrameInfo();
04989     AllocaInst *FnCtx =
04990       cast<AllocaInst>(I.getArgOperand(0)->stripPointerCasts());
04991     int FI = FuncInfo.StaticAllocaMap[FnCtx];
04992     MFI->setFunctionContextIndex(FI);
04993     return nullptr;
04994   }
04995   case Intrinsic::eh_sjlj_setjmp: {
04996     SDValue Ops[2];
04997     Ops[0] = getRoot();
04998     Ops[1] = getValue(I.getArgOperand(0));
04999     SDValue Op = DAG.getNode(ISD::EH_SJLJ_SETJMP, sdl,
05000                              DAG.getVTList(MVT::i32, MVT::Other), Ops);
05001     setValue(&I, Op.getValue(0));
05002     DAG.setRoot(Op.getValue(1));
05003     return nullptr;
05004   }
05005   case Intrinsic::eh_sjlj_longjmp: {
05006     DAG.setRoot(DAG.getNode(ISD::EH_SJLJ_LONGJMP, sdl, MVT::Other,
05007                             getRoot(), getValue(I.getArgOperand(0))));
05008     return nullptr;
05009   }
05010 
05011   case Intrinsic::masked_load:
05012     visitMaskedLoad(I);
05013     return nullptr;
05014   case Intrinsic::masked_store:
05015     visitMaskedStore(I);
05016     return nullptr;
05017   case Intrinsic::x86_mmx_pslli_w:
05018   case Intrinsic::x86_mmx_pslli_d:
05019   case Intrinsic::x86_mmx_pslli_q:
05020   case Intrinsic::x86_mmx_psrli_w:
05021   case Intrinsic::x86_mmx_psrli_d:
05022   case Intrinsic::x86_mmx_psrli_q:
05023   case Intrinsic::x86_mmx_psrai_w:
05024   case Intrinsic::x86_mmx_psrai_d: {
05025     SDValue ShAmt = getValue(I.getArgOperand(1));
05026     if (isa<ConstantSDNode>(ShAmt)) {
05027       visitTargetIntrinsic(I, Intrinsic);
05028       return nullptr;
05029     }
05030     unsigned NewIntrinsic = 0;
05031     EVT ShAmtVT = MVT::v2i32;
05032     switch (Intrinsic) {
05033     case Intrinsic::x86_mmx_pslli_w:
05034       NewIntrinsic = Intrinsic::x86_mmx_psll_w;
05035       break;
05036     case Intrinsic::x86_mmx_pslli_d:
05037       NewIntrinsic = Intrinsic::x86_mmx_psll_d;
05038       break;
05039     case Intrinsic::x86_mmx_pslli_q:
05040       NewIntrinsic = Intrinsic::x86_mmx_psll_q;
05041       break;
05042     case Intrinsic::x86_mmx_psrli_w:
05043       NewIntrinsic = Intrinsic::x86_mmx_psrl_w;
05044       break;
05045     case Intrinsic::x86_mmx_psrli_d:
05046       NewIntrinsic = Intrinsic::x86_mmx_psrl_d;
05047       break;
05048     case Intrinsic::x86_mmx_psrli_q:
05049       NewIntrinsic = Intrinsic::x86_mmx_psrl_q;
05050       break;
05051     case Intrinsic::x86_mmx_psrai_w:
05052       NewIntrinsic = Intrinsic::x86_mmx_psra_w;
05053       break;
05054     case Intrinsic::x86_mmx_psrai_d:
05055       NewIntrinsic = Intrinsic::x86_mmx_psra_d;
05056       break;
05057     default: llvm_unreachable("Impossible intrinsic");  // Can't reach here.
05058     }
05059 
05060     // The vector shift intrinsics with scalars uses 32b shift amounts but
05061     // the sse2/mmx shift instructions reads 64 bits. Set the upper 32 bits
05062     // to be zero.
05063     // We must do this early because v2i32 is not a legal type.
05064     SDValue ShOps[2];
05065     ShOps[0] = ShAmt;
05066     ShOps[1] = DAG.getConstant(0, MVT::i32);
05067     ShAmt =  DAG.getNode(ISD::BUILD_VECTOR, sdl, ShAmtVT, ShOps);
05068     EVT DestVT = TLI.getValueType(I.getType());
05069     ShAmt = DAG.getNode(ISD::BITCAST, sdl, DestVT, ShAmt);
05070     Res = DAG.getNode(ISD::INTRINSIC_WO_CHAIN, sdl, DestVT,
05071                        DAG.getConstant(NewIntrinsic, MVT::i32),
05072                        getValue(I.getArgOperand(0)), ShAmt);
05073     setValue(&I, Res);
05074     return nullptr;
05075   }
05076   case Intrinsic::x86_avx_vinsertf128_pd_256:
05077   case Intrinsic::x86_avx_vinsertf128_ps_256:
05078   case Intrinsic::x86_avx_vinsertf128_si_256:
05079   case Intrinsic::x86_avx2_vinserti128: {
05080     EVT DestVT = TLI.getValueType(I.getType());
05081     EVT ElVT = TLI.getValueType(I.getArgOperand(1)->getType());
05082     uint64_t Idx = (cast<ConstantInt>(I.getArgOperand(2))->getZExtValue() & 1) *
05083                    ElVT.getVectorNumElements();
05084     Res =
05085         DAG.getNode(ISD::INSERT_SUBVECTOR, sdl, DestVT,
05086                     getValue(I.getArgOperand(0)), getValue(I.getArgOperand(1)),
05087                     DAG.getConstant(Idx, TLI.getVectorIdxTy()));
05088     setValue(&I, Res);
05089     return nullptr;
05090   }
05091   case Intrinsic::x86_avx_vextractf128_pd_256:
05092   case Intrinsic::x86_avx_vextractf128_ps_256:
05093   case Intrinsic::x86_avx_vextractf128_si_256:
05094   case Intrinsic::x86_avx2_vextracti128: {
05095     EVT DestVT = TLI.getValueType(I.getType());
05096     uint64_t Idx = (cast<ConstantInt>(I.getArgOperand(1))->getZExtValue() & 1) *
05097                    DestVT.getVectorNumElements();
05098     Res = DAG.getNode(ISD::EXTRACT_SUBVECTOR, sdl, DestVT,
05099                       getValue(I.getArgOperand(0)),
05100                       DAG.getConstant(Idx, TLI.getVectorIdxTy()));
05101     setValue(&I, Res);
05102     return nullptr;
05103   }
05104   case Intrinsic::convertff:
05105   case Intrinsic::convertfsi:
05106   case Intrinsic::convertfui:
05107   case Intrinsic::convertsif:
05108   case Intrinsic::convertuif:
05109   case Intrinsic::convertss:
05110   case Intrinsic::convertsu:
05111   case Intrinsic::convertus:
05112   case Intrinsic::convertuu: {
05113     ISD::CvtCode Code = ISD::CVT_INVALID;
05114     switch (Intrinsic) {
05115     default: llvm_unreachable("Impossible intrinsic");  // Can't reach here.
05116     case Intrinsic::convertff:  Code = ISD::CVT_FF; break;
05117     case Intrinsic::convertfsi: Code = ISD::CVT_FS; break;
05118     case Intrinsic::convertfui: Code = ISD::CVT_FU; break;
05119     case Intrinsic::convertsif: Code = ISD::CVT_SF; break;
05120     case Intrinsic::convertuif: Code = ISD::CVT_UF; break;
05121     case Intrinsic::convertss:  Code = ISD::CVT_SS; break;
05122     case Intrinsic::convertsu:  Code = ISD::CVT_SU; break;
05123     case Intrinsic::convertus:  Code = ISD::CVT_US; break;
05124     case Intrinsic::convertuu:  Code = ISD::CVT_UU; break;
05125     }
05126     EVT DestVT = TLI.getValueType(I.getType());
05127     const Value *Op1 = I.getArgOperand(0);
05128     Res = DAG.getConvertRndSat(DestVT, sdl, getValue(Op1),
05129                                DAG.getValueType(DestVT),
05130                                DAG.getValueType(getValue(Op1).getValueType()),
05131                                getValue(I.getArgOperand(1)),
05132                                getValue(I.getArgOperand(2)),
05133                                Code);
05134     setValue(&I, Res);
05135     return nullptr;
05136   }
05137   case Intrinsic::powi:
05138     setValue(&I, ExpandPowI(sdl, getValue(I.getArgOperand(0)),
05139                             getValue(I.getArgOperand(1)), DAG));
05140     return nullptr;
05141   case Intrinsic::log:
05142     setValue(&I, expandLog(sdl, getValue(I.getArgOperand(0)), DAG, TLI));
05143     return nullptr;
05144   case Intrinsic::log2:
05145     setValue(&I, expandLog2(sdl, getValue(I.getArgOperand(0)), DAG, TLI));
05146     return nullptr;
05147   case Intrinsic::log10:
05148     setValue(&I, expandLog10(sdl, getValue(I.getArgOperand(0)), DAG, TLI));
05149     return nullptr;
05150   case Intrinsic::exp:
05151     setValue(&I, expandExp(sdl, getValue(I.getArgOperand(0)), DAG, TLI));
05152     return nullptr;
05153   case Intrinsic::exp2:
05154     setValue(&I, expandExp2(sdl, getValue(I.getArgOperand(0)), DAG, TLI));
05155     return nullptr;
05156   case Intrinsic::pow:
05157     setValue(&I, expandPow(sdl, getValue(I.getArgOperand(0)),
05158                            getValue(I.getArgOperand(1)), DAG, TLI));
05159     return nullptr;
05160   case Intrinsic::sqrt:
05161   case Intrinsic::fabs:
05162   case Intrinsic::sin:
05163   case Intrinsic::cos:
05164   case Intrinsic::floor:
05165   case Intrinsic::ceil:
05166   case Intrinsic::trunc:
05167   case Intrinsic::rint:
05168   case Intrinsic::nearbyint:
05169   case Intrinsic::round: {
05170     unsigned Opcode;
05171     switch (Intrinsic) {
05172     default: llvm_unreachable("Impossible intrinsic");  // Can't reach here.
05173     case Intrinsic::sqrt:      Opcode = ISD::FSQRT;      break;
05174     case Intrinsic::fabs:      Opcode = ISD::FABS;       break;
05175     case Intrinsic::sin:       Opcode = ISD::FSIN;       break;
05176     case Intrinsic::cos:       Opcode = ISD::FCOS;       break;
05177     case Intrinsic::floor:     Opcode = ISD::FFLOOR;     break;
05178     case Intrinsic::ceil:      Opcode = ISD::FCEIL;      break;
05179     case Intrinsic::trunc:     Opcode = ISD::FTRUNC;     break;
05180     case Intrinsic::rint:      Opcode = ISD::FRINT;      break;
05181     case Intrinsic::nearbyint: Opcode = ISD::FNEARBYINT; break;
05182     case Intrinsic::round:     Opcode = ISD::FROUND;     break;
05183     }
05184 
05185     setValue(&I, DAG.getNode(Opcode, sdl,
05186                              getValue(I.getArgOperand(0)).getValueType(),
05187                              getValue(I.getArgOperand(0))));
05188     return nullptr;
05189   }
05190   case Intrinsic::minnum:
05191     setValue(&I, DAG.getNode(ISD::FMINNUM, sdl,
05192                              getValue(I.getArgOperand(0)).getValueType(),
05193                              getValue(I.getArgOperand(0)),
05194                              getValue(I.getArgOperand(1))));
05195     return nullptr;
05196   case Intrinsic::maxnum:
05197     setValue(&I, DAG.getNode(ISD::FMAXNUM, sdl,
05198                              getValue(I.getArgOperand(0)).getValueType(),
05199                              getValue(I.getArgOperand(0)),
05200                              getValue(I.getArgOperand(1))));
05201     return nullptr;
05202   case Intrinsic::copysign:
05203     setValue(&I, DAG.getNode(ISD::FCOPYSIGN, sdl,
05204                              getValue(I.getArgOperand(0)).getValueType(),
05205                              getValue(I.getArgOperand(0)),
05206                              getValue(I.getArgOperand(1))));
05207     return nullptr;
05208   case Intrinsic::fma:
05209     setValue(&I, DAG.getNode(ISD::FMA, sdl,
05210                              getValue(I.getArgOperand(0)).getValueType(),
05211                              getValue(I.getArgOperand(0)),
05212                              getValue(I.getArgOperand(1)),
05213                              getValue(I.getArgOperand(2))));
05214     return nullptr;
05215   case Intrinsic::fmuladd: {
05216     EVT VT = TLI.getValueType(I.getType());
05217     if (TM.Options.AllowFPOpFusion != FPOpFusion::Strict &&
05218         TLI.isFMAFasterThanFMulAndFAdd(VT)) {
05219       setValue(&I, DAG.getNode(ISD::FMA, sdl,
05220                                getValue(I.getArgOperand(0)).getValueType(),
05221                                getValue(I.getArgOperand(0)),
05222                                getValue(I.getArgOperand(1)),
05223                                getValue(I.getArgOperand(2))));
05224     } else {
05225       SDValue Mul = DAG.getNode(ISD::FMUL, sdl,
05226                                 getValue(I.getArgOperand(0)).getValueType(),
05227                                 getValue(I.getArgOperand(0)),
05228                                 getValue(I.getArgOperand(1)));
05229       SDValue Add = DAG.getNode(ISD::FADD, sdl,
05230                                 getValue(I.getArgOperand(0)).getValueType(),
05231                                 Mul,
05232                                 getValue(I.getArgOperand(2)));
05233       setValue(&I, Add);
05234     }
05235     return nullptr;
05236   }
05237   case Intrinsic::convert_to_fp16:
05238     setValue(&I, DAG.getNode(ISD::BITCAST, sdl, MVT::i16,
05239                              DAG.getNode(ISD::FP_ROUND, sdl, MVT::f16,
05240                                          getValue(I.getArgOperand(0)),
05241                                          DAG.getTargetConstant(0, MVT::i32))));
05242     return nullptr;
05243   case Intrinsic::convert_from_fp16:
05244     setValue(&I,
05245              DAG.getNode(ISD::FP_EXTEND, sdl, TLI.getValueType(I.getType()),
05246                          DAG.getNode(ISD::BITCAST, sdl, MVT::f16,
05247                                      getValue(I.getArgOperand(0)))));
05248     return nullptr;
05249   case Intrinsic::pcmarker: {
05250     SDValue Tmp = getValue(I.getArgOperand(0));
05251     DAG.setRoot(DAG.getNode(ISD::PCMARKER, sdl, MVT::Other, getRoot(), Tmp));
05252     return nullptr;
05253   }
05254   case Intrinsic::readcyclecounter: {
05255     SDValue Op = getRoot();
05256     Res = DAG.getNode(ISD::READCYCLECOUNTER, sdl,
05257                       DAG.getVTList(MVT::i64, MVT::Other), Op);
05258     setValue(&I, Res);
05259     DAG.setRoot(Res.getValue(1));
05260     return nullptr;
05261   }
05262   case Intrinsic::bswap:
05263     setValue(&I, DAG.getNode(ISD::BSWAP, sdl,
05264                              getValue(I.getArgOperand(0)).getValueType(),
05265                              getValue(I.getArgOperand(0))));
05266     return nullptr;
05267   case Intrinsic::cttz: {
05268     SDValue Arg = getValue(I.getArgOperand(0));
05269     ConstantInt *CI = cast<ConstantInt>(I.getArgOperand(1));
05270     EVT Ty = Arg.getValueType();
05271     setValue(&I, DAG.getNode(CI->isZero() ? ISD::CTTZ : ISD::CTTZ_ZERO_UNDEF,
05272                              sdl, Ty, Arg));
05273     return nullptr;
05274   }
05275   case Intrinsic::ctlz: {
05276     SDValue Arg = getValue(I.getArgOperand(0));
05277     ConstantInt *CI = cast<ConstantInt>(I.getArgOperand(1));
05278     EVT Ty = Arg.getValueType();
05279     setValue(&I, DAG.getNode(CI->isZero() ? ISD::CTLZ : ISD::CTLZ_ZERO_UNDEF,
05280                              sdl, Ty, Arg));
05281     return nullptr;
05282   }
05283   case Intrinsic::ctpop: {
05284     SDValue Arg = getValue(I.getArgOperand(0));
05285     EVT Ty = Arg.getValueType();
05286     setValue(&I, DAG.getNode(ISD::CTPOP, sdl, Ty, Arg));
05287     return nullptr;
05288   }
05289   case Intrinsic::stacksave: {
05290     SDValue Op = getRoot();
05291     Res = DAG.getNode(ISD::STACKSAVE, sdl,
05292                       DAG.getVTList(TLI.getPointerTy(), MVT::Other), Op);
05293     setValue(&I, Res);
05294     DAG.setRoot(Res.getValue(1));
05295     return nullptr;
05296   }
05297   case Intrinsic::stackrestore: {
05298     Res = getValue(I.getArgOperand(0));
05299     DAG.setRoot(DAG.getNode(ISD::STACKRESTORE, sdl, MVT::Other, getRoot(), Res));
05300     return nullptr;
05301   }
05302   case Intrinsic::stackprotector: {
05303     // Emit code into the DAG to store the stack guard onto the stack.
05304     MachineFunction &MF = DAG.getMachineFunction();
05305     MachineFrameInfo *MFI = MF.getFrameInfo();
05306     EVT PtrTy = TLI.getPointerTy();
05307     SDValue Src, Chain = getRoot();
05308     const Value *Ptr = cast<LoadInst>(I.getArgOperand(0))->getPointerOperand();
05309     const GlobalVariable *GV = dyn_cast<GlobalVariable>(Ptr);
05310 
05311     // See if Ptr is a bitcast. If it is, look through it and see if we can get
05312     // global variable __stack_chk_guard.
05313     if (!GV)
05314       if (const Operator *BC = dyn_cast<Operator>(Ptr))
05315         if (BC->getOpcode() == Instruction::BitCast)
05316           GV = dyn_cast<GlobalVariable>(BC->getOperand(0));
05317 
05318     if (GV && TLI.useLoadStackGuardNode()) {
05319       // Emit a LOAD_STACK_GUARD node.
05320       MachineSDNode *Node = DAG.getMachineNode(TargetOpcode::LOAD_STACK_GUARD,
05321                                                sdl, PtrTy, Chain);
05322       MachinePointerInfo MPInfo(GV);
05323       MachineInstr::mmo_iterator MemRefs = MF.allocateMemRefsArray(1);
05324       unsigned Flags = MachineMemOperand::MOLoad |
05325                        MachineMemOperand::MOInvariant;
05326       *MemRefs = MF.getMachineMemOperand(MPInfo, Flags,
05327                                          PtrTy.getSizeInBits() / 8,
05328                                          DAG.getEVTAlignment(PtrTy));
05329       Node->setMemRefs(MemRefs, MemRefs + 1);
05330 
05331       // Copy the guard value to a virtual register so that it can be
05332       // retrieved in the epilogue.
05333       Src = SDValue(Node, 0);
05334       const TargetRegisterClass *RC =
05335           TLI.getRegClassFor(Src.getSimpleValueType());
05336       unsigned Reg = MF.getRegInfo().createVirtualRegister(RC);
05337 
05338       SPDescriptor.setGuardReg(Reg);
05339       Chain = DAG.getCopyToReg(Chain, sdl, Reg, Src);
05340     } else {
05341       Src = getValue(I.getArgOperand(0));   // The guard's value.
05342     }
05343 
05344     AllocaInst *Slot = cast<AllocaInst>(I.getArgOperand(1));
05345 
05346     int FI = FuncInfo.StaticAllocaMap[Slot];
05347     MFI->setStackProtectorIndex(FI);
05348 
05349     SDValue FIN = DAG.getFrameIndex(FI, PtrTy);
05350 
05351     // Store the stack protector onto the stack.
05352     Res = DAG.getStore(Chain, sdl, Src, FIN,
05353                        MachinePointerInfo::getFixedStack(FI),
05354                        true, false, 0);
05355     setValue(&I, Res);
05356     DAG.setRoot(Res);
05357     return nullptr;
05358   }
05359   case Intrinsic::objectsize: {
05360     // If we don't know by now, we're never going to know.
05361     ConstantInt *CI = dyn_cast<ConstantInt>(I.getArgOperand(1));
05362 
05363     assert(CI && "Non-constant type in __builtin_object_size?");
05364 
05365     SDValue Arg = getValue(I.getCalledValue());
05366     EVT Ty = Arg.getValueType();
05367 
05368     if (CI->isZero())
05369       Res = DAG.getConstant(-1ULL, Ty);
05370     else
05371       Res = DAG.getConstant(0, Ty);
05372 
05373     setValue(&I, Res);
05374     return nullptr;
05375   }
05376   case Intrinsic::annotation:
05377   case Intrinsic::ptr_annotation:
05378     // Drop the intrinsic, but forward the value
05379     setValue(&I, getValue(I.getOperand(0)));
05380     return nullptr;
05381   case Intrinsic::assume:
05382   case Intrinsic::var_annotation:
05383     // Discard annotate attributes and assumptions
05384     return nullptr;
05385 
05386   case Intrinsic::init_trampoline: {
05387     const Function *F = cast<Function>(I.getArgOperand(1)->stripPointerCasts());
05388 
05389     SDValue Ops[6];
05390     Ops[0] = getRoot();
05391     Ops[1] = getValue(I.getArgOperand(0));
05392     Ops[2] = getValue(I.getArgOperand(1));
05393     Ops[3] = getValue(I.getArgOperand(2));
05394     Ops[4] = DAG.getSrcValue(I.getArgOperand(0));
05395     Ops[5] = DAG.getSrcValue(F);
05396 
05397     Res = DAG.getNode(ISD::INIT_TRAMPOLINE, sdl, MVT::Other, Ops);
05398 
05399     DAG.setRoot(Res);
05400     return nullptr;
05401   }
05402   case Intrinsic::adjust_trampoline: {
05403     setValue(&I, DAG.getNode(ISD::ADJUST_TRAMPOLINE, sdl,
05404                              TLI.getPointerTy(),
05405                              getValue(I.getArgOperand(0))));
05406     return nullptr;
05407   }
05408   case Intrinsic::gcroot:
05409     if (GFI) {
05410       const Value *Alloca = I.getArgOperand(0)->stripPointerCasts();
05411       const Constant *TypeMap = cast<Constant>(I.getArgOperand(1));
05412 
05413       FrameIndexSDNode *FI = cast<FrameIndexSDNode>(getValue(Alloca).getNode());
05414       GFI->addStackRoot(FI->getIndex(), TypeMap);
05415     }
05416     return nullptr;
05417   case Intrinsic::gcread:
05418   case Intrinsic::gcwrite:
05419     llvm_unreachable("GC failed to lower gcread/gcwrite intrinsics!");
05420   case Intrinsic::flt_rounds:
05421     setValue(&I, DAG.getNode(ISD::FLT_ROUNDS_, sdl, MVT::i32));
05422     return nullptr;
05423 
05424   case Intrinsic::expect: {
05425     // Just replace __builtin_expect(exp, c) with EXP.
05426     setValue(&I, getValue(I.getArgOperand(0)));
05427     return nullptr;
05428   }
05429 
05430   case Intrinsic::debugtrap:
05431   case Intrinsic::trap: {
05432     StringRef TrapFuncName = TM.Options.getTrapFunctionName();
05433     if (TrapFuncName.empty()) {
05434       ISD::NodeType Op = (Intrinsic == Intrinsic::trap) ?
05435         ISD::TRAP : ISD::DEBUGTRAP;
05436       DAG.setRoot(DAG.getNode(Op, sdl,MVT::Other, getRoot()));
05437       return nullptr;
05438     }
05439     TargetLowering::ArgListTy Args;
05440 
05441     TargetLowering::CallLoweringInfo CLI(DAG);
05442     CLI.setDebugLoc(sdl).setChain(getRoot())
05443       .setCallee(CallingConv::C, I.getType(),
05444                  DAG.getExternalSymbol(TrapFuncName.data(), TLI.getPointerTy()),
05445                  std::move(Args), 0);
05446 
05447     std::pair<SDValue, SDValue> Result = TLI.LowerCallTo(CLI);
05448     DAG.setRoot(Result.second);
05449     return nullptr;
05450   }
05451 
05452   case Intrinsic::uadd_with_overflow:
05453   case Intrinsic::sadd_with_overflow:
05454   case Intrinsic::usub_with_overflow:
05455   case Intrinsic::ssub_with_overflow:
05456   case Intrinsic::umul_with_overflow:
05457   case Intrinsic::smul_with_overflow: {
05458     ISD::NodeType Op;
05459     switch (Intrinsic) {
05460     default: llvm_unreachable("Impossible intrinsic");  // Can't reach here.
05461     case Intrinsic::uadd_with_overflow: Op = ISD::UADDO; break;
05462     case Intrinsic::sadd_with_overflow: Op = ISD::SADDO; break;
05463     case Intrinsic::usub_with_overflow: Op = ISD::USUBO; break;
05464     case Intrinsic::ssub_with_overflow: Op = ISD::SSUBO; break;
05465     case Intrinsic::umul_with_overflow: Op = ISD::UMULO; break;
05466     case Intrinsic::smul_with_overflow: Op = ISD::SMULO; break;
05467     }
05468     SDValue Op1 = getValue(I.getArgOperand(0));
05469     SDValue Op2 = getValue(I.getArgOperand(1));
05470 
05471     SDVTList VTs = DAG.getVTList(Op1.getValueType(), MVT::i1);
05472     setValue(&I, DAG.getNode(Op, sdl, VTs, Op1, Op2));
05473     return nullptr;
05474   }
05475   case Intrinsic::prefetch: {
05476     SDValue Ops[5];
05477     unsigned rw = cast<ConstantInt>(I.getArgOperand(1))->getZExtValue();
05478     Ops[0] = getRoot();
05479     Ops[1] = getValue(I.getArgOperand(0));
05480     Ops[2] = getValue(I.getArgOperand(1));
05481     Ops[3] = getValue(I.getArgOperand(2));
05482     Ops[4] = getValue(I.getArgOperand(3));
05483     DAG.setRoot(DAG.getMemIntrinsicNode(ISD::PREFETCH, sdl,
05484                                         DAG.getVTList(MVT::Other), Ops,
05485                                         EVT::getIntegerVT(*Context, 8),
05486                                         MachinePointerInfo(I.getArgOperand(0)),
05487                                         0, /* align */
05488                                         false, /* volatile */
05489                                         rw==0, /* read */
05490                                         rw==1)); /* write */
05491     return nullptr;
05492   }
05493   case Intrinsic::lifetime_start:
05494   case Intrinsic::lifetime_end: {
05495     bool IsStart = (Intrinsic == Intrinsic::lifetime_start);
05496     // Stack coloring is not enabled in O0, discard region information.
05497     if (TM.getOptLevel() == CodeGenOpt::None)
05498       return nullptr;
05499 
05500     SmallVector<Value *, 4> Allocas;
05501     GetUnderlyingObjects(I.getArgOperand(1), Allocas, DL);
05502 
05503     for (SmallVectorImpl<Value*>::iterator Object = Allocas.begin(),
05504            E = Allocas.end(); Object != E; ++Object) {
05505       AllocaInst *LifetimeObject = dyn_cast_or_null<AllocaInst>(*Object);
05506 
05507       // Could not find an Alloca.
05508       if (!LifetimeObject)
05509         continue;
05510 
05511       // First check that the Alloca is static, otherwise it won't have a
05512       // valid frame index.
05513       auto SI = FuncInfo.StaticAllocaMap.find(LifetimeObject);
05514       if (SI == FuncInfo.StaticAllocaMap.end())
05515         return nullptr;
05516 
05517       int FI = SI->second;
05518 
05519       SDValue Ops[2];
05520       Ops[0] = getRoot();
05521       Ops[1] = DAG.getFrameIndex(FI, TLI.getPointerTy(), true);
05522       unsigned Opcode = (IsStart ? ISD::LIFETIME_START : ISD::LIFETIME_END);
05523 
05524       Res = DAG.getNode(Opcode, sdl, MVT::Other, Ops);
05525       DAG.setRoot(Res);
05526     }
05527     return nullptr;
05528   }
05529   case Intrinsic::invariant_start:
05530     // Discard region information.
05531     setValue(&I, DAG.getUNDEF(TLI.getPointerTy()));
05532     return nullptr;
05533   case Intrinsic::invariant_end:
05534     // Discard region information.
05535     return nullptr;
05536   case Intrinsic::stackprotectorcheck: {
05537     // Do not actually emit anything for this basic block. Instead we initialize
05538     // the stack protector descriptor and export the guard variable so we can
05539     // access it in FinishBasicBlock.
05540     const BasicBlock *BB = I.getParent();
05541     SPDescriptor.initialize(BB, FuncInfo.MBBMap[BB], I);
05542     ExportFromCurrentBlock(SPDescriptor.getGuard());
05543 
05544     // Flush our exports since we are going to process a terminator.
05545     (void)getControlRoot();
05546     return nullptr;
05547   }
05548   case Intrinsic::clear_cache:
05549     return TLI.getClearCacheBuiltinName();
05550   case Intrinsic::donothing:
05551     // ignore
05552     return nullptr;
05553   case Intrinsic::experimental_stackmap: {
05554     visitStackmap(I);
05555     return nullptr;
05556   }
05557   case Intrinsic::experimental_patchpoint_void:
05558   case Intrinsic::experimental_patchpoint_i64: {
05559     visitPatchpoint(&I);
05560     return nullptr;
05561   }
05562   case Intrinsic::experimental_gc_statepoint: {
05563     visitStatepoint(I);
05564     return nullptr;
05565   }
05566   case Intrinsic::experimental_gc_result_int:
05567   case Intrinsic::experimental_gc_result_float:
05568   case Intrinsic::experimental_gc_result_ptr: {
05569     visitGCResult(I);
05570     return nullptr;
05571   }
05572   case Intrinsic::experimental_gc_relocate: {
05573     visitGCRelocate(I);
05574     return nullptr;
05575   }
05576   case Intrinsic::instrprof_increment:
05577     llvm_unreachable("instrprof failed to lower an increment");
05578   }
05579 }
05580 
05581 std::pair<SDValue, SDValue>
05582 SelectionDAGBuilder::lowerInvokable(TargetLowering::CallLoweringInfo &CLI,
05583                                     MachineBasicBlock *LandingPad) {
05584   MachineModuleInfo &MMI = DAG.getMachineFunction().getMMI();
05585   MCSymbol *BeginLabel = nullptr;
05586 
05587   if (LandingPad) {
05588     // Insert a label before the invoke call to mark the try range.  This can be
05589     // used to detect deletion of the invoke via the MachineModuleInfo.
05590     BeginLabel = MMI.getContext().CreateTempSymbol();
05591 
05592     // For SjLj, keep track of which landing pads go with which invokes
05593     // so as to maintain the ordering of pads in the LSDA.
05594     unsigned CallSiteIndex = MMI.getCurrentCallSite();
05595     if (CallSiteIndex) {
05596       MMI.setCallSiteBeginLabel(BeginLabel, CallSiteIndex);
05597       LPadToCallSiteMap[LandingPad].push_back(CallSiteIndex);
05598 
05599       // Now that the call site is handled, stop tracking it.
05600       MMI.setCurrentCallSite(0);
05601     }
05602 
05603     // Both PendingLoads and PendingExports must be flushed here;
05604     // this call might not return.
05605     (void)getRoot();
05606     DAG.setRoot(DAG.getEHLabel(getCurSDLoc(), getControlRoot(), BeginLabel));
05607 
05608     CLI.setChain(getRoot());
05609   }
05610 
05611   const TargetLowering *TLI = TM.getSubtargetImpl()->getTargetLowering();
05612   std::pair<SDValue, SDValue> Result = TLI->LowerCallTo(CLI);
05613 
05614   assert((CLI.IsTailCall || Result.second.getNode()) &&
05615          "Non-null chain expected with non-tail call!");
05616   assert((Result.second.getNode() || !Result.first.getNode()) &&
05617          "Null value expected with tail call!");
05618 
05619   if (!Result.second.getNode()) {
05620     // As a special case, a null chain means that a tail call has been emitted
05621     // and the DAG root is already updated.
05622     HasTailCall = true;
05623 
05624     // Since there's no actual continuation from this block, nothing can be
05625     // relying on us setting vregs for them.
05626     PendingExports.clear();
05627   } else {
05628     DAG.setRoot(Result.second);
05629   }
05630 
05631   if (LandingPad) {
05632     // Insert a label at the end of the invoke call to mark the try range.  This
05633     // can be used to detect deletion of the invoke via the MachineModuleInfo.
05634     MCSymbol *EndLabel = MMI.getContext().CreateTempSymbol();
05635     DAG.setRoot(DAG.getEHLabel(getCurSDLoc(), getRoot(), EndLabel));
05636 
05637     // Inform MachineModuleInfo of range.
05638     MMI.addInvoke(LandingPad, BeginLabel, EndLabel);
05639   }
05640 
05641   return Result;
05642 }
05643 
05644 void SelectionDAGBuilder::LowerCallTo(ImmutableCallSite CS, SDValue Callee,
05645                                       bool isTailCall,
05646                                       MachineBasicBlock *LandingPad) {
05647   PointerType *PT = cast<PointerType>(CS.getCalledValue()->getType());
05648   FunctionType *FTy = cast<FunctionType>(PT->getElementType());
05649   Type *RetTy = FTy->getReturnType();
05650 
05651   TargetLowering::ArgListTy Args;
05652   TargetLowering::ArgListEntry Entry;
05653   Args.reserve(CS.arg_size());
05654 
05655   for (ImmutableCallSite::arg_iterator i = CS.arg_begin(), e = CS.arg_end();
05656        i != e; ++i) {
05657     const Value *V = *i;
05658 
05659     // Skip empty types
05660     if (V->getType()->isEmptyTy())
05661       continue;
05662 
05663     SDValue ArgNode = getValue(V);
05664     Entry.Node = ArgNode; Entry.Ty = V->getType();
05665 
05666     // Skip the first return-type Attribute to get to params.
05667     Entry.setAttributes(&CS, i - CS.arg_begin() + 1);
05668     Args.push_back(Entry);
05669   }
05670 
05671   // Check if target-independent constraints permit a tail call here.
05672   // Target-dependent constraints are checked within TLI->LowerCallTo.
05673   if (isTailCall && !isInTailCallPosition(CS, DAG.getTarget()))
05674     isTailCall = false;
05675 
05676   TargetLowering::CallLoweringInfo CLI(DAG);
05677   CLI.setDebugLoc(getCurSDLoc()).setChain(getRoot())
05678     .setCallee(RetTy, FTy, Callee, std::move(Args), CS)
05679     .setTailCall(isTailCall);
05680   std::pair<SDValue,SDValue> Result = lowerInvokable(CLI, LandingPad);
05681 
05682   if (Result.first.getNode())
05683     setValue(CS.getInstruction(), Result.first);
05684 }
05685 
05686 /// IsOnlyUsedInZeroEqualityComparison - Return true if it only matters that the
05687 /// value is equal or not-equal to zero.
05688 static bool IsOnlyUsedInZeroEqualityComparison(const Value *V) {
05689   for (const User *U : V->users()) {
05690     if (const ICmpInst *IC = dyn_cast<ICmpInst>(U))
05691       if (IC->isEquality())
05692         if (const Constant *C = dyn_cast<Constant>(IC->getOperand(1)))
05693           if (C->isNullValue())
05694             continue;
05695     // Unknown instruction.
05696     return false;
05697   }
05698   return true;
05699 }
05700 
05701 static SDValue getMemCmpLoad(const Value *PtrVal, MVT LoadVT,
05702                              Type *LoadTy,
05703                              SelectionDAGBuilder &Builder) {
05704 
05705   // Check to see if this load can be trivially constant folded, e.g. if the
05706   // input is from a string literal.
05707   if (const Constant *LoadInput = dyn_cast<Constant>(PtrVal)) {
05708     // Cast pointer to the type we really want to load.
05709     LoadInput = ConstantExpr::getBitCast(const_cast<Constant *>(LoadInput),
05710                                          PointerType::getUnqual(LoadTy));
05711 
05712     if (const Constant *LoadCst =
05713           ConstantFoldLoadFromConstPtr(const_cast<Constant *>(LoadInput),
05714                                        Builder.DL))
05715       return Builder.getValue(LoadCst);
05716   }
05717 
05718   // Otherwise, we have to emit the load.  If the pointer is to unfoldable but
05719   // still constant memory, the input chain can be the entry node.
05720   SDValue Root;
05721   bool ConstantMemory = false;
05722 
05723   // Do not serialize (non-volatile) loads of constant memory with anything.
05724   if (Builder.AA->pointsToConstantMemory(PtrVal)) {
05725     Root = Builder.DAG.getEntryNode();
05726     ConstantMemory = true;
05727   } else {
05728     // Do not serialize non-volatile loads against each other.
05729     Root = Builder.DAG.getRoot();
05730   }
05731 
05732   SDValue Ptr = Builder.getValue(PtrVal);
05733   SDValue LoadVal = Builder.DAG.getLoad(LoadVT, Builder.getCurSDLoc(), Root,
05734                                         Ptr, MachinePointerInfo(PtrVal),
05735                                         false /*volatile*/,
05736                                         false /*nontemporal*/,
05737                                         false /*isinvariant*/, 1 /* align=1 */);
05738 
05739   if (!ConstantMemory)
05740     Builder.PendingLoads.push_back(LoadVal.getValue(1));
05741   return LoadVal;
05742 }
05743 
05744 /// processIntegerCallValue - Record the value for an instruction that
05745 /// produces an integer result, converting the type where necessary.
05746 void SelectionDAGBuilder::processIntegerCallValue(const Instruction &I,
05747                                                   SDValue Value,
05748                                                   bool IsSigned) {
05749   EVT VT = DAG.getTargetLoweringInfo().getValueType(I.getType(), true);
05750   if (IsSigned)
05751     Value = DAG.getSExtOrTrunc(Value, getCurSDLoc(), VT);
05752   else
05753     Value = DAG.getZExtOrTrunc(Value, getCurSDLoc(), VT);
05754   setValue(&I, Value);
05755 }
05756 
05757 /// visitMemCmpCall - See if we can lower a call to memcmp in an optimized form.
05758 /// If so, return true and lower it, otherwise return false and it will be
05759 /// lowered like a normal call.
05760 bool SelectionDAGBuilder::visitMemCmpCall(const CallInst &I) {
05761   // Verify that the prototype makes sense.  int memcmp(void*,void*,size_t)
05762   if (I.getNumArgOperands() != 3)
05763     return false;
05764 
05765   const Value *LHS = I.getArgOperand(0), *RHS = I.getArgOperand(1);
05766   if (!LHS->getType()->isPointerTy() || !RHS->getType()->isPointerTy() ||
05767       !I.getArgOperand(2)->getType()->isIntegerTy() ||
05768       !I.getType()->isIntegerTy())
05769     return false;
05770 
05771   const Value *Size = I.getArgOperand(2);
05772   const ConstantInt *CSize = dyn_cast<ConstantInt>(Size);
05773   if (CSize && CSize->getZExtValue() == 0) {
05774     EVT CallVT = DAG.getTargetLoweringInfo().getValueType(I.getType(), true);
05775     setValue(&I, DAG.getConstant(0, CallVT));
05776     return true;
05777   }
05778 
05779   const TargetSelectionDAGInfo &TSI = DAG.getSelectionDAGInfo();
05780   std::pair<SDValue, SDValue> Res =
05781     TSI.EmitTargetCodeForMemcmp(DAG, getCurSDLoc(), DAG.getRoot(),
05782                                 getValue(LHS), getValue(RHS), getValue(Size),
05783                                 MachinePointerInfo(LHS),
05784                                 MachinePointerInfo(RHS));
05785   if (Res.first.getNode()) {
05786     processIntegerCallValue(I, Res.first, true);
05787     PendingLoads.push_back(Res.second);
05788     return true;
05789   }
05790 
05791   // memcmp(S1,S2,2) != 0 -> (*(short*)LHS != *(short*)RHS)  != 0
05792   // memcmp(S1,S2,4) != 0 -> (*(int*)LHS != *(int*)RHS)  != 0
05793   if (CSize && IsOnlyUsedInZeroEqualityComparison(&I)) {
05794     bool ActuallyDoIt = true;
05795     MVT LoadVT;
05796     Type *LoadTy;
05797     switch (CSize->getZExtValue()) {
05798     default:
05799       LoadVT = MVT::Other;
05800       LoadTy = nullptr;
05801       ActuallyDoIt = false;
05802       break;
05803     case 2:
05804       LoadVT = MVT::i16;
05805       LoadTy = Type::getInt16Ty(CSize->getContext());
05806       break;
05807     case 4:
05808       LoadVT = MVT::i32;
05809       LoadTy = Type::getInt32Ty(CSize->getContext());
05810       break;
05811     case 8:
05812       LoadVT = MVT::i64;
05813       LoadTy = Type::getInt64Ty(CSize->getContext());
05814       break;
05815         /*
05816     case 16:
05817       LoadVT = MVT::v4i32;
05818       LoadTy = Type::getInt32Ty(CSize->getContext());
05819       LoadTy = VectorType::get(LoadTy, 4);
05820       break;
05821          */
05822     }
05823 
05824     // This turns into unaligned loads.  We only do this if the target natively
05825     // supports the MVT we'll be loading or if it is small enough (<= 4) that
05826     // we'll only produce a small number of byte loads.
05827 
05828     // Require that we can find a legal MVT, and only do this if the target
05829     // supports unaligned loads of that type.  Expanding into byte loads would
05830     // bloat the code.
05831     const TargetLowering &TLI = DAG.getTargetLoweringInfo();
05832     if (ActuallyDoIt && CSize->getZExtValue() > 4) {
05833       unsigned DstAS = LHS->getType()->getPointerAddressSpace();
05834       unsigned SrcAS = RHS->getType()->getPointerAddressSpace();
05835       // TODO: Handle 5 byte compare as 4-byte + 1 byte.
05836       // TODO: Handle 8 byte compare on x86-32 as two 32-bit loads.
05837       // TODO: Check alignment of src and dest ptrs.
05838       if (!TLI.isTypeLegal(LoadVT) ||
05839           !TLI.allowsMisalignedMemoryAccesses(LoadVT, SrcAS) ||
05840           !TLI.allowsMisalignedMemoryAccesses(LoadVT, DstAS))
05841         ActuallyDoIt = false;
05842     }
05843 
05844     if (ActuallyDoIt) {
05845       SDValue LHSVal = getMemCmpLoad(LHS, LoadVT, LoadTy, *this);
05846       SDValue RHSVal = getMemCmpLoad(RHS, LoadVT, LoadTy, *this);
05847 
05848       SDValue Res = DAG.getSetCC(getCurSDLoc(), MVT::i1, LHSVal, RHSVal,
05849                                  ISD::SETNE);
05850       processIntegerCallValue(I, Res, false);
05851       return true;
05852     }
05853   }
05854 
05855 
05856   return false;
05857 }
05858 
05859 /// visitMemChrCall -- See if we can lower a memchr call into an optimized
05860 /// form.  If so, return true and lower it, otherwise return false and it
05861 /// will be lowered like a normal call.
05862 bool SelectionDAGBuilder::visitMemChrCall(const CallInst &I) {
05863   // Verify that the prototype makes sense.  void *memchr(void *, int, size_t)
05864   if (I.getNumArgOperands() != 3)
05865     return false;
05866 
05867   const Value *Src = I.getArgOperand(0);
05868   const Value *Char = I.getArgOperand(1);
05869   const Value *Length = I.getArgOperand(2);
05870   if (!Src->getType()->isPointerTy() ||
05871       !Char->getType()->isIntegerTy() ||
05872       !Length->getType()->isIntegerTy() ||
05873       !I.getType()->isPointerTy())
05874     return false;
05875 
05876   const TargetSelectionDAGInfo &TSI = DAG.getSelectionDAGInfo();
05877   std::pair<SDValue, SDValue> Res =
05878     TSI.EmitTargetCodeForMemchr(DAG, getCurSDLoc(), DAG.getRoot(),
05879                                 getValue(Src), getValue(Char), getValue(Length),
05880                                 MachinePointerInfo(Src));
05881   if (Res.first.getNode()) {
05882     setValue(&I, Res.first);
05883     PendingLoads.push_back(Res.second);
05884     return true;
05885   }
05886 
05887   return false;
05888 }
05889 
05890 /// visitStrCpyCall -- See if we can lower a strcpy or stpcpy call into an
05891 /// optimized form.  If so, return true and lower it, otherwise return false
05892 /// and it will be lowered like a normal call.
05893 bool SelectionDAGBuilder::visitStrCpyCall(const CallInst &I, bool isStpcpy) {
05894   // Verify that the prototype makes sense.  char *strcpy(char *, char *)
05895   if (I.getNumArgOperands() != 2)
05896     return false;
05897 
05898   const Value *Arg0 = I.getArgOperand(0), *Arg1 = I.getArgOperand(1);
05899   if (!Arg0->getType()->isPointerTy() ||
05900       !Arg1->getType()->isPointerTy() ||
05901       !I.getType()->isPointerTy())
05902     return false;
05903 
05904   const TargetSelectionDAGInfo &TSI = DAG.getSelectionDAGInfo();
05905   std::pair<SDValue, SDValue> Res =
05906     TSI.EmitTargetCodeForStrcpy(DAG, getCurSDLoc(), getRoot(),
05907                                 getValue(Arg0), getValue(Arg1),
05908                                 MachinePointerInfo(Arg0),
05909                                 MachinePointerInfo(Arg1), isStpcpy);
05910   if (Res.first.getNode()) {
05911     setValue(&I, Res.first);
05912     DAG.setRoot(Res.second);
05913     return true;
05914   }
05915 
05916   return false;
05917 }
05918 
05919 /// visitStrCmpCall - See if we can lower a call to strcmp in an optimized form.
05920 /// If so, return true and lower it, otherwise return false and it will be
05921 /// lowered like a normal call.
05922 bool SelectionDAGBuilder::visitStrCmpCall(const CallInst &I) {
05923   // Verify that the prototype makes sense.  int strcmp(void*,void*)
05924   if (I.getNumArgOperands() != 2)
05925     return false;
05926 
05927   const Value *Arg0 = I.getArgOperand(0), *Arg1 = I.getArgOperand(1);
05928   if (!Arg0->getType()->isPointerTy() ||
05929       !Arg1->getType()->isPointerTy() ||
05930       !I.getType()->isIntegerTy())
05931     return false;
05932 
05933   const TargetSelectionDAGInfo &TSI = DAG.getSelectionDAGInfo();
05934   std::pair<SDValue, SDValue> Res =
05935     TSI.EmitTargetCodeForStrcmp(DAG, getCurSDLoc(), DAG.getRoot(),
05936                                 getValue(Arg0), getValue(Arg1),
05937                                 MachinePointerInfo(Arg0),
05938                                 MachinePointerInfo(Arg1));
05939   if (Res.first.getNode()) {
05940     processIntegerCallValue(I, Res.first, true);
05941     PendingLoads.push_back(Res.second);
05942     return true;
05943   }
05944 
05945   return false;
05946 }
05947 
05948 /// visitStrLenCall -- See if we can lower a strlen call into an optimized
05949 /// form.  If so, return true and lower it, otherwise return false and it
05950 /// will be lowered like a normal call.
05951 bool SelectionDAGBuilder::visitStrLenCall(const CallInst &I) {
05952   // Verify that the prototype makes sense.  size_t strlen(char *)
05953   if (I.getNumArgOperands() != 1)
05954     return false;
05955 
05956   const Value *Arg0 = I.getArgOperand(0);
05957   if (!Arg0->getType()->isPointerTy() || !I.getType()->isIntegerTy())
05958     return false;
05959 
05960   const TargetSelectionDAGInfo &TSI = DAG.getSelectionDAGInfo();
05961   std::pair<SDValue, SDValue> Res =
05962     TSI.EmitTargetCodeForStrlen(DAG, getCurSDLoc(), DAG.getRoot(),
05963                                 getValue(Arg0), MachinePointerInfo(Arg0));
05964   if (Res.first.getNode()) {
05965     processIntegerCallValue(I, Res.first, false);
05966     PendingLoads.push_back(Res.second);
05967     return true;
05968   }
05969 
05970   return false;
05971 }
05972 
05973 /// visitStrNLenCall -- See if we can lower a strnlen call into an optimized
05974 /// form.  If so, return true and lower it, otherwise return false and it
05975 /// will be lowered like a normal call.
05976 bool SelectionDAGBuilder::visitStrNLenCall(const CallInst &I) {
05977   // Verify that the prototype makes sense.  size_t strnlen(char *, size_t)
05978   if (I.getNumArgOperands() != 2)
05979     return false;
05980 
05981   const Value *Arg0 = I.getArgOperand(0), *Arg1 = I.getArgOperand(1);
05982   if (!Arg0->getType()->isPointerTy() ||
05983       !Arg1->getType()->isIntegerTy() ||
05984       !I.getType()->isIntegerTy())
05985     return false;
05986 
05987   const TargetSelectionDAGInfo &TSI = DAG.getSelectionDAGInfo();
05988   std::pair<SDValue, SDValue> Res =
05989     TSI.EmitTargetCodeForStrnlen(DAG, getCurSDLoc(), DAG.getRoot(),
05990                                  getValue(Arg0), getValue(Arg1),
05991                                  MachinePointerInfo(Arg0));
05992   if (Res.first.getNode()) {
05993     processIntegerCallValue(I, Res.first, false);
05994     PendingLoads.push_back(Res.second);
05995     return true;
05996   }
05997 
05998   return false;
05999 }
06000 
06001 /// visitUnaryFloatCall - If a call instruction is a unary floating-point
06002 /// operation (as expected), translate it to an SDNode with the specified opcode
06003 /// and return true.
06004 bool SelectionDAGBuilder::visitUnaryFloatCall(const CallInst &I,
06005                                               unsigned Opcode) {
06006   // Sanity check that it really is a unary floating-point call.
06007   if (I.getNumArgOperands() != 1 ||
06008       !I.getArgOperand(0)->getType()->isFloatingPointTy() ||
06009       I.getType() != I.getArgOperand(0)->getType() ||
06010       !I.onlyReadsMemory())
06011     return false;
06012 
06013   SDValue Tmp = getValue(I.getArgOperand(0));
06014   setValue(&I, DAG.getNode(Opcode, getCurSDLoc(), Tmp.getValueType(), Tmp));
06015   return true;
06016 }
06017 
06018 /// visitBinaryFloatCall - If a call instruction is a binary floating-point
06019 /// operation (as expected), translate it to an SDNode with the specified opcode
06020 /// and return true.
06021 bool SelectionDAGBuilder::visitBinaryFloatCall(const CallInst &I,
06022                                                unsigned Opcode) {
06023   // Sanity check that it really is a binary floating-point call.
06024   if (I.getNumArgOperands() != 2 ||
06025       !I.getArgOperand(0)->getType()->isFloatingPointTy() ||
06026       I.getType() != I.getArgOperand(0)->getType() ||
06027       I.getType() != I.getArgOperand(1)->getType() ||
06028       !I.onlyReadsMemory())
06029     return false;
06030 
06031   SDValue Tmp0 = getValue(I.getArgOperand(0));
06032   SDValue Tmp1 = getValue(I.getArgOperand(1));
06033   EVT VT = Tmp0.getValueType();
06034   setValue(&I, DAG.getNode(Opcode, getCurSDLoc(), VT, Tmp0, Tmp1));
06035   return true;
06036 }
06037 
06038 void SelectionDAGBuilder::visitCall(const CallInst &I) {
06039   // Handle inline assembly differently.
06040   if (isa<InlineAsm>(I.getCalledValue())) {
06041     visitInlineAsm(&I);
06042     return;
06043   }
06044 
06045   MachineModuleInfo &MMI = DAG.getMachineFunction().getMMI();
06046   ComputeUsesVAFloatArgument(I, &MMI);
06047 
06048   const char *RenameFn = nullptr;
06049   if (Function *F = I.getCalledFunction()) {
06050     if (F->isDeclaration()) {
06051       if (const TargetIntrinsicInfo *II = TM.getIntrinsicInfo()) {
06052         if (unsigned IID = II->getIntrinsicID(F)) {
06053           RenameFn = visitIntrinsicCall(I, IID);
06054           if (!RenameFn)
06055             return;
06056         }
06057       }
06058       if (unsigned IID = F->getIntrinsicID()) {
06059         RenameFn = visitIntrinsicCall(I, IID);
06060         if (!RenameFn)
06061           return;
06062       }
06063     }
06064 
06065     // Check for well-known libc/libm calls.  If the function is internal, it
06066     // can't be a library call.
06067     LibFunc::Func Func;
06068     if (!F->hasLocalLinkage() && F->hasName() &&
06069         LibInfo->getLibFunc(F->getName(), Func) &&
06070         LibInfo->hasOptimizedCodeGen(Func)) {
06071       switch (Func) {
06072       default: break;
06073       case LibFunc::copysign:
06074       case LibFunc::copysignf:
06075       case LibFunc::copysignl:
06076         if (I.getNumArgOperands() == 2 &&   // Basic sanity checks.
06077             I.getArgOperand(0)->getType()->isFloatingPointTy() &&
06078             I.getType() == I.getArgOperand(0)->getType() &&
06079             I.getType() == I.getArgOperand(1)->getType() &&
06080             I.onlyReadsMemory()) {
06081           SDValue LHS = getValue(I.getArgOperand(0));
06082           SDValue RHS = getValue(I.getArgOperand(1));
06083           setValue(&I, DAG.getNode(ISD::FCOPYSIGN, getCurSDLoc(),
06084                                    LHS.getValueType(), LHS, RHS));
06085           return;
06086         }
06087         break;
06088       case LibFunc::fabs:
06089       case LibFunc::fabsf:
06090       case LibFunc::fabsl:
06091         if (visitUnaryFloatCall(I, ISD::FABS))
06092           return;
06093         break;
06094       case LibFunc::fmin:
06095       case LibFunc::fminf:
06096       case LibFunc::fminl:
06097         if (visitBinaryFloatCall(I, ISD::FMINNUM))
06098           return;
06099         break;
06100       case LibFunc::fmax:
06101       case LibFunc::fmaxf:
06102       case LibFunc::fmaxl:
06103         if (visitBinaryFloatCall(I, ISD::FMAXNUM))
06104           return;
06105         break;
06106       case LibFunc::sin:
06107       case LibFunc::sinf:
06108       case LibFunc::sinl:
06109         if (visitUnaryFloatCall(I, ISD::FSIN))
06110           return;
06111         break;
06112       case LibFunc::cos:
06113       case LibFunc::cosf:
06114       case LibFunc::cosl:
06115         if (visitUnaryFloatCall(I, ISD::FCOS))
06116           return;
06117         break;
06118       case LibFunc::sqrt:
06119       case LibFunc::sqrtf:
06120       case LibFunc::sqrtl:
06121       case LibFunc::sqrt_finite:
06122       case LibFunc::sqrtf_finite:
06123       case LibFunc::sqrtl_finite:
06124         if (visitUnaryFloatCall(I, ISD::FSQRT))
06125           return;
06126         break;
06127       case LibFunc::floor:
06128       case LibFunc::floorf:
06129       case LibFunc::floorl:
06130         if (visitUnaryFloatCall(I, ISD::FFLOOR))
06131           return;
06132         break;
06133       case LibFunc::nearbyint:
06134       case LibFunc::nearbyintf:
06135       case LibFunc::nearbyintl:
06136         if (visitUnaryFloatCall(I, ISD::FNEARBYINT))
06137           return;
06138         break;
06139       case LibFunc::ceil:
06140       case LibFunc::ceilf:
06141       case LibFunc::ceill:
06142         if (visitUnaryFloatCall(I, ISD::FCEIL))
06143           return;
06144         break;
06145       case LibFunc::rint:
06146       case LibFunc::ri