LLVM API Documentation

SelectionDAGBuilder.cpp
Go to the documentation of this file.
00001 //===-- SelectionDAGBuilder.cpp - Selection-DAG building ------------------===//
00002 //
00003 //                     The LLVM Compiler Infrastructure
00004 //
00005 // This file is distributed under the University of Illinois Open Source
00006 // License. See LICENSE.TXT for details.
00007 //
00008 //===----------------------------------------------------------------------===//
00009 //
00010 // This implements routines for translating from LLVM IR into SelectionDAG IR.
00011 //
00012 //===----------------------------------------------------------------------===//
00013 
00014 #include "SelectionDAGBuilder.h"
00015 #include "SDNodeDbgValue.h"
00016 #include "llvm/ADT/BitVector.h"
00017 #include "llvm/ADT/Optional.h"
00018 #include "llvm/ADT/SmallSet.h"
00019 #include "llvm/Analysis/AliasAnalysis.h"
00020 #include "llvm/Analysis/BranchProbabilityInfo.h"
00021 #include "llvm/Analysis/ConstantFolding.h"
00022 #include "llvm/Analysis/ValueTracking.h"
00023 #include "llvm/CodeGen/Analysis.h"
00024 #include "llvm/CodeGen/FastISel.h"
00025 #include "llvm/CodeGen/FunctionLoweringInfo.h"
00026 #include "llvm/CodeGen/GCMetadata.h"
00027 #include "llvm/CodeGen/GCStrategy.h"
00028 #include "llvm/CodeGen/MachineFrameInfo.h"
00029 #include "llvm/CodeGen/MachineFunction.h"
00030 #include "llvm/CodeGen/MachineInstrBuilder.h"
00031 #include "llvm/CodeGen/MachineJumpTableInfo.h"
00032 #include "llvm/CodeGen/MachineModuleInfo.h"
00033 #include "llvm/CodeGen/MachineRegisterInfo.h"
00034 #include "llvm/CodeGen/SelectionDAG.h"
00035 #include "llvm/CodeGen/StackMaps.h"
00036 #include "llvm/IR/CallingConv.h"
00037 #include "llvm/IR/Constants.h"
00038 #include "llvm/IR/DataLayout.h"
00039 #include "llvm/IR/DebugInfo.h"
00040 #include "llvm/IR/DerivedTypes.h"
00041 #include "llvm/IR/Function.h"
00042 #include "llvm/IR/GlobalVariable.h"
00043 #include "llvm/IR/InlineAsm.h"
00044 #include "llvm/IR/Instructions.h"
00045 #include "llvm/IR/IntrinsicInst.h"
00046 #include "llvm/IR/Intrinsics.h"
00047 #include "llvm/IR/LLVMContext.h"
00048 #include "llvm/IR/Module.h"
00049 #include "llvm/Support/CommandLine.h"
00050 #include "llvm/Support/Debug.h"
00051 #include "llvm/Support/ErrorHandling.h"
00052 #include "llvm/Support/MathExtras.h"
00053 #include "llvm/Support/raw_ostream.h"
00054 #include "llvm/Target/TargetFrameLowering.h"
00055 #include "llvm/Target/TargetInstrInfo.h"
00056 #include "llvm/Target/TargetIntrinsicInfo.h"
00057 #include "llvm/Target/TargetLibraryInfo.h"
00058 #include "llvm/Target/TargetLowering.h"
00059 #include "llvm/Target/TargetOptions.h"
00060 #include "llvm/Target/TargetSelectionDAGInfo.h"
00061 #include "llvm/Target/TargetSubtargetInfo.h"
00062 #include <algorithm>
00063 using namespace llvm;
00064 
00065 #define DEBUG_TYPE "isel"
00066 
00067 /// LimitFloatPrecision - Generate low-precision inline sequences for
00068 /// some float libcalls (6, 8 or 12 bits).
00069 static unsigned LimitFloatPrecision;
00070 
00071 static cl::opt<unsigned, true>
00072 LimitFPPrecision("limit-float-precision",
00073                  cl::desc("Generate low-precision inline sequences "
00074                           "for some float libcalls"),
00075                  cl::location(LimitFloatPrecision),
00076                  cl::init(0));
00077 
00078 // Limit the width of DAG chains. This is important in general to prevent
00079 // prevent DAG-based analysis from blowing up. For example, alias analysis and
00080 // load clustering may not complete in reasonable time. It is difficult to
00081 // recognize and avoid this situation within each individual analysis, and
00082 // future analyses are likely to have the same behavior. Limiting DAG width is
00083 // the safe approach, and will be especially important with global DAGs.
00084 //
00085 // MaxParallelChains default is arbitrarily high to avoid affecting
00086 // optimization, but could be lowered to improve compile time. Any ld-ld-st-st
00087 // sequence over this should have been converted to llvm.memcpy by the
00088 // frontend. It easy to induce this behavior with .ll code such as:
00089 // %buffer = alloca [4096 x i8]
00090 // %data = load [4096 x i8]* %argPtr
00091 // store [4096 x i8] %data, [4096 x i8]* %buffer
00092 static const unsigned MaxParallelChains = 64;
00093 
00094 static SDValue getCopyFromPartsVector(SelectionDAG &DAG, SDLoc DL,
00095                                       const SDValue *Parts, unsigned NumParts,
00096                                       MVT PartVT, EVT ValueVT, const Value *V);
00097 
00098 /// getCopyFromParts - Create a value that contains the specified legal parts
00099 /// combined into the value they represent.  If the parts combine to a type
00100 /// larger then ValueVT then AssertOp can be used to specify whether the extra
00101 /// bits are known to be zero (ISD::AssertZext) or sign extended from ValueVT
00102 /// (ISD::AssertSext).
00103 static SDValue getCopyFromParts(SelectionDAG &DAG, SDLoc DL,
00104                                 const SDValue *Parts,
00105                                 unsigned NumParts, MVT PartVT, EVT ValueVT,
00106                                 const Value *V,
00107                                 ISD::NodeType AssertOp = ISD::DELETED_NODE) {
00108   if (ValueVT.isVector())
00109     return getCopyFromPartsVector(DAG, DL, Parts, NumParts,
00110                                   PartVT, ValueVT, V);
00111 
00112   assert(NumParts > 0 && "No parts to assemble!");
00113   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
00114   SDValue Val = Parts[0];
00115 
00116   if (NumParts > 1) {
00117     // Assemble the value from multiple parts.
00118     if (ValueVT.isInteger()) {
00119       unsigned PartBits = PartVT.getSizeInBits();
00120       unsigned ValueBits = ValueVT.getSizeInBits();
00121 
00122       // Assemble the power of 2 part.
00123       unsigned RoundParts = NumParts & (NumParts - 1) ?
00124         1 << Log2_32(NumParts) : NumParts;
00125       unsigned RoundBits = PartBits * RoundParts;
00126       EVT RoundVT = RoundBits == ValueBits ?
00127         ValueVT : EVT::getIntegerVT(*DAG.getContext(), RoundBits);
00128       SDValue Lo, Hi;
00129 
00130       EVT HalfVT = EVT::getIntegerVT(*DAG.getContext(), RoundBits/2);
00131 
00132       if (RoundParts > 2) {
00133         Lo = getCopyFromParts(DAG, DL, Parts, RoundParts / 2,
00134                               PartVT, HalfVT, V);
00135         Hi = getCopyFromParts(DAG, DL, Parts + RoundParts / 2,
00136                               RoundParts / 2, PartVT, HalfVT, V);
00137       } else {
00138         Lo = DAG.getNode(ISD::BITCAST, DL, HalfVT, Parts[0]);
00139         Hi = DAG.getNode(ISD::BITCAST, DL, HalfVT, Parts[1]);
00140       }
00141 
00142       if (TLI.isBigEndian())
00143         std::swap(Lo, Hi);
00144 
00145       Val = DAG.getNode(ISD::BUILD_PAIR, DL, RoundVT, Lo, Hi);
00146 
00147       if (RoundParts < NumParts) {
00148         // Assemble the trailing non-power-of-2 part.
00149         unsigned OddParts = NumParts - RoundParts;
00150         EVT OddVT = EVT::getIntegerVT(*DAG.getContext(), OddParts * PartBits);
00151         Hi = getCopyFromParts(DAG, DL,
00152                               Parts + RoundParts, OddParts, PartVT, OddVT, V);
00153 
00154         // Combine the round and odd parts.
00155         Lo = Val;
00156         if (TLI.isBigEndian())
00157           std::swap(Lo, Hi);
00158         EVT TotalVT = EVT::getIntegerVT(*DAG.getContext(), NumParts * PartBits);
00159         Hi = DAG.getNode(ISD::ANY_EXTEND, DL, TotalVT, Hi);
00160         Hi = DAG.getNode(ISD::SHL, DL, TotalVT, Hi,
00161                          DAG.getConstant(Lo.getValueType().getSizeInBits(),
00162                                          TLI.getPointerTy()));
00163         Lo = DAG.getNode(ISD::ZERO_EXTEND, DL, TotalVT, Lo);
00164         Val = DAG.getNode(ISD::OR, DL, TotalVT, Lo, Hi);
00165       }
00166     } else if (PartVT.isFloatingPoint()) {
00167       // FP split into multiple FP parts (for ppcf128)
00168       assert(ValueVT == EVT(MVT::ppcf128) && PartVT == MVT::f64 &&
00169              "Unexpected split");
00170       SDValue Lo, Hi;
00171       Lo = DAG.getNode(ISD::BITCAST, DL, EVT(MVT::f64), Parts[0]);
00172       Hi = DAG.getNode(ISD::BITCAST, DL, EVT(MVT::f64), Parts[1]);
00173       if (TLI.hasBigEndianPartOrdering(ValueVT))
00174         std::swap(Lo, Hi);
00175       Val = DAG.getNode(ISD::BUILD_PAIR, DL, ValueVT, Lo, Hi);
00176     } else {
00177       // FP split into integer parts (soft fp)
00178       assert(ValueVT.isFloatingPoint() && PartVT.isInteger() &&
00179              !PartVT.isVector() && "Unexpected split");
00180       EVT IntVT = EVT::getIntegerVT(*DAG.getContext(), ValueVT.getSizeInBits());
00181       Val = getCopyFromParts(DAG, DL, Parts, NumParts, PartVT, IntVT, V);
00182     }
00183   }
00184 
00185   // There is now one part, held in Val.  Correct it to match ValueVT.
00186   EVT PartEVT = Val.getValueType();
00187 
00188   if (PartEVT == ValueVT)
00189     return Val;
00190 
00191   if (PartEVT.isInteger() && ValueVT.isInteger()) {
00192     if (ValueVT.bitsLT(PartEVT)) {
00193       // For a truncate, see if we have any information to
00194       // indicate whether the truncated bits will always be
00195       // zero or sign-extension.
00196       if (AssertOp != ISD::DELETED_NODE)
00197         Val = DAG.getNode(AssertOp, DL, PartEVT, Val,
00198                           DAG.getValueType(ValueVT));
00199       return DAG.getNode(ISD::TRUNCATE, DL, ValueVT, Val);
00200     }
00201     return DAG.getNode(ISD::ANY_EXTEND, DL, ValueVT, Val);
00202   }
00203 
00204   if (PartEVT.isFloatingPoint() && ValueVT.isFloatingPoint()) {
00205     // FP_ROUND's are always exact here.
00206     if (ValueVT.bitsLT(Val.getValueType()))
00207       return DAG.getNode(ISD::FP_ROUND, DL, ValueVT, Val,
00208                          DAG.getTargetConstant(1, TLI.getPointerTy()));
00209 
00210     return DAG.getNode(ISD::FP_EXTEND, DL, ValueVT, Val);
00211   }
00212 
00213   if (PartEVT.getSizeInBits() == ValueVT.getSizeInBits())
00214     return DAG.getNode(ISD::BITCAST, DL, ValueVT, Val);
00215 
00216   llvm_unreachable("Unknown mismatch!");
00217 }
00218 
00219 static void diagnosePossiblyInvalidConstraint(LLVMContext &Ctx, const Value *V,
00220                                               const Twine &ErrMsg) {
00221   const Instruction *I = dyn_cast_or_null<Instruction>(V);
00222   if (!V)
00223     return Ctx.emitError(ErrMsg);
00224 
00225   const char *AsmError = ", possible invalid constraint for vector type";
00226   if (const CallInst *CI = dyn_cast<CallInst>(I))
00227     if (isa<InlineAsm>(CI->getCalledValue()))
00228       return Ctx.emitError(I, ErrMsg + AsmError);
00229 
00230   return Ctx.emitError(I, ErrMsg);
00231 }
00232 
00233 /// getCopyFromPartsVector - Create a value that contains the specified legal
00234 /// parts combined into the value they represent.  If the parts combine to a
00235 /// type larger then ValueVT then AssertOp can be used to specify whether the
00236 /// extra bits are known to be zero (ISD::AssertZext) or sign extended from
00237 /// ValueVT (ISD::AssertSext).
00238 static SDValue getCopyFromPartsVector(SelectionDAG &DAG, SDLoc DL,
00239                                       const SDValue *Parts, unsigned NumParts,
00240                                       MVT PartVT, EVT ValueVT, const Value *V) {
00241   assert(ValueVT.isVector() && "Not a vector value");
00242   assert(NumParts > 0 && "No parts to assemble!");
00243   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
00244   SDValue Val = Parts[0];
00245 
00246   // Handle a multi-element vector.
00247   if (NumParts > 1) {
00248     EVT IntermediateVT;
00249     MVT RegisterVT;
00250     unsigned NumIntermediates;
00251     unsigned NumRegs =
00252     TLI.getVectorTypeBreakdown(*DAG.getContext(), ValueVT, IntermediateVT,
00253                                NumIntermediates, RegisterVT);
00254     assert(NumRegs == NumParts && "Part count doesn't match vector breakdown!");
00255     NumParts = NumRegs; // Silence a compiler warning.
00256     assert(RegisterVT == PartVT && "Part type doesn't match vector breakdown!");
00257     assert(RegisterVT == Parts[0].getSimpleValueType() &&
00258            "Part type doesn't match part!");
00259 
00260     // Assemble the parts into intermediate operands.
00261     SmallVector<SDValue, 8> Ops(NumIntermediates);
00262     if (NumIntermediates == NumParts) {
00263       // If the register was not expanded, truncate or copy the value,
00264       // as appropriate.
00265       for (unsigned i = 0; i != NumParts; ++i)
00266         Ops[i] = getCopyFromParts(DAG, DL, &Parts[i], 1,
00267                                   PartVT, IntermediateVT, V);
00268     } else if (NumParts > 0) {
00269       // If the intermediate type was expanded, build the intermediate
00270       // operands from the parts.
00271       assert(NumParts % NumIntermediates == 0 &&
00272              "Must expand into a divisible number of parts!");
00273       unsigned Factor = NumParts / NumIntermediates;
00274       for (unsigned i = 0; i != NumIntermediates; ++i)
00275         Ops[i] = getCopyFromParts(DAG, DL, &Parts[i * Factor], Factor,
00276                                   PartVT, IntermediateVT, V);
00277     }
00278 
00279     // Build a vector with BUILD_VECTOR or CONCAT_VECTORS from the
00280     // intermediate operands.
00281     Val = DAG.getNode(IntermediateVT.isVector() ? ISD::CONCAT_VECTORS
00282                                                 : ISD::BUILD_VECTOR,
00283                       DL, ValueVT, Ops);
00284   }
00285 
00286   // There is now one part, held in Val.  Correct it to match ValueVT.
00287   EVT PartEVT = Val.getValueType();
00288 
00289   if (PartEVT == ValueVT)
00290     return Val;
00291 
00292   if (PartEVT.isVector()) {
00293     // If the element type of the source/dest vectors are the same, but the
00294     // parts vector has more elements than the value vector, then we have a
00295     // vector widening case (e.g. <2 x float> -> <4 x float>).  Extract the
00296     // elements we want.
00297     if (PartEVT.getVectorElementType() == ValueVT.getVectorElementType()) {
00298       assert(PartEVT.getVectorNumElements() > ValueVT.getVectorNumElements() &&
00299              "Cannot narrow, it would be a lossy transformation");
00300       return DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, ValueVT, Val,
00301                          DAG.getConstant(0, TLI.getVectorIdxTy()));
00302     }
00303 
00304     // Vector/Vector bitcast.
00305     if (ValueVT.getSizeInBits() == PartEVT.getSizeInBits())
00306       return DAG.getNode(ISD::BITCAST, DL, ValueVT, Val);
00307 
00308     assert(PartEVT.getVectorNumElements() == ValueVT.getVectorNumElements() &&
00309       "Cannot handle this kind of promotion");
00310     // Promoted vector extract
00311     bool Smaller = ValueVT.bitsLE(PartEVT);
00312     return DAG.getNode((Smaller ? ISD::TRUNCATE : ISD::ANY_EXTEND),
00313                        DL, ValueVT, Val);
00314 
00315   }
00316 
00317   // Trivial bitcast if the types are the same size and the destination
00318   // vector type is legal.
00319   if (PartEVT.getSizeInBits() == ValueVT.getSizeInBits() &&
00320       TLI.isTypeLegal(ValueVT))
00321     return DAG.getNode(ISD::BITCAST, DL, ValueVT, Val);
00322 
00323   // Handle cases such as i8 -> <1 x i1>
00324   if (ValueVT.getVectorNumElements() != 1) {
00325     diagnosePossiblyInvalidConstraint(*DAG.getContext(), V,
00326                                       "non-trivial scalar-to-vector conversion");
00327     return DAG.getUNDEF(ValueVT);
00328   }
00329 
00330   if (ValueVT.getVectorNumElements() == 1 &&
00331       ValueVT.getVectorElementType() != PartEVT) {
00332     bool Smaller = ValueVT.bitsLE(PartEVT);
00333     Val = DAG.getNode((Smaller ? ISD::TRUNCATE : ISD::ANY_EXTEND),
00334                        DL, ValueVT.getScalarType(), Val);
00335   }
00336 
00337   return DAG.getNode(ISD::BUILD_VECTOR, DL, ValueVT, Val);
00338 }
00339 
00340 static void getCopyToPartsVector(SelectionDAG &DAG, SDLoc dl,
00341                                  SDValue Val, SDValue *Parts, unsigned NumParts,
00342                                  MVT PartVT, const Value *V);
00343 
00344 /// getCopyToParts - Create a series of nodes that contain the specified value
00345 /// split into legal parts.  If the parts contain more bits than Val, then, for
00346 /// integers, ExtendKind can be used to specify how to generate the extra bits.
00347 static void getCopyToParts(SelectionDAG &DAG, SDLoc DL,
00348                            SDValue Val, SDValue *Parts, unsigned NumParts,
00349                            MVT PartVT, const Value *V,
00350                            ISD::NodeType ExtendKind = ISD::ANY_EXTEND) {
00351   EVT ValueVT = Val.getValueType();
00352 
00353   // Handle the vector case separately.
00354   if (ValueVT.isVector())
00355     return getCopyToPartsVector(DAG, DL, Val, Parts, NumParts, PartVT, V);
00356 
00357   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
00358   unsigned PartBits = PartVT.getSizeInBits();
00359   unsigned OrigNumParts = NumParts;
00360   assert(TLI.isTypeLegal(PartVT) && "Copying to an illegal type!");
00361 
00362   if (NumParts == 0)
00363     return;
00364 
00365   assert(!ValueVT.isVector() && "Vector case handled elsewhere");
00366   EVT PartEVT = PartVT;
00367   if (PartEVT == ValueVT) {
00368     assert(NumParts == 1 && "No-op copy with multiple parts!");
00369     Parts[0] = Val;
00370     return;
00371   }
00372 
00373   if (NumParts * PartBits > ValueVT.getSizeInBits()) {
00374     // If the parts cover more bits than the value has, promote the value.
00375     if (PartVT.isFloatingPoint() && ValueVT.isFloatingPoint()) {
00376       assert(NumParts == 1 && "Do not know what to promote to!");
00377       Val = DAG.getNode(ISD::FP_EXTEND, DL, PartVT, Val);
00378     } else {
00379       assert((PartVT.isInteger() || PartVT == MVT::x86mmx) &&
00380              ValueVT.isInteger() &&
00381              "Unknown mismatch!");
00382       ValueVT = EVT::getIntegerVT(*DAG.getContext(), NumParts * PartBits);
00383       Val = DAG.getNode(ExtendKind, DL, ValueVT, Val);
00384       if (PartVT == MVT::x86mmx)
00385         Val = DAG.getNode(ISD::BITCAST, DL, PartVT, Val);
00386     }
00387   } else if (PartBits == ValueVT.getSizeInBits()) {
00388     // Different types of the same size.
00389     assert(NumParts == 1 && PartEVT != ValueVT);
00390     Val = DAG.getNode(ISD::BITCAST, DL, PartVT, Val);
00391   } else if (NumParts * PartBits < ValueVT.getSizeInBits()) {
00392     // If the parts cover less bits than value has, truncate the value.
00393     assert((PartVT.isInteger() || PartVT == MVT::x86mmx) &&
00394            ValueVT.isInteger() &&
00395            "Unknown mismatch!");
00396     ValueVT = EVT::getIntegerVT(*DAG.getContext(), NumParts * PartBits);
00397     Val = DAG.getNode(ISD::TRUNCATE, DL, ValueVT, Val);
00398     if (PartVT == MVT::x86mmx)
00399       Val = DAG.getNode(ISD::BITCAST, DL, PartVT, Val);
00400   }
00401 
00402   // The value may have changed - recompute ValueVT.
00403   ValueVT = Val.getValueType();
00404   assert(NumParts * PartBits == ValueVT.getSizeInBits() &&
00405          "Failed to tile the value with PartVT!");
00406 
00407   if (NumParts == 1) {
00408     if (PartEVT != ValueVT)
00409       diagnosePossiblyInvalidConstraint(*DAG.getContext(), V,
00410                                         "scalar-to-vector conversion failed");
00411 
00412     Parts[0] = Val;
00413     return;
00414   }
00415 
00416   // Expand the value into multiple parts.
00417   if (NumParts & (NumParts - 1)) {
00418     // The number of parts is not a power of 2.  Split off and copy the tail.
00419     assert(PartVT.isInteger() && ValueVT.isInteger() &&
00420            "Do not know what to expand to!");
00421     unsigned RoundParts = 1 << Log2_32(NumParts);
00422     unsigned RoundBits = RoundParts * PartBits;
00423     unsigned OddParts = NumParts - RoundParts;
00424     SDValue OddVal = DAG.getNode(ISD::SRL, DL, ValueVT, Val,
00425                                  DAG.getIntPtrConstant(RoundBits));
00426     getCopyToParts(DAG, DL, OddVal, Parts + RoundParts, OddParts, PartVT, V);
00427 
00428     if (TLI.isBigEndian())
00429       // The odd parts were reversed by getCopyToParts - unreverse them.
00430       std::reverse(Parts + RoundParts, Parts + NumParts);
00431 
00432     NumParts = RoundParts;
00433     ValueVT = EVT::getIntegerVT(*DAG.getContext(), NumParts * PartBits);
00434     Val = DAG.getNode(ISD::TRUNCATE, DL, ValueVT, Val);
00435   }
00436 
00437   // The number of parts is a power of 2.  Repeatedly bisect the value using
00438   // EXTRACT_ELEMENT.
00439   Parts[0] = DAG.getNode(ISD::BITCAST, DL,
00440                          EVT::getIntegerVT(*DAG.getContext(),
00441                                            ValueVT.getSizeInBits()),
00442                          Val);
00443 
00444   for (unsigned StepSize = NumParts; StepSize > 1; StepSize /= 2) {
00445     for (unsigned i = 0; i < NumParts; i += StepSize) {
00446       unsigned ThisBits = StepSize * PartBits / 2;
00447       EVT ThisVT = EVT::getIntegerVT(*DAG.getContext(), ThisBits);
00448       SDValue &Part0 = Parts[i];
00449       SDValue &Part1 = Parts[i+StepSize/2];
00450 
00451       Part1 = DAG.getNode(ISD::EXTRACT_ELEMENT, DL,
00452                           ThisVT, Part0, DAG.getIntPtrConstant(1));
00453       Part0 = DAG.getNode(ISD::EXTRACT_ELEMENT, DL,
00454                           ThisVT, Part0, DAG.getIntPtrConstant(0));
00455 
00456       if (ThisBits == PartBits && ThisVT != PartVT) {
00457         Part0 = DAG.getNode(ISD::BITCAST, DL, PartVT, Part0);
00458         Part1 = DAG.getNode(ISD::BITCAST, DL, PartVT, Part1);
00459       }
00460     }
00461   }
00462 
00463   if (TLI.isBigEndian())
00464     std::reverse(Parts, Parts + OrigNumParts);
00465 }
00466 
00467 
00468 /// getCopyToPartsVector - Create a series of nodes that contain the specified
00469 /// value split into legal parts.
00470 static void getCopyToPartsVector(SelectionDAG &DAG, SDLoc DL,
00471                                  SDValue Val, SDValue *Parts, unsigned NumParts,
00472                                  MVT PartVT, const Value *V) {
00473   EVT ValueVT = Val.getValueType();
00474   assert(ValueVT.isVector() && "Not a vector");
00475   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
00476 
00477   if (NumParts == 1) {
00478     EVT PartEVT = PartVT;
00479     if (PartEVT == ValueVT) {
00480       // Nothing to do.
00481     } else if (PartVT.getSizeInBits() == ValueVT.getSizeInBits()) {
00482       // Bitconvert vector->vector case.
00483       Val = DAG.getNode(ISD::BITCAST, DL, PartVT, Val);
00484     } else if (PartVT.isVector() &&
00485                PartEVT.getVectorElementType() == ValueVT.getVectorElementType() &&
00486                PartEVT.getVectorNumElements() > ValueVT.getVectorNumElements()) {
00487       EVT ElementVT = PartVT.getVectorElementType();
00488       // Vector widening case, e.g. <2 x float> -> <4 x float>.  Shuffle in
00489       // undef elements.
00490       SmallVector<SDValue, 16> Ops;
00491       for (unsigned i = 0, e = ValueVT.getVectorNumElements(); i != e; ++i)
00492         Ops.push_back(DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL,
00493                                   ElementVT, Val, DAG.getConstant(i,
00494                                                   TLI.getVectorIdxTy())));
00495 
00496       for (unsigned i = ValueVT.getVectorNumElements(),
00497            e = PartVT.getVectorNumElements(); i != e; ++i)
00498         Ops.push_back(DAG.getUNDEF(ElementVT));
00499 
00500       Val = DAG.getNode(ISD::BUILD_VECTOR, DL, PartVT, Ops);
00501 
00502       // FIXME: Use CONCAT for 2x -> 4x.
00503 
00504       //SDValue UndefElts = DAG.getUNDEF(VectorTy);
00505       //Val = DAG.getNode(ISD::CONCAT_VECTORS, DL, PartVT, Val, UndefElts);
00506     } else if (PartVT.isVector() &&
00507                PartEVT.getVectorElementType().bitsGE(
00508                  ValueVT.getVectorElementType()) &&
00509                PartEVT.getVectorNumElements() == ValueVT.getVectorNumElements()) {
00510 
00511       // Promoted vector extract
00512       bool Smaller = PartEVT.bitsLE(ValueVT);
00513       Val = DAG.getNode((Smaller ? ISD::TRUNCATE : ISD::ANY_EXTEND),
00514                         DL, PartVT, Val);
00515     } else{
00516       // Vector -> scalar conversion.
00517       assert(ValueVT.getVectorNumElements() == 1 &&
00518              "Only trivial vector-to-scalar conversions should get here!");
00519       Val = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL,
00520                         PartVT, Val, DAG.getConstant(0, TLI.getVectorIdxTy()));
00521 
00522       bool Smaller = ValueVT.bitsLE(PartVT);
00523       Val = DAG.getNode((Smaller ? ISD::TRUNCATE : ISD::ANY_EXTEND),
00524                          DL, PartVT, Val);
00525     }
00526 
00527     Parts[0] = Val;
00528     return;
00529   }
00530 
00531   // Handle a multi-element vector.
00532   EVT IntermediateVT;
00533   MVT RegisterVT;
00534   unsigned NumIntermediates;
00535   unsigned NumRegs = TLI.getVectorTypeBreakdown(*DAG.getContext(), ValueVT,
00536                                                 IntermediateVT,
00537                                                 NumIntermediates, RegisterVT);
00538   unsigned NumElements = ValueVT.getVectorNumElements();
00539 
00540   assert(NumRegs == NumParts && "Part count doesn't match vector breakdown!");
00541   NumParts = NumRegs; // Silence a compiler warning.
00542   assert(RegisterVT == PartVT && "Part type doesn't match vector breakdown!");
00543 
00544   // Split the vector into intermediate operands.
00545   SmallVector<SDValue, 8> Ops(NumIntermediates);
00546   for (unsigned i = 0; i != NumIntermediates; ++i) {
00547     if (IntermediateVT.isVector())
00548       Ops[i] = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL,
00549                            IntermediateVT, Val,
00550                    DAG.getConstant(i * (NumElements / NumIntermediates),
00551                                    TLI.getVectorIdxTy()));
00552     else
00553       Ops[i] = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL,
00554                            IntermediateVT, Val,
00555                            DAG.getConstant(i, TLI.getVectorIdxTy()));
00556   }
00557 
00558   // Split the intermediate operands into legal parts.
00559   if (NumParts == NumIntermediates) {
00560     // If the register was not expanded, promote or copy the value,
00561     // as appropriate.
00562     for (unsigned i = 0; i != NumParts; ++i)
00563       getCopyToParts(DAG, DL, Ops[i], &Parts[i], 1, PartVT, V);
00564   } else if (NumParts > 0) {
00565     // If the intermediate type was expanded, split each the value into
00566     // legal parts.
00567     assert(NumParts % NumIntermediates == 0 &&
00568            "Must expand into a divisible number of parts!");
00569     unsigned Factor = NumParts / NumIntermediates;
00570     for (unsigned i = 0; i != NumIntermediates; ++i)
00571       getCopyToParts(DAG, DL, Ops[i], &Parts[i*Factor], Factor, PartVT, V);
00572   }
00573 }
00574 
00575 namespace {
00576   /// RegsForValue - This struct represents the registers (physical or virtual)
00577   /// that a particular set of values is assigned, and the type information
00578   /// about the value. The most common situation is to represent one value at a
00579   /// time, but struct or array values are handled element-wise as multiple
00580   /// values.  The splitting of aggregates is performed recursively, so that we
00581   /// never have aggregate-typed registers. The values at this point do not
00582   /// necessarily have legal types, so each value may require one or more
00583   /// registers of some legal type.
00584   ///
00585   struct RegsForValue {
00586     /// ValueVTs - The value types of the values, which may not be legal, and
00587     /// may need be promoted or synthesized from one or more registers.
00588     ///
00589     SmallVector<EVT, 4> ValueVTs;
00590 
00591     /// RegVTs - The value types of the registers. This is the same size as
00592     /// ValueVTs and it records, for each value, what the type of the assigned
00593     /// register or registers are. (Individual values are never synthesized
00594     /// from more than one type of register.)
00595     ///
00596     /// With virtual registers, the contents of RegVTs is redundant with TLI's
00597     /// getRegisterType member function, however when with physical registers
00598     /// it is necessary to have a separate record of the types.
00599     ///
00600     SmallVector<MVT, 4> RegVTs;
00601 
00602     /// Regs - This list holds the registers assigned to the values.
00603     /// Each legal or promoted value requires one register, and each
00604     /// expanded value requires multiple registers.
00605     ///
00606     SmallVector<unsigned, 4> Regs;
00607 
00608     RegsForValue() {}
00609 
00610     RegsForValue(const SmallVector<unsigned, 4> &regs,
00611                  MVT regvt, EVT valuevt)
00612       : ValueVTs(1, valuevt), RegVTs(1, regvt), Regs(regs) {}
00613 
00614     RegsForValue(LLVMContext &Context, const TargetLowering &tli,
00615                  unsigned Reg, Type *Ty) {
00616       ComputeValueVTs(tli, Ty, ValueVTs);
00617 
00618       for (unsigned Value = 0, e = ValueVTs.size(); Value != e; ++Value) {
00619         EVT ValueVT = ValueVTs[Value];
00620         unsigned NumRegs = tli.getNumRegisters(Context, ValueVT);
00621         MVT RegisterVT = tli.getRegisterType(Context, ValueVT);
00622         for (unsigned i = 0; i != NumRegs; ++i)
00623           Regs.push_back(Reg + i);
00624         RegVTs.push_back(RegisterVT);
00625         Reg += NumRegs;
00626       }
00627     }
00628 
00629     /// append - Add the specified values to this one.
00630     void append(const RegsForValue &RHS) {
00631       ValueVTs.append(RHS.ValueVTs.begin(), RHS.ValueVTs.end());
00632       RegVTs.append(RHS.RegVTs.begin(), RHS.RegVTs.end());
00633       Regs.append(RHS.Regs.begin(), RHS.Regs.end());
00634     }
00635 
00636     /// getCopyFromRegs - Emit a series of CopyFromReg nodes that copies from
00637     /// this value and returns the result as a ValueVTs value.  This uses
00638     /// Chain/Flag as the input and updates them for the output Chain/Flag.
00639     /// If the Flag pointer is NULL, no flag is used.
00640     SDValue getCopyFromRegs(SelectionDAG &DAG, FunctionLoweringInfo &FuncInfo,
00641                             SDLoc dl,
00642                             SDValue &Chain, SDValue *Flag,
00643                             const Value *V = nullptr) const;
00644 
00645     /// getCopyToRegs - Emit a series of CopyToReg nodes that copies the
00646     /// specified value into the registers specified by this object.  This uses
00647     /// Chain/Flag as the input and updates them for the output Chain/Flag.
00648     /// If the Flag pointer is NULL, no flag is used.
00649     void
00650     getCopyToRegs(SDValue Val, SelectionDAG &DAG, SDLoc dl, SDValue &Chain,
00651                   SDValue *Flag, const Value *V,
00652                   ISD::NodeType PreferredExtendType = ISD::ANY_EXTEND) const;
00653 
00654     /// AddInlineAsmOperands - Add this value to the specified inlineasm node
00655     /// operand list.  This adds the code marker, matching input operand index
00656     /// (if applicable), and includes the number of values added into it.
00657     void AddInlineAsmOperands(unsigned Kind,
00658                               bool HasMatching, unsigned MatchingIdx,
00659                               SelectionDAG &DAG,
00660                               std::vector<SDValue> &Ops) const;
00661   };
00662 }
00663 
00664 /// getCopyFromRegs - Emit a series of CopyFromReg nodes that copies from
00665 /// this value and returns the result as a ValueVT value.  This uses
00666 /// Chain/Flag as the input and updates them for the output Chain/Flag.
00667 /// If the Flag pointer is NULL, no flag is used.
00668 SDValue RegsForValue::getCopyFromRegs(SelectionDAG &DAG,
00669                                       FunctionLoweringInfo &FuncInfo,
00670                                       SDLoc dl,
00671                                       SDValue &Chain, SDValue *Flag,
00672                                       const Value *V) const {
00673   // A Value with type {} or [0 x %t] needs no registers.
00674   if (ValueVTs.empty())
00675     return SDValue();
00676 
00677   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
00678 
00679   // Assemble the legal parts into the final values.
00680   SmallVector<SDValue, 4> Values(ValueVTs.size());
00681   SmallVector<SDValue, 8> Parts;
00682   for (unsigned Value = 0, Part = 0, e = ValueVTs.size(); Value != e; ++Value) {
00683     // Copy the legal parts from the registers.
00684     EVT ValueVT = ValueVTs[Value];
00685     unsigned NumRegs = TLI.getNumRegisters(*DAG.getContext(), ValueVT);
00686     MVT RegisterVT = RegVTs[Value];
00687 
00688     Parts.resize(NumRegs);
00689     for (unsigned i = 0; i != NumRegs; ++i) {
00690       SDValue P;
00691       if (!Flag) {
00692         P = DAG.getCopyFromReg(Chain, dl, Regs[Part+i], RegisterVT);
00693       } else {
00694         P = DAG.getCopyFromReg(Chain, dl, Regs[Part+i], RegisterVT, *Flag);
00695         *Flag = P.getValue(2);
00696       }
00697 
00698       Chain = P.getValue(1);
00699       Parts[i] = P;
00700 
00701       // If the source register was virtual and if we know something about it,
00702       // add an assert node.
00703       if (!TargetRegisterInfo::isVirtualRegister(Regs[Part+i]) ||
00704           !RegisterVT.isInteger() || RegisterVT.isVector())
00705         continue;
00706 
00707       const FunctionLoweringInfo::LiveOutInfo *LOI =
00708         FuncInfo.GetLiveOutRegInfo(Regs[Part+i]);
00709       if (!LOI)
00710         continue;
00711 
00712       unsigned RegSize = RegisterVT.getSizeInBits();
00713       unsigned NumSignBits = LOI->NumSignBits;
00714       unsigned NumZeroBits = LOI->KnownZero.countLeadingOnes();
00715 
00716       if (NumZeroBits == RegSize) {
00717         // The current value is a zero.
00718         // Explicitly express that as it would be easier for
00719         // optimizations to kick in.
00720         Parts[i] = DAG.getConstant(0, RegisterVT);
00721         continue;
00722       }
00723 
00724       // FIXME: We capture more information than the dag can represent.  For
00725       // now, just use the tightest assertzext/assertsext possible.
00726       bool isSExt = true;
00727       EVT FromVT(MVT::Other);
00728       if (NumSignBits == RegSize)
00729         isSExt = true, FromVT = MVT::i1;   // ASSERT SEXT 1
00730       else if (NumZeroBits >= RegSize-1)
00731         isSExt = false, FromVT = MVT::i1;  // ASSERT ZEXT 1
00732       else if (NumSignBits > RegSize-8)
00733         isSExt = true, FromVT = MVT::i8;   // ASSERT SEXT 8
00734       else if (NumZeroBits >= RegSize-8)
00735         isSExt = false, FromVT = MVT::i8;  // ASSERT ZEXT 8
00736       else if (NumSignBits > RegSize-16)
00737         isSExt = true, FromVT = MVT::i16;  // ASSERT SEXT 16
00738       else if (NumZeroBits >= RegSize-16)
00739         isSExt = false, FromVT = MVT::i16; // ASSERT ZEXT 16
00740       else if (NumSignBits > RegSize-32)
00741         isSExt = true, FromVT = MVT::i32;  // ASSERT SEXT 32
00742       else if (NumZeroBits >= RegSize-32)
00743         isSExt = false, FromVT = MVT::i32; // ASSERT ZEXT 32
00744       else
00745         continue;
00746 
00747       // Add an assertion node.
00748       assert(FromVT != MVT::Other);
00749       Parts[i] = DAG.getNode(isSExt ? ISD::AssertSext : ISD::AssertZext, dl,
00750                              RegisterVT, P, DAG.getValueType(FromVT));
00751     }
00752 
00753     Values[Value] = getCopyFromParts(DAG, dl, Parts.begin(),
00754                                      NumRegs, RegisterVT, ValueVT, V);
00755     Part += NumRegs;
00756     Parts.clear();
00757   }
00758 
00759   return DAG.getNode(ISD::MERGE_VALUES, dl, DAG.getVTList(ValueVTs), Values);
00760 }
00761 
00762 /// getCopyToRegs - Emit a series of CopyToReg nodes that copies the
00763 /// specified value into the registers specified by this object.  This uses
00764 /// Chain/Flag as the input and updates them for the output Chain/Flag.
00765 /// If the Flag pointer is NULL, no flag is used.
00766 void RegsForValue::getCopyToRegs(SDValue Val, SelectionDAG &DAG, SDLoc dl,
00767                                  SDValue &Chain, SDValue *Flag, const Value *V,
00768                                  ISD::NodeType PreferredExtendType) const {
00769   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
00770   ISD::NodeType ExtendKind = PreferredExtendType;
00771 
00772   // Get the list of the values's legal parts.
00773   unsigned NumRegs = Regs.size();
00774   SmallVector<SDValue, 8> Parts(NumRegs);
00775   for (unsigned Value = 0, Part = 0, e = ValueVTs.size(); Value != e; ++Value) {
00776     EVT ValueVT = ValueVTs[Value];
00777     unsigned NumParts = TLI.getNumRegisters(*DAG.getContext(), ValueVT);
00778     MVT RegisterVT = RegVTs[Value];
00779 
00780     if (ExtendKind == ISD::ANY_EXTEND && TLI.isZExtFree(Val, RegisterVT))
00781       ExtendKind = ISD::ZERO_EXTEND;
00782 
00783     getCopyToParts(DAG, dl, Val.getValue(Val.getResNo() + Value),
00784                    &Parts[Part], NumParts, RegisterVT, V, ExtendKind);
00785     Part += NumParts;
00786   }
00787 
00788   // Copy the parts into the registers.
00789   SmallVector<SDValue, 8> Chains(NumRegs);
00790   for (unsigned i = 0; i != NumRegs; ++i) {
00791     SDValue Part;
00792     if (!Flag) {
00793       Part = DAG.getCopyToReg(Chain, dl, Regs[i], Parts[i]);
00794     } else {
00795       Part = DAG.getCopyToReg(Chain, dl, Regs[i], Parts[i], *Flag);
00796       *Flag = Part.getValue(1);
00797     }
00798 
00799     Chains[i] = Part.getValue(0);
00800   }
00801 
00802   if (NumRegs == 1 || Flag)
00803     // If NumRegs > 1 && Flag is used then the use of the last CopyToReg is
00804     // flagged to it. That is the CopyToReg nodes and the user are considered
00805     // a single scheduling unit. If we create a TokenFactor and return it as
00806     // chain, then the TokenFactor is both a predecessor (operand) of the
00807     // user as well as a successor (the TF operands are flagged to the user).
00808     // c1, f1 = CopyToReg
00809     // c2, f2 = CopyToReg
00810     // c3     = TokenFactor c1, c2
00811     // ...
00812     //        = op c3, ..., f2
00813     Chain = Chains[NumRegs-1];
00814   else
00815     Chain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Chains);
00816 }
00817 
00818 /// AddInlineAsmOperands - Add this value to the specified inlineasm node
00819 /// operand list.  This adds the code marker and includes the number of
00820 /// values added into it.
00821 void RegsForValue::AddInlineAsmOperands(unsigned Code, bool HasMatching,
00822                                         unsigned MatchingIdx,
00823                                         SelectionDAG &DAG,
00824                                         std::vector<SDValue> &Ops) const {
00825   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
00826 
00827   unsigned Flag = InlineAsm::getFlagWord(Code, Regs.size());
00828   if (HasMatching)
00829     Flag = InlineAsm::getFlagWordForMatchingOp(Flag, MatchingIdx);
00830   else if (!Regs.empty() &&
00831            TargetRegisterInfo::isVirtualRegister(Regs.front())) {
00832     // Put the register class of the virtual registers in the flag word.  That
00833     // way, later passes can recompute register class constraints for inline
00834     // assembly as well as normal instructions.
00835     // Don't do this for tied operands that can use the regclass information
00836     // from the def.
00837     const MachineRegisterInfo &MRI = DAG.getMachineFunction().getRegInfo();
00838     const TargetRegisterClass *RC = MRI.getRegClass(Regs.front());
00839     Flag = InlineAsm::getFlagWordForRegClass(Flag, RC->getID());
00840   }
00841 
00842   SDValue Res = DAG.getTargetConstant(Flag, MVT::i32);
00843   Ops.push_back(Res);
00844 
00845   unsigned SP = TLI.getStackPointerRegisterToSaveRestore();
00846   for (unsigned Value = 0, Reg = 0, e = ValueVTs.size(); Value != e; ++Value) {
00847     unsigned NumRegs = TLI.getNumRegisters(*DAG.getContext(), ValueVTs[Value]);
00848     MVT RegisterVT = RegVTs[Value];
00849     for (unsigned i = 0; i != NumRegs; ++i) {
00850       assert(Reg < Regs.size() && "Mismatch in # registers expected");
00851       unsigned TheReg = Regs[Reg++];
00852       Ops.push_back(DAG.getRegister(TheReg, RegisterVT));
00853 
00854       if (TheReg == SP && Code == InlineAsm::Kind_Clobber) {
00855         // If we clobbered the stack pointer, MFI should know about it.
00856         assert(DAG.getMachineFunction().getFrameInfo()->
00857             hasInlineAsmWithSPAdjust());
00858       }
00859     }
00860   }
00861 }
00862 
00863 void SelectionDAGBuilder::init(GCFunctionInfo *gfi, AliasAnalysis &aa,
00864                                const TargetLibraryInfo *li) {
00865   AA = &aa;
00866   GFI = gfi;
00867   LibInfo = li;
00868   DL = DAG.getSubtarget().getDataLayout();
00869   Context = DAG.getContext();
00870   LPadToCallSiteMap.clear();
00871 }
00872 
00873 /// clear - Clear out the current SelectionDAG and the associated
00874 /// state and prepare this SelectionDAGBuilder object to be used
00875 /// for a new block. This doesn't clear out information about
00876 /// additional blocks that are needed to complete switch lowering
00877 /// or PHI node updating; that information is cleared out as it is
00878 /// consumed.
00879 void SelectionDAGBuilder::clear() {
00880   NodeMap.clear();
00881   UnusedArgNodeMap.clear();
00882   PendingLoads.clear();
00883   PendingExports.clear();
00884   CurInst = nullptr;
00885   HasTailCall = false;
00886   SDNodeOrder = LowestSDNodeOrder;
00887 }
00888 
00889 /// clearDanglingDebugInfo - Clear the dangling debug information
00890 /// map. This function is separated from the clear so that debug
00891 /// information that is dangling in a basic block can be properly
00892 /// resolved in a different basic block. This allows the
00893 /// SelectionDAG to resolve dangling debug information attached
00894 /// to PHI nodes.
00895 void SelectionDAGBuilder::clearDanglingDebugInfo() {
00896   DanglingDebugInfoMap.clear();
00897 }
00898 
00899 /// getRoot - Return the current virtual root of the Selection DAG,
00900 /// flushing any PendingLoad items. This must be done before emitting
00901 /// a store or any other node that may need to be ordered after any
00902 /// prior load instructions.
00903 ///
00904 SDValue SelectionDAGBuilder::getRoot() {
00905   if (PendingLoads.empty())
00906     return DAG.getRoot();
00907 
00908   if (PendingLoads.size() == 1) {
00909     SDValue Root = PendingLoads[0];
00910     DAG.setRoot(Root);
00911     PendingLoads.clear();
00912     return Root;
00913   }
00914 
00915   // Otherwise, we have to make a token factor node.
00916   SDValue Root = DAG.getNode(ISD::TokenFactor, getCurSDLoc(), MVT::Other,
00917                              PendingLoads);
00918   PendingLoads.clear();
00919   DAG.setRoot(Root);
00920   return Root;
00921 }
00922 
00923 /// getControlRoot - Similar to getRoot, but instead of flushing all the
00924 /// PendingLoad items, flush all the PendingExports items. It is necessary
00925 /// to do this before emitting a terminator instruction.
00926 ///
00927 SDValue SelectionDAGBuilder::getControlRoot() {
00928   SDValue Root = DAG.getRoot();
00929 
00930   if (PendingExports.empty())
00931     return Root;
00932 
00933   // Turn all of the CopyToReg chains into one factored node.
00934   if (Root.getOpcode() != ISD::EntryToken) {
00935     unsigned i = 0, e = PendingExports.size();
00936     for (; i != e; ++i) {
00937       assert(PendingExports[i].getNode()->getNumOperands() > 1);
00938       if (PendingExports[i].getNode()->getOperand(0) == Root)
00939         break;  // Don't add the root if we already indirectly depend on it.
00940     }
00941 
00942     if (i == e)
00943       PendingExports.push_back(Root);
00944   }
00945 
00946   Root = DAG.getNode(ISD::TokenFactor, getCurSDLoc(), MVT::Other,
00947                      PendingExports);
00948   PendingExports.clear();
00949   DAG.setRoot(Root);
00950   return Root;
00951 }
00952 
00953 void SelectionDAGBuilder::visit(const Instruction &I) {
00954   // Set up outgoing PHI node register values before emitting the terminator.
00955   if (isa<TerminatorInst>(&I))
00956     HandlePHINodesInSuccessorBlocks(I.getParent());
00957 
00958   ++SDNodeOrder;
00959 
00960   CurInst = &I;
00961 
00962   visit(I.getOpcode(), I);
00963 
00964   if (!isa<TerminatorInst>(&I) && !HasTailCall)
00965     CopyToExportRegsIfNeeded(&I);
00966 
00967   CurInst = nullptr;
00968 }
00969 
00970 void SelectionDAGBuilder::visitPHI(const PHINode &) {
00971   llvm_unreachable("SelectionDAGBuilder shouldn't visit PHI nodes!");
00972 }
00973 
00974 void SelectionDAGBuilder::visit(unsigned Opcode, const User &I) {
00975   // Note: this doesn't use InstVisitor, because it has to work with
00976   // ConstantExpr's in addition to instructions.
00977   switch (Opcode) {
00978   default: llvm_unreachable("Unknown instruction type encountered!");
00979     // Build the switch statement using the Instruction.def file.
00980 #define HANDLE_INST(NUM, OPCODE, CLASS) \
00981     case Instruction::OPCODE: visit##OPCODE((const CLASS&)I); break;
00982 #include "llvm/IR/Instruction.def"
00983   }
00984 }
00985 
00986 // resolveDanglingDebugInfo - if we saw an earlier dbg_value referring to V,
00987 // generate the debug data structures now that we've seen its definition.
00988 void SelectionDAGBuilder::resolveDanglingDebugInfo(const Value *V,
00989                                                    SDValue Val) {
00990   DanglingDebugInfo &DDI = DanglingDebugInfoMap[V];
00991   if (DDI.getDI()) {
00992     const DbgValueInst *DI = DDI.getDI();
00993     DebugLoc dl = DDI.getdl();
00994     unsigned DbgSDNodeOrder = DDI.getSDNodeOrder();
00995     MDNode *Variable = DI->getVariable();
00996     uint64_t Offset = DI->getOffset();
00997     // A dbg.value for an alloca is always indirect.
00998     bool IsIndirect = isa<AllocaInst>(V) || Offset != 0;
00999     SDDbgValue *SDV;
01000     if (Val.getNode()) {
01001       if (!EmitFuncArgumentDbgValue(V, Variable, Offset, IsIndirect, Val)) {
01002         SDV = DAG.getDbgValue(Variable, Val.getNode(),
01003                               Val.getResNo(), IsIndirect,
01004             Offset, dl, DbgSDNodeOrder);
01005         DAG.AddDbgValue(SDV, Val.getNode(), false);
01006       }
01007     } else
01008       DEBUG(dbgs() << "Dropping debug info for " << *DI << "\n");
01009     DanglingDebugInfoMap[V] = DanglingDebugInfo();
01010   }
01011 }
01012 
01013 /// getValue - Return an SDValue for the given Value.
01014 SDValue SelectionDAGBuilder::getValue(const Value *V) {
01015   // If we already have an SDValue for this value, use it. It's important
01016   // to do this first, so that we don't create a CopyFromReg if we already
01017   // have a regular SDValue.
01018   SDValue &N = NodeMap[V];
01019   if (N.getNode()) return N;
01020 
01021   // If there's a virtual register allocated and initialized for this
01022   // value, use it.
01023   DenseMap<const Value *, unsigned>::iterator It = FuncInfo.ValueMap.find(V);
01024   if (It != FuncInfo.ValueMap.end()) {
01025     unsigned InReg = It->second;
01026     RegsForValue RFV(*DAG.getContext(),
01027                      *TM.getSubtargetImpl()->getTargetLowering(), InReg,
01028                      V->getType());
01029     SDValue Chain = DAG.getEntryNode();
01030     N = RFV.getCopyFromRegs(DAG, FuncInfo, getCurSDLoc(), Chain, nullptr, V);
01031     resolveDanglingDebugInfo(V, N);
01032     return N;
01033   }
01034 
01035   // Otherwise create a new SDValue and remember it.
01036   SDValue Val = getValueImpl(V);
01037   NodeMap[V] = Val;
01038   resolveDanglingDebugInfo(V, Val);
01039   return Val;
01040 }
01041 
01042 /// getNonRegisterValue - Return an SDValue for the given Value, but
01043 /// don't look in FuncInfo.ValueMap for a virtual register.
01044 SDValue SelectionDAGBuilder::getNonRegisterValue(const Value *V) {
01045   // If we already have an SDValue for this value, use it.
01046   SDValue &N = NodeMap[V];
01047   if (N.getNode()) return N;
01048 
01049   // Otherwise create a new SDValue and remember it.
01050   SDValue Val = getValueImpl(V);
01051   NodeMap[V] = Val;
01052   resolveDanglingDebugInfo(V, Val);
01053   return Val;
01054 }
01055 
01056 /// getValueImpl - Helper function for getValue and getNonRegisterValue.
01057 /// Create an SDValue for the given value.
01058 SDValue SelectionDAGBuilder::getValueImpl(const Value *V) {
01059   const TargetLowering *TLI = TM.getSubtargetImpl()->getTargetLowering();
01060 
01061   if (const Constant *C = dyn_cast<Constant>(V)) {
01062     EVT VT = TLI->getValueType(V->getType(), true);
01063 
01064     if (const ConstantInt *CI = dyn_cast<ConstantInt>(C))
01065       return DAG.getConstant(*CI, VT);
01066 
01067     if (const GlobalValue *GV = dyn_cast<GlobalValue>(C))
01068       return DAG.getGlobalAddress(GV, getCurSDLoc(), VT);
01069 
01070     if (isa<ConstantPointerNull>(C)) {
01071       unsigned AS = V->getType()->getPointerAddressSpace();
01072       return DAG.getConstant(0, TLI->getPointerTy(AS));
01073     }
01074 
01075     if (const ConstantFP *CFP = dyn_cast<ConstantFP>(C))
01076       return DAG.getConstantFP(*CFP, VT);
01077 
01078     if (isa<UndefValue>(C) && !V->getType()->isAggregateType())
01079       return DAG.getUNDEF(VT);
01080 
01081     if (const ConstantExpr *CE = dyn_cast<ConstantExpr>(C)) {
01082       visit(CE->getOpcode(), *CE);
01083       SDValue N1 = NodeMap[V];
01084       assert(N1.getNode() && "visit didn't populate the NodeMap!");
01085       return N1;
01086     }
01087 
01088     if (isa<ConstantStruct>(C) || isa<ConstantArray>(C)) {
01089       SmallVector<SDValue, 4> Constants;
01090       for (User::const_op_iterator OI = C->op_begin(), OE = C->op_end();
01091            OI != OE; ++OI) {
01092         SDNode *Val = getValue(*OI).getNode();
01093         // If the operand is an empty aggregate, there are no values.
01094         if (!Val) continue;
01095         // Add each leaf value from the operand to the Constants list
01096         // to form a flattened list of all the values.
01097         for (unsigned i = 0, e = Val->getNumValues(); i != e; ++i)
01098           Constants.push_back(SDValue(Val, i));
01099       }
01100 
01101       return DAG.getMergeValues(Constants, getCurSDLoc());
01102     }
01103 
01104     if (const ConstantDataSequential *CDS =
01105           dyn_cast<ConstantDataSequential>(C)) {
01106       SmallVector<SDValue, 4> Ops;
01107       for (unsigned i = 0, e = CDS->getNumElements(); i != e; ++i) {
01108         SDNode *Val = getValue(CDS->getElementAsConstant(i)).getNode();
01109         // Add each leaf value from the operand to the Constants list
01110         // to form a flattened list of all the values.
01111         for (unsigned i = 0, e = Val->getNumValues(); i != e; ++i)
01112           Ops.push_back(SDValue(Val, i));
01113       }
01114 
01115       if (isa<ArrayType>(CDS->getType()))
01116         return DAG.getMergeValues(Ops, getCurSDLoc());
01117       return NodeMap[V] = DAG.getNode(ISD::BUILD_VECTOR, getCurSDLoc(),
01118                                       VT, Ops);
01119     }
01120 
01121     if (C->getType()->isStructTy() || C->getType()->isArrayTy()) {
01122       assert((isa<ConstantAggregateZero>(C) || isa<UndefValue>(C)) &&
01123              "Unknown struct or array constant!");
01124 
01125       SmallVector<EVT, 4> ValueVTs;
01126       ComputeValueVTs(*TLI, C->getType(), ValueVTs);
01127       unsigned NumElts = ValueVTs.size();
01128       if (NumElts == 0)
01129         return SDValue(); // empty struct
01130       SmallVector<SDValue, 4> Constants(NumElts);
01131       for (unsigned i = 0; i != NumElts; ++i) {
01132         EVT EltVT = ValueVTs[i];
01133         if (isa<UndefValue>(C))
01134           Constants[i] = DAG.getUNDEF(EltVT);
01135         else if (EltVT.isFloatingPoint())
01136           Constants[i] = DAG.getConstantFP(0, EltVT);
01137         else
01138           Constants[i] = DAG.getConstant(0, EltVT);
01139       }
01140 
01141       return DAG.getMergeValues(Constants, getCurSDLoc());
01142     }
01143 
01144     if (const BlockAddress *BA = dyn_cast<BlockAddress>(C))
01145       return DAG.getBlockAddress(BA, VT);
01146 
01147     VectorType *VecTy = cast<VectorType>(V->getType());
01148     unsigned NumElements = VecTy->getNumElements();
01149 
01150     // Now that we know the number and type of the elements, get that number of
01151     // elements into the Ops array based on what kind of constant it is.
01152     SmallVector<SDValue, 16> Ops;
01153     if (const ConstantVector *CV = dyn_cast<ConstantVector>(C)) {
01154       for (unsigned i = 0; i != NumElements; ++i)
01155         Ops.push_back(getValue(CV->getOperand(i)));
01156     } else {
01157       assert(isa<ConstantAggregateZero>(C) && "Unknown vector constant!");
01158       EVT EltVT = TLI->getValueType(VecTy->getElementType());
01159 
01160       SDValue Op;
01161       if (EltVT.isFloatingPoint())
01162         Op = DAG.getConstantFP(0, EltVT);
01163       else
01164         Op = DAG.getConstant(0, EltVT);
01165       Ops.assign(NumElements, Op);
01166     }
01167 
01168     // Create a BUILD_VECTOR node.
01169     return NodeMap[V] = DAG.getNode(ISD::BUILD_VECTOR, getCurSDLoc(), VT, Ops);
01170   }
01171 
01172   // If this is a static alloca, generate it as the frameindex instead of
01173   // computation.
01174   if (const AllocaInst *AI = dyn_cast<AllocaInst>(V)) {
01175     DenseMap<const AllocaInst*, int>::iterator SI =
01176       FuncInfo.StaticAllocaMap.find(AI);
01177     if (SI != FuncInfo.StaticAllocaMap.end())
01178       return DAG.getFrameIndex(SI->second, TLI->getPointerTy());
01179   }
01180 
01181   // If this is an instruction which fast-isel has deferred, select it now.
01182   if (const Instruction *Inst = dyn_cast<Instruction>(V)) {
01183     unsigned InReg = FuncInfo.InitializeRegForValue(Inst);
01184     RegsForValue RFV(*DAG.getContext(), *TLI, InReg, Inst->getType());
01185     SDValue Chain = DAG.getEntryNode();
01186     return RFV.getCopyFromRegs(DAG, FuncInfo, getCurSDLoc(), Chain, nullptr, V);
01187   }
01188 
01189   llvm_unreachable("Can't get register for value!");
01190 }
01191 
01192 void SelectionDAGBuilder::visitRet(const ReturnInst &I) {
01193   const TargetLowering *TLI = TM.getSubtargetImpl()->getTargetLowering();
01194   SDValue Chain = getControlRoot();
01195   SmallVector<ISD::OutputArg, 8> Outs;
01196   SmallVector<SDValue, 8> OutVals;
01197 
01198   if (!FuncInfo.CanLowerReturn) {
01199     unsigned DemoteReg = FuncInfo.DemoteRegister;
01200     const Function *F = I.getParent()->getParent();
01201 
01202     // Emit a store of the return value through the virtual register.
01203     // Leave Outs empty so that LowerReturn won't try to load return
01204     // registers the usual way.
01205     SmallVector<EVT, 1> PtrValueVTs;
01206     ComputeValueVTs(*TLI, PointerType::getUnqual(F->getReturnType()),
01207                     PtrValueVTs);
01208 
01209     SDValue RetPtr = DAG.getRegister(DemoteReg, PtrValueVTs[0]);
01210     SDValue RetOp = getValue(I.getOperand(0));
01211 
01212     SmallVector<EVT, 4> ValueVTs;
01213     SmallVector<uint64_t, 4> Offsets;
01214     ComputeValueVTs(*TLI, I.getOperand(0)->getType(), ValueVTs, &Offsets);
01215     unsigned NumValues = ValueVTs.size();
01216 
01217     SmallVector<SDValue, 4> Chains(NumValues);
01218     for (unsigned i = 0; i != NumValues; ++i) {
01219       SDValue Add = DAG.getNode(ISD::ADD, getCurSDLoc(),
01220                                 RetPtr.getValueType(), RetPtr,
01221                                 DAG.getIntPtrConstant(Offsets[i]));
01222       Chains[i] =
01223         DAG.getStore(Chain, getCurSDLoc(),
01224                      SDValue(RetOp.getNode(), RetOp.getResNo() + i),
01225                      // FIXME: better loc info would be nice.
01226                      Add, MachinePointerInfo(), false, false, 0);
01227     }
01228 
01229     Chain = DAG.getNode(ISD::TokenFactor, getCurSDLoc(),
01230                         MVT::Other, Chains);
01231   } else if (I.getNumOperands() != 0) {
01232     SmallVector<EVT, 4> ValueVTs;
01233     ComputeValueVTs(*TLI, I.getOperand(0)->getType(), ValueVTs);
01234     unsigned NumValues = ValueVTs.size();
01235     if (NumValues) {
01236       SDValue RetOp = getValue(I.getOperand(0));
01237       for (unsigned j = 0, f = NumValues; j != f; ++j) {
01238         EVT VT = ValueVTs[j];
01239 
01240         ISD::NodeType ExtendKind = ISD::ANY_EXTEND;
01241 
01242         const Function *F = I.getParent()->getParent();
01243         if (F->getAttributes().hasAttribute(AttributeSet::ReturnIndex,
01244                                             Attribute::SExt))
01245           ExtendKind = ISD::SIGN_EXTEND;
01246         else if (F->getAttributes().hasAttribute(AttributeSet::ReturnIndex,
01247                                                  Attribute::ZExt))
01248           ExtendKind = ISD::ZERO_EXTEND;
01249 
01250         if (ExtendKind != ISD::ANY_EXTEND && VT.isInteger())
01251           VT = TLI->getTypeForExtArgOrReturn(*DAG.getContext(), VT, ExtendKind);
01252 
01253         unsigned NumParts = TLI->getNumRegisters(*DAG.getContext(), VT);
01254         MVT PartVT = TLI->getRegisterType(*DAG.getContext(), VT);
01255         SmallVector<SDValue, 4> Parts(NumParts);
01256         getCopyToParts(DAG, getCurSDLoc(),
01257                        SDValue(RetOp.getNode(), RetOp.getResNo() + j),
01258                        &Parts[0], NumParts, PartVT, &I, ExtendKind);
01259 
01260         // 'inreg' on function refers to return value
01261         ISD::ArgFlagsTy Flags = ISD::ArgFlagsTy();
01262         if (F->getAttributes().hasAttribute(AttributeSet::ReturnIndex,
01263                                             Attribute::InReg))
01264           Flags.setInReg();
01265 
01266         // Propagate extension type if any
01267         if (ExtendKind == ISD::SIGN_EXTEND)
01268           Flags.setSExt();
01269         else if (ExtendKind == ISD::ZERO_EXTEND)
01270           Flags.setZExt();
01271 
01272         for (unsigned i = 0; i < NumParts; ++i) {
01273           Outs.push_back(ISD::OutputArg(Flags, Parts[i].getValueType(),
01274                                         VT, /*isfixed=*/true, 0, 0));
01275           OutVals.push_back(Parts[i]);
01276         }
01277       }
01278     }
01279   }
01280 
01281   bool isVarArg = DAG.getMachineFunction().getFunction()->isVarArg();
01282   CallingConv::ID CallConv =
01283     DAG.getMachineFunction().getFunction()->getCallingConv();
01284   Chain = TM.getSubtargetImpl()->getTargetLowering()->LowerReturn(
01285       Chain, CallConv, isVarArg, Outs, OutVals, getCurSDLoc(), DAG);
01286 
01287   // Verify that the target's LowerReturn behaved as expected.
01288   assert(Chain.getNode() && Chain.getValueType() == MVT::Other &&
01289          "LowerReturn didn't return a valid chain!");
01290 
01291   // Update the DAG with the new chain value resulting from return lowering.
01292   DAG.setRoot(Chain);
01293 }
01294 
01295 /// CopyToExportRegsIfNeeded - If the given value has virtual registers
01296 /// created for it, emit nodes to copy the value into the virtual
01297 /// registers.
01298 void SelectionDAGBuilder::CopyToExportRegsIfNeeded(const Value *V) {
01299   // Skip empty types
01300   if (V->getType()->isEmptyTy())
01301     return;
01302 
01303   DenseMap<const Value *, unsigned>::iterator VMI = FuncInfo.ValueMap.find(V);
01304   if (VMI != FuncInfo.ValueMap.end()) {
01305     assert(!V->use_empty() && "Unused value assigned virtual registers!");
01306     CopyValueToVirtualRegister(V, VMI->second);
01307   }
01308 }
01309 
01310 /// ExportFromCurrentBlock - If this condition isn't known to be exported from
01311 /// the current basic block, add it to ValueMap now so that we'll get a
01312 /// CopyTo/FromReg.
01313 void SelectionDAGBuilder::ExportFromCurrentBlock(const Value *V) {
01314   // No need to export constants.
01315   if (!isa<Instruction>(V) && !isa<Argument>(V)) return;
01316 
01317   // Already exported?
01318   if (FuncInfo.isExportedInst(V)) return;
01319 
01320   unsigned Reg = FuncInfo.InitializeRegForValue(V);
01321   CopyValueToVirtualRegister(V, Reg);
01322 }
01323 
01324 bool SelectionDAGBuilder::isExportableFromCurrentBlock(const Value *V,
01325                                                      const BasicBlock *FromBB) {
01326   // The operands of the setcc have to be in this block.  We don't know
01327   // how to export them from some other block.
01328   if (const Instruction *VI = dyn_cast<Instruction>(V)) {
01329     // Can export from current BB.
01330     if (VI->getParent() == FromBB)
01331       return true;
01332 
01333     // Is already exported, noop.
01334     return FuncInfo.isExportedInst(V);
01335   }
01336 
01337   // If this is an argument, we can export it if the BB is the entry block or
01338   // if it is already exported.
01339   if (isa<Argument>(V)) {
01340     if (FromBB == &FromBB->getParent()->getEntryBlock())
01341       return true;
01342 
01343     // Otherwise, can only export this if it is already exported.
01344     return FuncInfo.isExportedInst(V);
01345   }
01346 
01347   // Otherwise, constants can always be exported.
01348   return true;
01349 }
01350 
01351 /// Return branch probability calculated by BranchProbabilityInfo for IR blocks.
01352 uint32_t SelectionDAGBuilder::getEdgeWeight(const MachineBasicBlock *Src,
01353                                             const MachineBasicBlock *Dst) const {
01354   BranchProbabilityInfo *BPI = FuncInfo.BPI;
01355   if (!BPI)
01356     return 0;
01357   const BasicBlock *SrcBB = Src->getBasicBlock();
01358   const BasicBlock *DstBB = Dst->getBasicBlock();
01359   return BPI->getEdgeWeight(SrcBB, DstBB);
01360 }
01361 
01362 void SelectionDAGBuilder::
01363 addSuccessorWithWeight(MachineBasicBlock *Src, MachineBasicBlock *Dst,
01364                        uint32_t Weight /* = 0 */) {
01365   if (!Weight)
01366     Weight = getEdgeWeight(Src, Dst);
01367   Src->addSuccessor(Dst, Weight);
01368 }
01369 
01370 
01371 static bool InBlock(const Value *V, const BasicBlock *BB) {
01372   if (const Instruction *I = dyn_cast<Instruction>(V))
01373     return I->getParent() == BB;
01374   return true;
01375 }
01376 
01377 /// EmitBranchForMergedCondition - Helper method for FindMergedConditions.
01378 /// This function emits a branch and is used at the leaves of an OR or an
01379 /// AND operator tree.
01380 ///
01381 void
01382 SelectionDAGBuilder::EmitBranchForMergedCondition(const Value *Cond,
01383                                                   MachineBasicBlock *TBB,
01384                                                   MachineBasicBlock *FBB,
01385                                                   MachineBasicBlock *CurBB,
01386                                                   MachineBasicBlock *SwitchBB,
01387                                                   uint32_t TWeight,
01388                                                   uint32_t FWeight) {
01389   const BasicBlock *BB = CurBB->getBasicBlock();
01390 
01391   // If the leaf of the tree is a comparison, merge the condition into
01392   // the caseblock.
01393   if (const CmpInst *BOp = dyn_cast<CmpInst>(Cond)) {
01394     // The operands of the cmp have to be in this block.  We don't know
01395     // how to export them from some other block.  If this is the first block
01396     // of the sequence, no exporting is needed.
01397     if (CurBB == SwitchBB ||
01398         (isExportableFromCurrentBlock(BOp->getOperand(0), BB) &&
01399          isExportableFromCurrentBlock(BOp->getOperand(1), BB))) {
01400       ISD::CondCode Condition;
01401       if (const ICmpInst *IC = dyn_cast<ICmpInst>(Cond)) {
01402         Condition = getICmpCondCode(IC->getPredicate());
01403       } else if (const FCmpInst *FC = dyn_cast<FCmpInst>(Cond)) {
01404         Condition = getFCmpCondCode(FC->getPredicate());
01405         if (TM.Options.NoNaNsFPMath)
01406           Condition = getFCmpCodeWithoutNaN(Condition);
01407       } else {
01408         Condition = ISD::SETEQ; // silence warning.
01409         llvm_unreachable("Unknown compare instruction");
01410       }
01411 
01412       CaseBlock CB(Condition, BOp->getOperand(0), BOp->getOperand(1), nullptr,
01413                    TBB, FBB, CurBB, TWeight, FWeight);
01414       SwitchCases.push_back(CB);
01415       return;
01416     }
01417   }
01418 
01419   // Create a CaseBlock record representing this branch.
01420   CaseBlock CB(ISD::SETEQ, Cond, ConstantInt::getTrue(*DAG.getContext()),
01421                nullptr, TBB, FBB, CurBB, TWeight, FWeight);
01422   SwitchCases.push_back(CB);
01423 }
01424 
01425 /// Scale down both weights to fit into uint32_t.
01426 static void ScaleWeights(uint64_t &NewTrue, uint64_t &NewFalse) {
01427   uint64_t NewMax = (NewTrue > NewFalse) ? NewTrue : NewFalse;
01428   uint32_t Scale = (NewMax / UINT32_MAX) + 1;
01429   NewTrue = NewTrue / Scale;
01430   NewFalse = NewFalse / Scale;
01431 }
01432 
01433 /// FindMergedConditions - If Cond is an expression like
01434 void SelectionDAGBuilder::FindMergedConditions(const Value *Cond,
01435                                                MachineBasicBlock *TBB,
01436                                                MachineBasicBlock *FBB,
01437                                                MachineBasicBlock *CurBB,
01438                                                MachineBasicBlock *SwitchBB,
01439                                                unsigned Opc, uint32_t TWeight,
01440                                                uint32_t FWeight) {
01441   // If this node is not part of the or/and tree, emit it as a branch.
01442   const Instruction *BOp = dyn_cast<Instruction>(Cond);
01443   if (!BOp || !(isa<BinaryOperator>(BOp) || isa<CmpInst>(BOp)) ||
01444       (unsigned)BOp->getOpcode() != Opc || !BOp->hasOneUse() ||
01445       BOp->getParent() != CurBB->getBasicBlock() ||
01446       !InBlock(BOp->getOperand(0), CurBB->getBasicBlock()) ||
01447       !InBlock(BOp->getOperand(1), CurBB->getBasicBlock())) {
01448     EmitBranchForMergedCondition(Cond, TBB, FBB, CurBB, SwitchBB,
01449                                  TWeight, FWeight);
01450     return;
01451   }
01452 
01453   //  Create TmpBB after CurBB.
01454   MachineFunction::iterator BBI = CurBB;
01455   MachineFunction &MF = DAG.getMachineFunction();
01456   MachineBasicBlock *TmpBB = MF.CreateMachineBasicBlock(CurBB->getBasicBlock());
01457   CurBB->getParent()->insert(++BBI, TmpBB);
01458 
01459   if (Opc == Instruction::Or) {
01460     // Codegen X | Y as:
01461     // BB1:
01462     //   jmp_if_X TBB
01463     //   jmp TmpBB
01464     // TmpBB:
01465     //   jmp_if_Y TBB
01466     //   jmp FBB
01467     //
01468 
01469     // We have flexibility in setting Prob for BB1 and Prob for TmpBB.
01470     // The requirement is that
01471     //   TrueProb for BB1 + (FalseProb for BB1 * TrueProb for TmpBB)
01472     //     = TrueProb for orignal BB.
01473     // Assuming the orignal weights are A and B, one choice is to set BB1's
01474     // weights to A and A+2B, and set TmpBB's weights to A and 2B. This choice
01475     // assumes that
01476     //   TrueProb for BB1 == FalseProb for BB1 * TrueProb for TmpBB.
01477     // Another choice is to assume TrueProb for BB1 equals to TrueProb for
01478     // TmpBB, but the math is more complicated.
01479 
01480     uint64_t NewTrueWeight = TWeight;
01481     uint64_t NewFalseWeight = (uint64_t)TWeight + 2 * (uint64_t)FWeight;
01482     ScaleWeights(NewTrueWeight, NewFalseWeight);
01483     // Emit the LHS condition.
01484     FindMergedConditions(BOp->getOperand(0), TBB, TmpBB, CurBB, SwitchBB, Opc,
01485                          NewTrueWeight, NewFalseWeight);
01486 
01487     NewTrueWeight = TWeight;
01488     NewFalseWeight = 2 * (uint64_t)FWeight;
01489     ScaleWeights(NewTrueWeight, NewFalseWeight);
01490     // Emit the RHS condition into TmpBB.
01491     FindMergedConditions(BOp->getOperand(1), TBB, FBB, TmpBB, SwitchBB, Opc,
01492                          NewTrueWeight, NewFalseWeight);
01493   } else {
01494     assert(Opc == Instruction::And && "Unknown merge op!");
01495     // Codegen X & Y as:
01496     // BB1:
01497     //   jmp_if_X TmpBB
01498     //   jmp FBB
01499     // TmpBB:
01500     //   jmp_if_Y TBB
01501     //   jmp FBB
01502     //
01503     //  This requires creation of TmpBB after CurBB.
01504 
01505     // We have flexibility in setting Prob for BB1 and Prob for TmpBB.
01506     // The requirement is that
01507     //   FalseProb for BB1 + (TrueProb for BB1 * FalseProb for TmpBB)
01508     //     = FalseProb for orignal BB.
01509     // Assuming the orignal weights are A and B, one choice is to set BB1's
01510     // weights to 2A+B and B, and set TmpBB's weights to 2A and B. This choice
01511     // assumes that
01512     //   FalseProb for BB1 == TrueProb for BB1 * FalseProb for TmpBB.
01513 
01514     uint64_t NewTrueWeight = 2 * (uint64_t)TWeight + (uint64_t)FWeight;
01515     uint64_t NewFalseWeight = FWeight;
01516     ScaleWeights(NewTrueWeight, NewFalseWeight);
01517     // Emit the LHS condition.
01518     FindMergedConditions(BOp->getOperand(0), TmpBB, FBB, CurBB, SwitchBB, Opc,
01519                          NewTrueWeight, NewFalseWeight);
01520 
01521     NewTrueWeight = 2 * (uint64_t)TWeight;
01522     NewFalseWeight = FWeight;
01523     ScaleWeights(NewTrueWeight, NewFalseWeight);
01524     // Emit the RHS condition into TmpBB.
01525     FindMergedConditions(BOp->getOperand(1), TBB, FBB, TmpBB, SwitchBB, Opc,
01526                          NewTrueWeight, NewFalseWeight);
01527   }
01528 }
01529 
01530 /// If the set of cases should be emitted as a series of branches, return true.
01531 /// If we should emit this as a bunch of and/or'd together conditions, return
01532 /// false.
01533 bool
01534 SelectionDAGBuilder::ShouldEmitAsBranches(const std::vector<CaseBlock> &Cases) {
01535   if (Cases.size() != 2) return true;
01536 
01537   // If this is two comparisons of the same values or'd or and'd together, they
01538   // will get folded into a single comparison, so don't emit two blocks.
01539   if ((Cases[0].CmpLHS == Cases[1].CmpLHS &&
01540        Cases[0].CmpRHS == Cases[1].CmpRHS) ||
01541       (Cases[0].CmpRHS == Cases[1].CmpLHS &&
01542        Cases[0].CmpLHS == Cases[1].CmpRHS)) {
01543     return false;
01544   }
01545 
01546   // Handle: (X != null) | (Y != null) --> (X|Y) != 0
01547   // Handle: (X == null) & (Y == null) --> (X|Y) == 0
01548   if (Cases[0].CmpRHS == Cases[1].CmpRHS &&
01549       Cases[0].CC == Cases[1].CC &&
01550       isa<Constant>(Cases[0].CmpRHS) &&
01551       cast<Constant>(Cases[0].CmpRHS)->isNullValue()) {
01552     if (Cases[0].CC == ISD::SETEQ && Cases[0].TrueBB == Cases[1].ThisBB)
01553       return false;
01554     if (Cases[0].CC == ISD::SETNE && Cases[0].FalseBB == Cases[1].ThisBB)
01555       return false;
01556   }
01557 
01558   return true;
01559 }
01560 
01561 void SelectionDAGBuilder::visitBr(const BranchInst &I) {
01562   MachineBasicBlock *BrMBB = FuncInfo.MBB;
01563 
01564   // Update machine-CFG edges.
01565   MachineBasicBlock *Succ0MBB = FuncInfo.MBBMap[I.getSuccessor(0)];
01566 
01567   // Figure out which block is immediately after the current one.
01568   MachineBasicBlock *NextBlock = nullptr;
01569   MachineFunction::iterator BBI = BrMBB;
01570   if (++BBI != FuncInfo.MF->end())
01571     NextBlock = BBI;
01572 
01573   if (I.isUnconditional()) {
01574     // Update machine-CFG edges.
01575     BrMBB->addSuccessor(Succ0MBB);
01576 
01577     // If this is not a fall-through branch or optimizations are switched off,
01578     // emit the branch.
01579     if (Succ0MBB != NextBlock || TM.getOptLevel() == CodeGenOpt::None)
01580       DAG.setRoot(DAG.getNode(ISD::BR, getCurSDLoc(),
01581                               MVT::Other, getControlRoot(),
01582                               DAG.getBasicBlock(Succ0MBB)));
01583 
01584     return;
01585   }
01586 
01587   // If this condition is one of the special cases we handle, do special stuff
01588   // now.
01589   const Value *CondVal = I.getCondition();
01590   MachineBasicBlock *Succ1MBB = FuncInfo.MBBMap[I.getSuccessor(1)];
01591 
01592   // If this is a series of conditions that are or'd or and'd together, emit
01593   // this as a sequence of branches instead of setcc's with and/or operations.
01594   // As long as jumps are not expensive, this should improve performance.
01595   // For example, instead of something like:
01596   //     cmp A, B
01597   //     C = seteq
01598   //     cmp D, E
01599   //     F = setle
01600   //     or C, F
01601   //     jnz foo
01602   // Emit:
01603   //     cmp A, B
01604   //     je foo
01605   //     cmp D, E
01606   //     jle foo
01607   //
01608   if (const BinaryOperator *BOp = dyn_cast<BinaryOperator>(CondVal)) {
01609     if (!TM.getSubtargetImpl()->getTargetLowering()->isJumpExpensive() &&
01610         BOp->hasOneUse() && (BOp->getOpcode() == Instruction::And ||
01611                              BOp->getOpcode() == Instruction::Or)) {
01612       FindMergedConditions(BOp, Succ0MBB, Succ1MBB, BrMBB, BrMBB,
01613                            BOp->getOpcode(), getEdgeWeight(BrMBB, Succ0MBB),
01614                            getEdgeWeight(BrMBB, Succ1MBB));
01615       // If the compares in later blocks need to use values not currently
01616       // exported from this block, export them now.  This block should always
01617       // be the first entry.
01618       assert(SwitchCases[0].ThisBB == BrMBB && "Unexpected lowering!");
01619 
01620       // Allow some cases to be rejected.
01621       if (ShouldEmitAsBranches(SwitchCases)) {
01622         for (unsigned i = 1, e = SwitchCases.size(); i != e; ++i) {
01623           ExportFromCurrentBlock(SwitchCases[i].CmpLHS);
01624           ExportFromCurrentBlock(SwitchCases[i].CmpRHS);
01625         }
01626 
01627         // Emit the branch for this block.
01628         visitSwitchCase(SwitchCases[0], BrMBB);
01629         SwitchCases.erase(SwitchCases.begin());
01630         return;
01631       }
01632 
01633       // Okay, we decided not to do this, remove any inserted MBB's and clear
01634       // SwitchCases.
01635       for (unsigned i = 1, e = SwitchCases.size(); i != e; ++i)
01636         FuncInfo.MF->erase(SwitchCases[i].ThisBB);
01637 
01638       SwitchCases.clear();
01639     }
01640   }
01641 
01642   // Create a CaseBlock record representing this branch.
01643   CaseBlock CB(ISD::SETEQ, CondVal, ConstantInt::getTrue(*DAG.getContext()),
01644                nullptr, Succ0MBB, Succ1MBB, BrMBB);
01645 
01646   // Use visitSwitchCase to actually insert the fast branch sequence for this
01647   // cond branch.
01648   visitSwitchCase(CB, BrMBB);
01649 }
01650 
01651 /// visitSwitchCase - Emits the necessary code to represent a single node in
01652 /// the binary search tree resulting from lowering a switch instruction.
01653 void SelectionDAGBuilder::visitSwitchCase(CaseBlock &CB,
01654                                           MachineBasicBlock *SwitchBB) {
01655   SDValue Cond;
01656   SDValue CondLHS = getValue(CB.CmpLHS);
01657   SDLoc dl = getCurSDLoc();
01658 
01659   // Build the setcc now.
01660   if (!CB.CmpMHS) {
01661     // Fold "(X == true)" to X and "(X == false)" to !X to
01662     // handle common cases produced by branch lowering.
01663     if (CB.CmpRHS == ConstantInt::getTrue(*DAG.getContext()) &&
01664         CB.CC == ISD::SETEQ)
01665       Cond = CondLHS;
01666     else if (CB.CmpRHS == ConstantInt::getFalse(*DAG.getContext()) &&
01667              CB.CC == ISD::SETEQ) {
01668       SDValue True = DAG.getConstant(1, CondLHS.getValueType());
01669       Cond = DAG.getNode(ISD::XOR, dl, CondLHS.getValueType(), CondLHS, True);
01670     } else
01671       Cond = DAG.getSetCC(dl, MVT::i1, CondLHS, getValue(CB.CmpRHS), CB.CC);
01672   } else {
01673     assert(CB.CC == ISD::SETLE && "Can handle only LE ranges now");
01674 
01675     const APInt& Low = cast<ConstantInt>(CB.CmpLHS)->getValue();
01676     const APInt& High  = cast<ConstantInt>(CB.CmpRHS)->getValue();
01677 
01678     SDValue CmpOp = getValue(CB.CmpMHS);
01679     EVT VT = CmpOp.getValueType();
01680 
01681     if (cast<ConstantInt>(CB.CmpLHS)->isMinValue(true)) {
01682       Cond = DAG.getSetCC(dl, MVT::i1, CmpOp, DAG.getConstant(High, VT),
01683                           ISD::SETLE);
01684     } else {
01685       SDValue SUB = DAG.getNode(ISD::SUB, dl,
01686                                 VT, CmpOp, DAG.getConstant(Low, VT));
01687       Cond = DAG.getSetCC(dl, MVT::i1, SUB,
01688                           DAG.getConstant(High-Low, VT), ISD::SETULE);
01689     }
01690   }
01691 
01692   // Update successor info
01693   addSuccessorWithWeight(SwitchBB, CB.TrueBB, CB.TrueWeight);
01694   // TrueBB and FalseBB are always different unless the incoming IR is
01695   // degenerate. This only happens when running llc on weird IR.
01696   if (CB.TrueBB != CB.FalseBB)
01697     addSuccessorWithWeight(SwitchBB, CB.FalseBB, CB.FalseWeight);
01698 
01699   // Set NextBlock to be the MBB immediately after the current one, if any.
01700   // This is used to avoid emitting unnecessary branches to the next block.
01701   MachineBasicBlock *NextBlock = nullptr;
01702   MachineFunction::iterator BBI = SwitchBB;
01703   if (++BBI != FuncInfo.MF->end())
01704     NextBlock = BBI;
01705 
01706   // If the lhs block is the next block, invert the condition so that we can
01707   // fall through to the lhs instead of the rhs block.
01708   if (CB.TrueBB == NextBlock) {
01709     std::swap(CB.TrueBB, CB.FalseBB);
01710     SDValue True = DAG.getConstant(1, Cond.getValueType());
01711     Cond = DAG.getNode(ISD::XOR, dl, Cond.getValueType(), Cond, True);
01712   }
01713 
01714   SDValue BrCond = DAG.getNode(ISD::BRCOND, dl,
01715                                MVT::Other, getControlRoot(), Cond,
01716                                DAG.getBasicBlock(CB.TrueBB));
01717 
01718   // Insert the false branch. Do this even if it's a fall through branch,
01719   // this makes it easier to do DAG optimizations which require inverting
01720   // the branch condition.
01721   BrCond = DAG.getNode(ISD::BR, dl, MVT::Other, BrCond,
01722                        DAG.getBasicBlock(CB.FalseBB));
01723 
01724   DAG.setRoot(BrCond);
01725 }
01726 
01727 /// visitJumpTable - Emit JumpTable node in the current MBB
01728 void SelectionDAGBuilder::visitJumpTable(JumpTable &JT) {
01729   // Emit the code for the jump table
01730   assert(JT.Reg != -1U && "Should lower JT Header first!");
01731   EVT PTy = TM.getSubtargetImpl()->getTargetLowering()->getPointerTy();
01732   SDValue Index = DAG.getCopyFromReg(getControlRoot(), getCurSDLoc(),
01733                                      JT.Reg, PTy);
01734   SDValue Table = DAG.getJumpTable(JT.JTI, PTy);
01735   SDValue BrJumpTable = DAG.getNode(ISD::BR_JT, getCurSDLoc(),
01736                                     MVT::Other, Index.getValue(1),
01737                                     Table, Index);
01738   DAG.setRoot(BrJumpTable);
01739 }
01740 
01741 /// visitJumpTableHeader - This function emits necessary code to produce index
01742 /// in the JumpTable from switch case.
01743 void SelectionDAGBuilder::visitJumpTableHeader(JumpTable &JT,
01744                                                JumpTableHeader &JTH,
01745                                                MachineBasicBlock *SwitchBB) {
01746   // Subtract the lowest switch case value from the value being switched on and
01747   // conditional branch to default mbb if the result is greater than the
01748   // difference between smallest and largest cases.
01749   SDValue SwitchOp = getValue(JTH.SValue);
01750   EVT VT = SwitchOp.getValueType();
01751   SDValue Sub = DAG.getNode(ISD::SUB, getCurSDLoc(), VT, SwitchOp,
01752                             DAG.getConstant(JTH.First, VT));
01753 
01754   // The SDNode we just created, which holds the value being switched on minus
01755   // the smallest case value, needs to be copied to a virtual register so it
01756   // can be used as an index into the jump table in a subsequent basic block.
01757   // This value may be smaller or larger than the target's pointer type, and
01758   // therefore require extension or truncating.
01759   const TargetLowering *TLI = TM.getSubtargetImpl()->getTargetLowering();
01760   SwitchOp = DAG.getZExtOrTrunc(Sub, getCurSDLoc(), TLI->getPointerTy());
01761 
01762   unsigned JumpTableReg = FuncInfo.CreateReg(TLI->getPointerTy());
01763   SDValue CopyTo = DAG.getCopyToReg(getControlRoot(), getCurSDLoc(),
01764                                     JumpTableReg, SwitchOp);
01765   JT.Reg = JumpTableReg;
01766 
01767   // Emit the range check for the jump table, and branch to the default block
01768   // for the switch statement if the value being switched on exceeds the largest
01769   // case in the switch.
01770   SDValue CMP = DAG.getSetCC(getCurSDLoc(),
01771                              TLI->getSetCCResultType(*DAG.getContext(),
01772                                                      Sub.getValueType()),
01773                              Sub,
01774                              DAG.getConstant(JTH.Last - JTH.First,VT),
01775                              ISD::SETUGT);
01776 
01777   // Set NextBlock to be the MBB immediately after the current one, if any.
01778   // This is used to avoid emitting unnecessary branches to the next block.
01779   MachineBasicBlock *NextBlock = nullptr;
01780   MachineFunction::iterator BBI = SwitchBB;
01781 
01782   if (++BBI != FuncInfo.MF->end())
01783     NextBlock = BBI;
01784 
01785   SDValue BrCond = DAG.getNode(ISD::BRCOND, getCurSDLoc(),
01786                                MVT::Other, CopyTo, CMP,
01787                                DAG.getBasicBlock(JT.Default));
01788 
01789   if (JT.MBB != NextBlock)
01790     BrCond = DAG.getNode(ISD::BR, getCurSDLoc(), MVT::Other, BrCond,
01791                          DAG.getBasicBlock(JT.MBB));
01792 
01793   DAG.setRoot(BrCond);
01794 }
01795 
01796 /// Codegen a new tail for a stack protector check ParentMBB which has had its
01797 /// tail spliced into a stack protector check success bb.
01798 ///
01799 /// For a high level explanation of how this fits into the stack protector
01800 /// generation see the comment on the declaration of class
01801 /// StackProtectorDescriptor.
01802 void SelectionDAGBuilder::visitSPDescriptorParent(StackProtectorDescriptor &SPD,
01803                                                   MachineBasicBlock *ParentBB) {
01804 
01805   // First create the loads to the guard/stack slot for the comparison.
01806   const TargetLowering *TLI = TM.getSubtargetImpl()->getTargetLowering();
01807   EVT PtrTy = TLI->getPointerTy();
01808 
01809   MachineFrameInfo *MFI = ParentBB->getParent()->getFrameInfo();
01810   int FI = MFI->getStackProtectorIndex();
01811 
01812   const Value *IRGuard = SPD.getGuard();
01813   SDValue GuardPtr = getValue(IRGuard);
01814   SDValue StackSlotPtr = DAG.getFrameIndex(FI, PtrTy);
01815 
01816   unsigned Align =
01817     TLI->getDataLayout()->getPrefTypeAlignment(IRGuard->getType());
01818 
01819   SDValue Guard;
01820 
01821   // If GuardReg is set and useLoadStackGuardNode returns true, retrieve the
01822   // guard value from the virtual register holding the value. Otherwise, emit a
01823   // volatile load to retrieve the stack guard value.
01824   unsigned GuardReg = SPD.getGuardReg();
01825 
01826   if (GuardReg && TLI->useLoadStackGuardNode())
01827     Guard = DAG.getCopyFromReg(DAG.getEntryNode(), getCurSDLoc(), GuardReg,
01828                                PtrTy);
01829   else
01830     Guard = DAG.getLoad(PtrTy, getCurSDLoc(), DAG.getEntryNode(),
01831                         GuardPtr, MachinePointerInfo(IRGuard, 0),
01832                         true, false, false, Align);
01833 
01834   SDValue StackSlot = DAG.getLoad(PtrTy, getCurSDLoc(), DAG.getEntryNode(),
01835                                   StackSlotPtr,
01836                                   MachinePointerInfo::getFixedStack(FI),
01837                                   true, false, false, Align);
01838 
01839   // Perform the comparison via a subtract/getsetcc.
01840   EVT VT = Guard.getValueType();
01841   SDValue Sub = DAG.getNode(ISD::SUB, getCurSDLoc(), VT, Guard, StackSlot);
01842 
01843   SDValue Cmp = DAG.getSetCC(getCurSDLoc(),
01844                              TLI->getSetCCResultType(*DAG.getContext(),
01845                                                      Sub.getValueType()),
01846                              Sub, DAG.getConstant(0, VT),
01847                              ISD::SETNE);
01848 
01849   // If the sub is not 0, then we know the guard/stackslot do not equal, so
01850   // branch to failure MBB.
01851   SDValue BrCond = DAG.getNode(ISD::BRCOND, getCurSDLoc(),
01852                                MVT::Other, StackSlot.getOperand(0),
01853                                Cmp, DAG.getBasicBlock(SPD.getFailureMBB()));
01854   // Otherwise branch to success MBB.
01855   SDValue Br = DAG.getNode(ISD::BR, getCurSDLoc(),
01856                            MVT::Other, BrCond,
01857                            DAG.getBasicBlock(SPD.getSuccessMBB()));
01858 
01859   DAG.setRoot(Br);
01860 }
01861 
01862 /// Codegen the failure basic block for a stack protector check.
01863 ///
01864 /// A failure stack protector machine basic block consists simply of a call to
01865 /// __stack_chk_fail().
01866 ///
01867 /// For a high level explanation of how this fits into the stack protector
01868 /// generation see the comment on the declaration of class
01869 /// StackProtectorDescriptor.
01870 void
01871 SelectionDAGBuilder::visitSPDescriptorFailure(StackProtectorDescriptor &SPD) {
01872   const TargetLowering *TLI = TM.getSubtargetImpl()->getTargetLowering();
01873   SDValue Chain = TLI->makeLibCall(DAG, RTLIB::STACKPROTECTOR_CHECK_FAIL,
01874                                    MVT::isVoid, nullptr, 0, false,
01875                                    getCurSDLoc(), false, false).second;
01876   DAG.setRoot(Chain);
01877 }
01878 
01879 /// visitBitTestHeader - This function emits necessary code to produce value
01880 /// suitable for "bit tests"
01881 void SelectionDAGBuilder::visitBitTestHeader(BitTestBlock &B,
01882                                              MachineBasicBlock *SwitchBB) {
01883   // Subtract the minimum value
01884   SDValue SwitchOp = getValue(B.SValue);
01885   EVT VT = SwitchOp.getValueType();
01886   SDValue Sub = DAG.getNode(ISD::SUB, getCurSDLoc(), VT, SwitchOp,
01887                             DAG.getConstant(B.First, VT));
01888 
01889   // Check range
01890   const TargetLowering *TLI = TM.getSubtargetImpl()->getTargetLowering();
01891   SDValue RangeCmp = DAG.getSetCC(getCurSDLoc(),
01892                                   TLI->getSetCCResultType(*DAG.getContext(),
01893                                                          Sub.getValueType()),
01894                                   Sub, DAG.getConstant(B.Range, VT),
01895                                   ISD::SETUGT);
01896 
01897   // Determine the type of the test operands.
01898   bool UsePtrType = false;
01899   if (!TLI->isTypeLegal(VT))
01900     UsePtrType = true;
01901   else {
01902     for (unsigned i = 0, e = B.Cases.size(); i != e; ++i)
01903       if (!isUIntN(VT.getSizeInBits(), B.Cases[i].Mask)) {
01904         // Switch table case range are encoded into series of masks.
01905         // Just use pointer type, it's guaranteed to fit.
01906         UsePtrType = true;
01907         break;
01908       }
01909   }
01910   if (UsePtrType) {
01911     VT = TLI->getPointerTy();
01912     Sub = DAG.getZExtOrTrunc(Sub, getCurSDLoc(), VT);
01913   }
01914 
01915   B.RegVT = VT.getSimpleVT();
01916   B.Reg = FuncInfo.CreateReg(B.RegVT);
01917   SDValue CopyTo = DAG.getCopyToReg(getControlRoot(), getCurSDLoc(),
01918                                     B.Reg, Sub);
01919 
01920   // Set NextBlock to be the MBB immediately after the current one, if any.
01921   // This is used to avoid emitting unnecessary branches to the next block.
01922   MachineBasicBlock *NextBlock = nullptr;
01923   MachineFunction::iterator BBI = SwitchBB;
01924   if (++BBI != FuncInfo.MF->end())
01925     NextBlock = BBI;
01926 
01927   MachineBasicBlock* MBB = B.Cases[0].ThisBB;
01928 
01929   addSuccessorWithWeight(SwitchBB, B.Default);
01930   addSuccessorWithWeight(SwitchBB, MBB);
01931 
01932   SDValue BrRange = DAG.getNode(ISD::BRCOND, getCurSDLoc(),
01933                                 MVT::Other, CopyTo, RangeCmp,
01934                                 DAG.getBasicBlock(B.Default));
01935 
01936   if (MBB != NextBlock)
01937     BrRange = DAG.getNode(ISD::BR, getCurSDLoc(), MVT::Other, CopyTo,
01938                           DAG.getBasicBlock(MBB));
01939 
01940   DAG.setRoot(BrRange);
01941 }
01942 
01943 /// visitBitTestCase - this function produces one "bit test"
01944 void SelectionDAGBuilder::visitBitTestCase(BitTestBlock &BB,
01945                                            MachineBasicBlock* NextMBB,
01946                                            uint32_t BranchWeightToNext,
01947                                            unsigned Reg,
01948                                            BitTestCase &B,
01949                                            MachineBasicBlock *SwitchBB) {
01950   MVT VT = BB.RegVT;
01951   SDValue ShiftOp = DAG.getCopyFromReg(getControlRoot(), getCurSDLoc(),
01952                                        Reg, VT);
01953   SDValue Cmp;
01954   unsigned PopCount = CountPopulation_64(B.Mask);
01955   const TargetLowering *TLI = TM.getSubtargetImpl()->getTargetLowering();
01956   if (PopCount == 1) {
01957     // Testing for a single bit; just compare the shift count with what it
01958     // would need to be to shift a 1 bit in that position.
01959     Cmp = DAG.getSetCC(getCurSDLoc(),
01960                        TLI->getSetCCResultType(*DAG.getContext(), VT),
01961                        ShiftOp,
01962                        DAG.getConstant(countTrailingZeros(B.Mask), VT),
01963                        ISD::SETEQ);
01964   } else if (PopCount == BB.Range) {
01965     // There is only one zero bit in the range, test for it directly.
01966     Cmp = DAG.getSetCC(getCurSDLoc(),
01967                        TLI->getSetCCResultType(*DAG.getContext(), VT),
01968                        ShiftOp,
01969                        DAG.getConstant(CountTrailingOnes_64(B.Mask), VT),
01970                        ISD::SETNE);
01971   } else {
01972     // Make desired shift
01973     SDValue SwitchVal = DAG.getNode(ISD::SHL, getCurSDLoc(), VT,
01974                                     DAG.getConstant(1, VT), ShiftOp);
01975 
01976     // Emit bit tests and jumps
01977     SDValue AndOp = DAG.getNode(ISD::AND, getCurSDLoc(),
01978                                 VT, SwitchVal, DAG.getConstant(B.Mask, VT));
01979     Cmp = DAG.getSetCC(getCurSDLoc(),
01980                        TLI->getSetCCResultType(*DAG.getContext(), VT),
01981                        AndOp, DAG.getConstant(0, VT),
01982                        ISD::SETNE);
01983   }
01984 
01985   // The branch weight from SwitchBB to B.TargetBB is B.ExtraWeight.
01986   addSuccessorWithWeight(SwitchBB, B.TargetBB, B.ExtraWeight);
01987   // The branch weight from SwitchBB to NextMBB is BranchWeightToNext.
01988   addSuccessorWithWeight(SwitchBB, NextMBB, BranchWeightToNext);
01989 
01990   SDValue BrAnd = DAG.getNode(ISD::BRCOND, getCurSDLoc(),
01991                               MVT::Other, getControlRoot(),
01992                               Cmp, DAG.getBasicBlock(B.TargetBB));
01993 
01994   // Set NextBlock to be the MBB immediately after the current one, if any.
01995   // This is used to avoid emitting unnecessary branches to the next block.
01996   MachineBasicBlock *NextBlock = nullptr;
01997   MachineFunction::iterator BBI = SwitchBB;
01998   if (++BBI != FuncInfo.MF->end())
01999     NextBlock = BBI;
02000 
02001   if (NextMBB != NextBlock)
02002     BrAnd = DAG.getNode(ISD::BR, getCurSDLoc(), MVT::Other, BrAnd,
02003                         DAG.getBasicBlock(NextMBB));
02004 
02005   DAG.setRoot(BrAnd);
02006 }
02007 
02008 void SelectionDAGBuilder::visitInvoke(const InvokeInst &I) {
02009   MachineBasicBlock *InvokeMBB = FuncInfo.MBB;
02010 
02011   // Retrieve successors.
02012   MachineBasicBlock *Return = FuncInfo.MBBMap[I.getSuccessor(0)];
02013   MachineBasicBlock *LandingPad = FuncInfo.MBBMap[I.getSuccessor(1)];
02014 
02015   const Value *Callee(I.getCalledValue());
02016   const Function *Fn = dyn_cast<Function>(Callee);
02017   if (isa<InlineAsm>(Callee))
02018     visitInlineAsm(&I);
02019   else if (Fn && Fn->isIntrinsic()) {
02020     assert(Fn->getIntrinsicID() == Intrinsic::donothing);
02021     // Ignore invokes to @llvm.donothing: jump directly to the next BB.
02022   } else
02023     LowerCallTo(&I, getValue(Callee), false, LandingPad);
02024 
02025   // If the value of the invoke is used outside of its defining block, make it
02026   // available as a virtual register.
02027   CopyToExportRegsIfNeeded(&I);
02028 
02029   // Update successor info
02030   addSuccessorWithWeight(InvokeMBB, Return);
02031   addSuccessorWithWeight(InvokeMBB, LandingPad);
02032 
02033   // Drop into normal successor.
02034   DAG.setRoot(DAG.getNode(ISD::BR, getCurSDLoc(),
02035                           MVT::Other, getControlRoot(),
02036                           DAG.getBasicBlock(Return)));
02037 }
02038 
02039 void SelectionDAGBuilder::visitResume(const ResumeInst &RI) {
02040   llvm_unreachable("SelectionDAGBuilder shouldn't visit resume instructions!");
02041 }
02042 
02043 void SelectionDAGBuilder::visitLandingPad(const LandingPadInst &LP) {
02044   assert(FuncInfo.MBB->isLandingPad() &&
02045          "Call to landingpad not in landing pad!");
02046 
02047   MachineBasicBlock *MBB = FuncInfo.MBB;
02048   MachineModuleInfo &MMI = DAG.getMachineFunction().getMMI();
02049   AddLandingPadInfo(LP, MMI, MBB);
02050 
02051   // If there aren't registers to copy the values into (e.g., during SjLj
02052   // exceptions), then don't bother to create these DAG nodes.
02053   const TargetLowering *TLI = TM.getSubtargetImpl()->getTargetLowering();
02054   if (TLI->getExceptionPointerRegister() == 0 &&
02055       TLI->getExceptionSelectorRegister() == 0)
02056     return;
02057 
02058   SmallVector<EVT, 2> ValueVTs;
02059   ComputeValueVTs(*TLI, LP.getType(), ValueVTs);
02060   assert(ValueVTs.size() == 2 && "Only two-valued landingpads are supported");
02061 
02062   // Get the two live-in registers as SDValues. The physregs have already been
02063   // copied into virtual registers.
02064   SDValue Ops[2];
02065   Ops[0] = DAG.getZExtOrTrunc(
02066     DAG.getCopyFromReg(DAG.getEntryNode(), getCurSDLoc(),
02067                        FuncInfo.ExceptionPointerVirtReg, TLI->getPointerTy()),
02068     getCurSDLoc(), ValueVTs[0]);
02069   Ops[1] = DAG.getZExtOrTrunc(
02070     DAG.getCopyFromReg(DAG.getEntryNode(), getCurSDLoc(),
02071                        FuncInfo.ExceptionSelectorVirtReg, TLI->getPointerTy()),
02072     getCurSDLoc(), ValueVTs[1]);
02073 
02074   // Merge into one.
02075   SDValue Res = DAG.getNode(ISD::MERGE_VALUES, getCurSDLoc(),
02076                             DAG.getVTList(ValueVTs), Ops);
02077   setValue(&LP, Res);
02078 }
02079 
02080 /// handleSmallSwitchCaseRange - Emit a series of specific tests (suitable for
02081 /// small case ranges).
02082 bool SelectionDAGBuilder::handleSmallSwitchRange(CaseRec& CR,
02083                                                  CaseRecVector& WorkList,
02084                                                  const Value* SV,
02085                                                  MachineBasicBlock *Default,
02086                                                  MachineBasicBlock *SwitchBB) {
02087   // Size is the number of Cases represented by this range.
02088   size_t Size = CR.Range.second - CR.Range.first;
02089   if (Size > 3)
02090     return false;
02091 
02092   // Get the MachineFunction which holds the current MBB.  This is used when
02093   // inserting any additional MBBs necessary to represent the switch.
02094   MachineFunction *CurMF = FuncInfo.MF;
02095 
02096   // Figure out which block is immediately after the current one.
02097   MachineBasicBlock *NextBlock = nullptr;
02098   MachineFunction::iterator BBI = CR.CaseBB;
02099 
02100   if (++BBI != FuncInfo.MF->end())
02101     NextBlock = BBI;
02102 
02103   BranchProbabilityInfo *BPI = FuncInfo.BPI;
02104   // If any two of the cases has the same destination, and if one value
02105   // is the same as the other, but has one bit unset that the other has set,
02106   // use bit manipulation to do two compares at once.  For example:
02107   // "if (X == 6 || X == 4)" -> "if ((X|2) == 6)"
02108   // TODO: This could be extended to merge any 2 cases in switches with 3 cases.
02109   // TODO: Handle cases where CR.CaseBB != SwitchBB.
02110   if (Size == 2 && CR.CaseBB == SwitchBB) {
02111     Case &Small = *CR.Range.first;
02112     Case &Big = *(CR.Range.second-1);
02113 
02114     if (Small.Low == Small.High && Big.Low == Big.High && Small.BB == Big.BB) {
02115       const APInt& SmallValue = cast<ConstantInt>(Small.Low)->getValue();
02116       const APInt& BigValue = cast<ConstantInt>(Big.Low)->getValue();
02117 
02118       // Check that there is only one bit different.
02119       if (BigValue.countPopulation() == SmallValue.countPopulation() + 1 &&
02120           (SmallValue | BigValue) == BigValue) {
02121         // Isolate the common bit.
02122         APInt CommonBit = BigValue & ~SmallValue;
02123         assert((SmallValue | CommonBit) == BigValue &&
02124                CommonBit.countPopulation() == 1 && "Not a common bit?");
02125 
02126         SDValue CondLHS = getValue(SV);
02127         EVT VT = CondLHS.getValueType();
02128         SDLoc DL = getCurSDLoc();
02129 
02130         SDValue Or = DAG.getNode(ISD::OR, DL, VT, CondLHS,
02131                                  DAG.getConstant(CommonBit, VT));
02132         SDValue Cond = DAG.getSetCC(DL, MVT::i1,
02133                                     Or, DAG.getConstant(BigValue, VT),
02134                                     ISD::SETEQ);
02135 
02136         // Update successor info.
02137         // Both Small and Big will jump to Small.BB, so we sum up the weights.
02138         addSuccessorWithWeight(SwitchBB, Small.BB,
02139                                Small.ExtraWeight + Big.ExtraWeight);
02140         addSuccessorWithWeight(SwitchBB, Default,
02141           // The default destination is the first successor in IR.
02142           BPI ? BPI->getEdgeWeight(SwitchBB->getBasicBlock(), (unsigned)0) : 0);
02143 
02144         // Insert the true branch.
02145         SDValue BrCond = DAG.getNode(ISD::BRCOND, DL, MVT::Other,
02146                                      getControlRoot(), Cond,
02147                                      DAG.getBasicBlock(Small.BB));
02148 
02149         // Insert the false branch.
02150         BrCond = DAG.getNode(ISD::BR, DL, MVT::Other, BrCond,
02151                              DAG.getBasicBlock(Default));
02152 
02153         DAG.setRoot(BrCond);
02154         return true;
02155       }
02156     }
02157   }
02158 
02159   // Order cases by weight so the most likely case will be checked first.
02160   uint32_t UnhandledWeights = 0;
02161   if (BPI) {
02162     for (CaseItr I = CR.Range.first, IE = CR.Range.second; I != IE; ++I) {
02163       uint32_t IWeight = I->ExtraWeight;
02164       UnhandledWeights += IWeight;
02165       for (CaseItr J = CR.Range.first; J < I; ++J) {
02166         uint32_t JWeight = J->ExtraWeight;
02167         if (IWeight > JWeight)
02168           std::swap(*I, *J);
02169       }
02170     }
02171   }
02172   // Rearrange the case blocks so that the last one falls through if possible.
02173   Case &BackCase = *(CR.Range.second-1);
02174   if (Size > 1 &&
02175       NextBlock && Default != NextBlock && BackCase.BB != NextBlock) {
02176     // The last case block won't fall through into 'NextBlock' if we emit the
02177     // branches in this order.  See if rearranging a case value would help.
02178     // We start at the bottom as it's the case with the least weight.
02179     for (Case *I = &*(CR.Range.second-2), *E = &*CR.Range.first-1; I != E; --I)
02180       if (I->BB == NextBlock) {
02181         std::swap(*I, BackCase);
02182         break;
02183       }
02184   }
02185 
02186   // Create a CaseBlock record representing a conditional branch to
02187   // the Case's target mbb if the value being switched on SV is equal
02188   // to C.
02189   MachineBasicBlock *CurBlock = CR.CaseBB;
02190   for (CaseItr I = CR.Range.first, E = CR.Range.second; I != E; ++I) {
02191     MachineBasicBlock *FallThrough;
02192     if (I != E-1) {
02193       FallThrough = CurMF->CreateMachineBasicBlock(CurBlock->getBasicBlock());
02194       CurMF->insert(BBI, FallThrough);
02195 
02196       // Put SV in a virtual register to make it available from the new blocks.
02197       ExportFromCurrentBlock(SV);
02198     } else {
02199       // If the last case doesn't match, go to the default block.
02200       FallThrough = Default;
02201     }
02202 
02203     const Value *RHS, *LHS, *MHS;
02204     ISD::CondCode CC;
02205     if (I->High == I->Low) {
02206       // This is just small small case range :) containing exactly 1 case
02207       CC = ISD::SETEQ;
02208       LHS = SV; RHS = I->High; MHS = nullptr;
02209     } else {
02210       CC = ISD::SETLE;
02211       LHS = I->Low; MHS = SV; RHS = I->High;
02212     }
02213 
02214     // The false weight should be sum of all un-handled cases.
02215     UnhandledWeights -= I->ExtraWeight;
02216     CaseBlock CB(CC, LHS, RHS, MHS, /* truebb */ I->BB, /* falsebb */ FallThrough,
02217                  /* me */ CurBlock,
02218                  /* trueweight */ I->ExtraWeight,
02219                  /* falseweight */ UnhandledWeights);
02220 
02221     // If emitting the first comparison, just call visitSwitchCase to emit the
02222     // code into the current block.  Otherwise, push the CaseBlock onto the
02223     // vector to be later processed by SDISel, and insert the node's MBB
02224     // before the next MBB.
02225     if (CurBlock == SwitchBB)
02226       visitSwitchCase(CB, SwitchBB);
02227     else
02228       SwitchCases.push_back(CB);
02229 
02230     CurBlock = FallThrough;
02231   }
02232 
02233   return true;
02234 }
02235 
02236 static inline bool areJTsAllowed(const TargetLowering &TLI) {
02237   return TLI.isOperationLegalOrCustom(ISD::BR_JT, MVT::Other) ||
02238          TLI.isOperationLegalOrCustom(ISD::BRIND, MVT::Other);
02239 }
02240 
02241 static APInt ComputeRange(const APInt &First, const APInt &Last) {
02242   uint32_t BitWidth = std::max(Last.getBitWidth(), First.getBitWidth()) + 1;
02243   APInt LastExt = Last.sext(BitWidth), FirstExt = First.sext(BitWidth);
02244   return (LastExt - FirstExt + 1ULL);
02245 }
02246 
02247 /// handleJTSwitchCase - Emit jumptable for current switch case range
02248 bool SelectionDAGBuilder::handleJTSwitchCase(CaseRec &CR,
02249                                              CaseRecVector &WorkList,
02250                                              const Value *SV,
02251                                              MachineBasicBlock *Default,
02252                                              MachineBasicBlock *SwitchBB) {
02253   Case& FrontCase = *CR.Range.first;
02254   Case& BackCase  = *(CR.Range.second-1);
02255 
02256   const APInt &First = cast<ConstantInt>(FrontCase.Low)->getValue();
02257   const APInt &Last  = cast<ConstantInt>(BackCase.High)->getValue();
02258 
02259   APInt TSize(First.getBitWidth(), 0);
02260   for (CaseItr I = CR.Range.first, E = CR.Range.second; I != E; ++I)
02261     TSize += I->size();
02262 
02263   const TargetLowering *TLI = TM.getSubtargetImpl()->getTargetLowering();
02264   if (!areJTsAllowed(*TLI) || TSize.ult(TLI->getMinimumJumpTableEntries()))
02265     return false;
02266 
02267   APInt Range = ComputeRange(First, Last);
02268   // The density is TSize / Range. Require at least 40%.
02269   // It should not be possible for IntTSize to saturate for sane code, but make
02270   // sure we handle Range saturation correctly.
02271   uint64_t IntRange = Range.getLimitedValue(UINT64_MAX/10);
02272   uint64_t IntTSize = TSize.getLimitedValue(UINT64_MAX/10);
02273   if (IntTSize * 10 < IntRange * 4)
02274     return false;
02275 
02276   DEBUG(dbgs() << "Lowering jump table\n"
02277                << "First entry: " << First << ". Last entry: " << Last << '\n'
02278                << "Range: " << Range << ". Size: " << TSize << ".\n\n");
02279 
02280   // Get the MachineFunction which holds the current MBB.  This is used when
02281   // inserting any additional MBBs necessary to represent the switch.
02282   MachineFunction *CurMF = FuncInfo.MF;
02283 
02284   // Figure out which block is immediately after the current one.
02285   MachineFunction::iterator BBI = CR.CaseBB;
02286   ++BBI;
02287 
02288   const BasicBlock *LLVMBB = CR.CaseBB->getBasicBlock();
02289 
02290   // Create a new basic block to hold the code for loading the address
02291   // of the jump table, and jumping to it.  Update successor information;
02292   // we will either branch to the default case for the switch, or the jump
02293   // table.
02294   MachineBasicBlock *JumpTableBB = CurMF->CreateMachineBasicBlock(LLVMBB);
02295   CurMF->insert(BBI, JumpTableBB);
02296 
02297   addSuccessorWithWeight(CR.CaseBB, Default);
02298   addSuccessorWithWeight(CR.CaseBB, JumpTableBB);
02299 
02300   // Build a vector of destination BBs, corresponding to each target
02301   // of the jump table. If the value of the jump table slot corresponds to
02302   // a case statement, push the case's BB onto the vector, otherwise, push
02303   // the default BB.
02304   std::vector<MachineBasicBlock*> DestBBs;
02305   APInt TEI = First;
02306   for (CaseItr I = CR.Range.first, E = CR.Range.second; I != E; ++TEI) {
02307     const APInt &Low = cast<ConstantInt>(I->Low)->getValue();
02308     const APInt &High = cast<ConstantInt>(I->High)->getValue();
02309 
02310     if (Low.sle(TEI) && TEI.sle(High)) {
02311       DestBBs.push_back(I->BB);
02312       if (TEI==High)
02313         ++I;
02314     } else {
02315       DestBBs.push_back(Default);
02316     }
02317   }
02318 
02319   // Calculate weight for each unique destination in CR.
02320   DenseMap<MachineBasicBlock*, uint32_t> DestWeights;
02321   if (FuncInfo.BPI)
02322     for (CaseItr I = CR.Range.first, E = CR.Range.second; I != E; ++I) {
02323       DenseMap<MachineBasicBlock*, uint32_t>::iterator Itr =
02324           DestWeights.find(I->BB);
02325       if (Itr != DestWeights.end())
02326         Itr->second += I->ExtraWeight;
02327       else
02328         DestWeights[I->BB] = I->ExtraWeight;
02329     }
02330 
02331   // Update successor info. Add one edge to each unique successor.
02332   BitVector SuccsHandled(CR.CaseBB->getParent()->getNumBlockIDs());
02333   for (std::vector<MachineBasicBlock*>::iterator I = DestBBs.begin(),
02334          E = DestBBs.end(); I != E; ++I) {
02335     if (!SuccsHandled[(*I)->getNumber()]) {
02336       SuccsHandled[(*I)->getNumber()] = true;
02337       DenseMap<MachineBasicBlock*, uint32_t>::iterator Itr =
02338           DestWeights.find(*I);
02339       addSuccessorWithWeight(JumpTableBB, *I,
02340                              Itr != DestWeights.end() ? Itr->second : 0);
02341     }
02342   }
02343 
02344   // Create a jump table index for this jump table.
02345   unsigned JTEncoding = TLI->getJumpTableEncoding();
02346   unsigned JTI = CurMF->getOrCreateJumpTableInfo(JTEncoding)
02347                        ->createJumpTableIndex(DestBBs);
02348 
02349   // Set the jump table information so that we can codegen it as a second
02350   // MachineBasicBlock
02351   JumpTable JT(-1U, JTI, JumpTableBB, Default);
02352   JumpTableHeader JTH(First, Last, SV, CR.CaseBB, (CR.CaseBB == SwitchBB));
02353   if (CR.CaseBB == SwitchBB)
02354     visitJumpTableHeader(JT, JTH, SwitchBB);
02355 
02356   JTCases.push_back(JumpTableBlock(JTH, JT));
02357   return true;
02358 }
02359 
02360 /// handleBTSplitSwitchCase - emit comparison and split binary search tree into
02361 /// 2 subtrees.
02362 bool SelectionDAGBuilder::handleBTSplitSwitchCase(CaseRec& CR,
02363                                                   CaseRecVector& WorkList,
02364                                                   const Value* SV,
02365                                                   MachineBasicBlock* Default,
02366                                                   MachineBasicBlock* SwitchBB) {
02367   // Get the MachineFunction which holds the current MBB.  This is used when
02368   // inserting any additional MBBs necessary to represent the switch.
02369   MachineFunction *CurMF = FuncInfo.MF;
02370 
02371   // Figure out which block is immediately after the current one.
02372   MachineFunction::iterator BBI = CR.CaseBB;
02373   ++BBI;
02374 
02375   Case& FrontCase = *CR.Range.first;
02376   Case& BackCase  = *(CR.Range.second-1);
02377   const BasicBlock *LLVMBB = CR.CaseBB->getBasicBlock();
02378 
02379   // Size is the number of Cases represented by this range.
02380   unsigned Size = CR.Range.second - CR.Range.first;
02381 
02382   const APInt &First = cast<ConstantInt>(FrontCase.Low)->getValue();
02383   const APInt &Last  = cast<ConstantInt>(BackCase.High)->getValue();
02384   double FMetric = 0;
02385   CaseItr Pivot = CR.Range.first + Size/2;
02386 
02387   // Select optimal pivot, maximizing sum density of LHS and RHS. This will
02388   // (heuristically) allow us to emit JumpTable's later.
02389   APInt TSize(First.getBitWidth(), 0);
02390   for (CaseItr I = CR.Range.first, E = CR.Range.second;
02391        I!=E; ++I)
02392     TSize += I->size();
02393 
02394   APInt LSize = FrontCase.size();
02395   APInt RSize = TSize-LSize;
02396   DEBUG(dbgs() << "Selecting best pivot: \n"
02397                << "First: " << First << ", Last: " << Last <<'\n'
02398                << "LSize: " << LSize << ", RSize: " << RSize << '\n');
02399   for (CaseItr I = CR.Range.first, J=I+1, E = CR.Range.second;
02400        J!=E; ++I, ++J) {
02401     const APInt &LEnd = cast<ConstantInt>(I->High)->getValue();
02402     const APInt &RBegin = cast<ConstantInt>(J->Low)->getValue();
02403     APInt Range = ComputeRange(LEnd, RBegin);
02404     assert((Range - 2ULL).isNonNegative() &&
02405            "Invalid case distance");
02406     // Use volatile double here to avoid excess precision issues on some hosts,
02407     // e.g. that use 80-bit X87 registers.
02408     volatile double LDensity =
02409        (double)LSize.roundToDouble() /
02410                            (LEnd - First + 1ULL).roundToDouble();
02411     volatile double RDensity =
02412       (double)RSize.roundToDouble() /
02413                            (Last - RBegin + 1ULL).roundToDouble();
02414     volatile double Metric = Range.logBase2()*(LDensity+RDensity);
02415     // Should always split in some non-trivial place
02416     DEBUG(dbgs() <<"=>Step\n"
02417                  << "LEnd: " << LEnd << ", RBegin: " << RBegin << '\n'
02418                  << "LDensity: " << LDensity
02419                  << ", RDensity: " << RDensity << '\n'
02420                  << "Metric: " << Metric << '\n');
02421     if (FMetric < Metric) {
02422       Pivot = J;
02423       FMetric = Metric;
02424       DEBUG(dbgs() << "Current metric set to: " << FMetric << '\n');
02425     }
02426 
02427     LSize += J->size();
02428     RSize -= J->size();
02429   }
02430 
02431   const TargetLowering *TLI = TM.getSubtargetImpl()->getTargetLowering();
02432   if (areJTsAllowed(*TLI)) {
02433     // If our case is dense we *really* should handle it earlier!
02434     assert((FMetric > 0) && "Should handle dense range earlier!");
02435   } else {
02436     Pivot = CR.Range.first + Size/2;
02437   }
02438 
02439   CaseRange LHSR(CR.Range.first, Pivot);
02440   CaseRange RHSR(Pivot, CR.Range.second);
02441   const Constant *C = Pivot->Low;
02442   MachineBasicBlock *FalseBB = nullptr, *TrueBB = nullptr;
02443 
02444   // We know that we branch to the LHS if the Value being switched on is
02445   // less than the Pivot value, C.  We use this to optimize our binary
02446   // tree a bit, by recognizing that if SV is greater than or equal to the
02447   // LHS's Case Value, and that Case Value is exactly one less than the
02448   // Pivot's Value, then we can branch directly to the LHS's Target,
02449   // rather than creating a leaf node for it.
02450   if ((LHSR.second - LHSR.first) == 1 &&
02451       LHSR.first->High == CR.GE &&
02452       cast<ConstantInt>(C)->getValue() ==
02453       (cast<ConstantInt>(CR.GE)->getValue() + 1LL)) {
02454     TrueBB = LHSR.first->BB;
02455   } else {
02456     TrueBB = CurMF->CreateMachineBasicBlock(LLVMBB);
02457     CurMF->insert(BBI, TrueBB);
02458     WorkList.push_back(CaseRec(TrueBB, C, CR.GE, LHSR));
02459 
02460     // Put SV in a virtual register to make it available from the new blocks.
02461     ExportFromCurrentBlock(SV);
02462   }
02463 
02464   // Similar to the optimization above, if the Value being switched on is
02465   // known to be less than the Constant CR.LT, and the current Case Value
02466   // is CR.LT - 1, then we can branch directly to the target block for
02467   // the current Case Value, rather than emitting a RHS leaf node for it.
02468   if ((RHSR.second - RHSR.first) == 1 && CR.LT &&
02469       cast<ConstantInt>(RHSR.first->Low)->getValue() ==
02470       (cast<ConstantInt>(CR.LT)->getValue() - 1LL)) {
02471     FalseBB = RHSR.first->BB;
02472   } else {
02473     FalseBB = CurMF->CreateMachineBasicBlock(LLVMBB);
02474     CurMF->insert(BBI, FalseBB);
02475     WorkList.push_back(CaseRec(FalseBB,CR.LT,C,RHSR));
02476 
02477     // Put SV in a virtual register to make it available from the new blocks.
02478     ExportFromCurrentBlock(SV);
02479   }
02480 
02481   // Create a CaseBlock record representing a conditional branch to
02482   // the LHS node if the value being switched on SV is less than C.
02483   // Otherwise, branch to LHS.
02484   CaseBlock CB(ISD::SETLT, SV, C, nullptr, TrueBB, FalseBB, CR.CaseBB);
02485 
02486   if (CR.CaseBB == SwitchBB)
02487     visitSwitchCase(CB, SwitchBB);
02488   else
02489     SwitchCases.push_back(CB);
02490 
02491   return true;
02492 }
02493 
02494 /// handleBitTestsSwitchCase - if current case range has few destination and
02495 /// range span less, than machine word bitwidth, encode case range into series
02496 /// of masks and emit bit tests with these masks.
02497 bool SelectionDAGBuilder::handleBitTestsSwitchCase(CaseRec& CR,
02498                                                    CaseRecVector& WorkList,
02499                                                    const Value* SV,
02500                                                    MachineBasicBlock* Default,
02501                                                    MachineBasicBlock* SwitchBB) {
02502   const TargetLowering *TLI = TM.getSubtargetImpl()->getTargetLowering();
02503   EVT PTy = TLI->getPointerTy();
02504   unsigned IntPtrBits = PTy.getSizeInBits();
02505 
02506   Case& FrontCase = *CR.Range.first;
02507   Case& BackCase  = *(CR.Range.second-1);
02508 
02509   // Get the MachineFunction which holds the current MBB.  This is used when
02510   // inserting any additional MBBs necessary to represent the switch.
02511   MachineFunction *CurMF = FuncInfo.MF;
02512 
02513   // If target does not have legal shift left, do not emit bit tests at all.
02514   if (!TLI->isOperationLegal(ISD::SHL, PTy))
02515     return false;
02516 
02517   size_t numCmps = 0;
02518   for (CaseItr I = CR.Range.first, E = CR.Range.second;
02519        I!=E; ++I) {
02520     // Single case counts one, case range - two.
02521     numCmps += (I->Low == I->High ? 1 : 2);
02522   }
02523 
02524   // Count unique destinations
02525   SmallSet<MachineBasicBlock*, 4> Dests;
02526   for (CaseItr I = CR.Range.first, E = CR.Range.second; I!=E; ++I) {
02527     Dests.insert(I->BB);
02528     if (Dests.size() > 3)
02529       // Don't bother the code below, if there are too much unique destinations
02530       return false;
02531   }
02532   DEBUG(dbgs() << "Total number of unique destinations: "
02533         << Dests.size() << '\n'
02534         << "Total number of comparisons: " << numCmps << '\n');
02535 
02536   // Compute span of values.
02537   const APInt& minValue = cast<ConstantInt>(FrontCase.Low)->getValue();
02538   const APInt& maxValue = cast<ConstantInt>(BackCase.High)->getValue();
02539   APInt cmpRange = maxValue - minValue;
02540 
02541   DEBUG(dbgs() << "Compare range: " << cmpRange << '\n'
02542                << "Low bound: " << minValue << '\n'
02543                << "High bound: " << maxValue << '\n');
02544 
02545   if (cmpRange.uge(IntPtrBits) ||
02546       (!(Dests.size() == 1 && numCmps >= 3) &&
02547        !(Dests.size() == 2 && numCmps >= 5) &&
02548        !(Dests.size() >= 3 && numCmps >= 6)))
02549     return false;
02550 
02551   DEBUG(dbgs() << "Emitting bit tests\n");
02552   APInt lowBound = APInt::getNullValue(cmpRange.getBitWidth());
02553 
02554   // Optimize the case where all the case values fit in a
02555   // word without having to subtract minValue. In this case,
02556   // we can optimize away the subtraction.
02557   if (minValue.isNonNegative() && maxValue.slt(IntPtrBits)) {
02558     cmpRange = maxValue;
02559   } else {
02560     lowBound = minValue;
02561   }
02562 
02563   CaseBitsVector CasesBits;
02564   unsigned i, count = 0;
02565 
02566   for (CaseItr I = CR.Range.first, E = CR.Range.second; I!=E; ++I) {
02567     MachineBasicBlock* Dest = I->BB;
02568     for (i = 0; i < count; ++i)
02569       if (Dest == CasesBits[i].BB)
02570         break;
02571 
02572     if (i == count) {
02573       assert((count < 3) && "Too much destinations to test!");
02574       CasesBits.push_back(CaseBits(0, Dest, 0, 0/*Weight*/));
02575       count++;
02576     }
02577 
02578     const APInt& lowValue = cast<ConstantInt>(I->Low)->getValue();
02579     const APInt& highValue = cast<ConstantInt>(I->High)->getValue();
02580 
02581     uint64_t lo = (lowValue - lowBound).getZExtValue();
02582     uint64_t hi = (highValue - lowBound).getZExtValue();
02583     CasesBits[i].ExtraWeight += I->ExtraWeight;
02584 
02585     for (uint64_t j = lo; j <= hi; j++) {
02586       CasesBits[i].Mask |=  1ULL << j;
02587       CasesBits[i].Bits++;
02588     }
02589 
02590   }
02591   std::sort(CasesBits.begin(), CasesBits.end(), CaseBitsCmp());
02592 
02593   BitTestInfo BTC;
02594 
02595   // Figure out which block is immediately after the current one.
02596   MachineFunction::iterator BBI = CR.CaseBB;
02597   ++BBI;
02598 
02599   const BasicBlock *LLVMBB = CR.CaseBB->getBasicBlock();
02600 
02601   DEBUG(dbgs() << "Cases:\n");
02602   for (unsigned i = 0, e = CasesBits.size(); i!=e; ++i) {
02603     DEBUG(dbgs() << "Mask: " << CasesBits[i].Mask
02604                  << ", Bits: " << CasesBits[i].Bits
02605                  << ", BB: " << CasesBits[i].BB << '\n');
02606 
02607     MachineBasicBlock *CaseBB = CurMF->CreateMachineBasicBlock(LLVMBB);
02608     CurMF->insert(BBI, CaseBB);
02609     BTC.push_back(BitTestCase(CasesBits[i].Mask,
02610                               CaseBB,
02611                               CasesBits[i].BB, CasesBits[i].ExtraWeight));
02612 
02613     // Put SV in a virtual register to make it available from the new blocks.
02614     ExportFromCurrentBlock(SV);
02615   }
02616 
02617   BitTestBlock BTB(lowBound, cmpRange, SV,
02618                    -1U, MVT::Other, (CR.CaseBB == SwitchBB),
02619                    CR.CaseBB, Default, BTC);
02620 
02621   if (CR.CaseBB == SwitchBB)
02622     visitBitTestHeader(BTB, SwitchBB);
02623 
02624   BitTestCases.push_back(BTB);
02625 
02626   return true;
02627 }
02628 
02629 /// Clusterify - Transform simple list of Cases into list of CaseRange's
02630 size_t SelectionDAGBuilder::Clusterify(CaseVector& Cases,
02631                                        const SwitchInst& SI) {
02632   size_t numCmps = 0;
02633 
02634   BranchProbabilityInfo *BPI = FuncInfo.BPI;
02635   // Start with "simple" cases
02636   for (SwitchInst::ConstCaseIt i = SI.case_begin(), e = SI.case_end();
02637        i != e; ++i) {
02638     const BasicBlock *SuccBB = i.getCaseSuccessor();
02639     MachineBasicBlock *SMBB = FuncInfo.MBBMap[SuccBB];
02640 
02641     uint32_t ExtraWeight =
02642       BPI ? BPI->getEdgeWeight(SI.getParent(), i.getSuccessorIndex()) : 0;
02643 
02644     Cases.push_back(Case(i.getCaseValue(), i.getCaseValue(),
02645                          SMBB, ExtraWeight));
02646   }
02647   std::sort(Cases.begin(), Cases.end(), CaseCmp());
02648 
02649   // Merge case into clusters
02650   if (Cases.size() >= 2)
02651     // Must recompute end() each iteration because it may be
02652     // invalidated by erase if we hold on to it
02653     for (CaseItr I = Cases.begin(), J = std::next(Cases.begin());
02654          J != Cases.end(); ) {
02655       const APInt& nextValue = cast<ConstantInt>(J->Low)->getValue();
02656       const APInt& currentValue = cast<ConstantInt>(I->High)->getValue();
02657       MachineBasicBlock* nextBB = J->BB;
02658       MachineBasicBlock* currentBB = I->BB;
02659 
02660       // If the two neighboring cases go to the same destination, merge them
02661       // into a single case.
02662       if ((nextValue - currentValue == 1) && (currentBB == nextBB)) {
02663         I->High = J->High;
02664         I->ExtraWeight += J->ExtraWeight;
02665         J = Cases.erase(J);
02666       } else {
02667         I = J++;
02668       }
02669     }
02670 
02671   for (CaseItr I=Cases.begin(), E=Cases.end(); I!=E; ++I, ++numCmps) {
02672     if (I->Low != I->High)
02673       // A range counts double, since it requires two compares.
02674       ++numCmps;
02675   }
02676 
02677   return numCmps;
02678 }
02679 
02680 void SelectionDAGBuilder::UpdateSplitBlock(MachineBasicBlock *First,
02681                                            MachineBasicBlock *Last) {
02682   // Update JTCases.
02683   for (unsigned i = 0, e = JTCases.size(); i != e; ++i)
02684     if (JTCases[i].first.HeaderBB == First)
02685       JTCases[i].first.HeaderBB = Last;
02686 
02687   // Update BitTestCases.
02688   for (unsigned i = 0, e = BitTestCases.size(); i != e; ++i)
02689     if (BitTestCases[i].Parent == First)
02690       BitTestCases[i].Parent = Last;
02691 }
02692 
02693 void SelectionDAGBuilder::visitSwitch(const SwitchInst &SI) {
02694   MachineBasicBlock *SwitchMBB = FuncInfo.MBB;
02695 
02696   // Figure out which block is immediately after the current one.
02697   MachineBasicBlock *NextBlock = nullptr;
02698   MachineBasicBlock *Default = FuncInfo.MBBMap[SI.getDefaultDest()];
02699 
02700   // If there is only the default destination, branch to it if it is not the
02701   // next basic block.  Otherwise, just fall through.
02702   if (!SI.getNumCases()) {
02703     // Update machine-CFG edges.
02704 
02705     // If this is not a fall-through branch, emit the branch.
02706     SwitchMBB->addSuccessor(Default);
02707     if (Default != NextBlock)
02708       DAG.setRoot(DAG.getNode(ISD::BR, getCurSDLoc(),
02709                               MVT::Other, getControlRoot(),
02710                               DAG.getBasicBlock(Default)));
02711 
02712     return;
02713   }
02714 
02715   // If there are any non-default case statements, create a vector of Cases
02716   // representing each one, and sort the vector so that we can efficiently
02717   // create a binary search tree from them.
02718   CaseVector Cases;
02719   size_t numCmps = Clusterify(Cases, SI);
02720   DEBUG(dbgs() << "Clusterify finished. Total clusters: " << Cases.size()
02721                << ". Total compares: " << numCmps << '\n');
02722   (void)numCmps;
02723 
02724   // Get the Value to be switched on and default basic blocks, which will be
02725   // inserted into CaseBlock records, representing basic blocks in the binary
02726   // search tree.
02727   const Value *SV = SI.getCondition();
02728 
02729   // Push the initial CaseRec onto the worklist
02730   CaseRecVector WorkList;
02731   WorkList.push_back(CaseRec(SwitchMBB,nullptr,nullptr,
02732                              CaseRange(Cases.begin(),Cases.end())));
02733 
02734   while (!WorkList.empty()) {
02735     // Grab a record representing a case range to process off the worklist
02736     CaseRec CR = WorkList.back();
02737     WorkList.pop_back();
02738 
02739     if (handleBitTestsSwitchCase(CR, WorkList, SV, Default, SwitchMBB))
02740       continue;
02741 
02742     // If the range has few cases (two or less) emit a series of specific
02743     // tests.
02744     if (handleSmallSwitchRange(CR, WorkList, SV, Default, SwitchMBB))
02745       continue;
02746 
02747     // If the switch has more than N blocks, and is at least 40% dense, and the
02748     // target supports indirect branches, then emit a jump table rather than
02749     // lowering the switch to a binary tree of conditional branches.
02750     // N defaults to 4 and is controlled via TLS.getMinimumJumpTableEntries().
02751     if (handleJTSwitchCase(CR, WorkList, SV, Default, SwitchMBB))
02752       continue;
02753 
02754     // Emit binary tree. We need to pick a pivot, and push left and right ranges
02755     // onto the worklist. Leafs are handled via handleSmallSwitchRange() call.
02756     handleBTSplitSwitchCase(CR, WorkList, SV, Default, SwitchMBB);
02757   }
02758 }
02759 
02760 void SelectionDAGBuilder::visitIndirectBr(const IndirectBrInst &I) {
02761   MachineBasicBlock *IndirectBrMBB = FuncInfo.MBB;
02762 
02763   // Update machine-CFG edges with unique successors.
02764   SmallSet<BasicBlock*, 32> Done;
02765   for (unsigned i = 0, e = I.getNumSuccessors(); i != e; ++i) {
02766     BasicBlock *BB = I.getSuccessor(i);
02767     bool Inserted = Done.insert(BB);
02768     if (!Inserted)
02769         continue;
02770 
02771     MachineBasicBlock *Succ = FuncInfo.MBBMap[BB];
02772     addSuccessorWithWeight(IndirectBrMBB, Succ);
02773   }
02774 
02775   DAG.setRoot(DAG.getNode(ISD::BRIND, getCurSDLoc(),
02776                           MVT::Other, getControlRoot(),
02777                           getValue(I.getAddress())));
02778 }
02779 
02780 void SelectionDAGBuilder::visitUnreachable(const UnreachableInst &I) {
02781   if (DAG.getTarget().Options.TrapUnreachable)
02782     DAG.setRoot(DAG.getNode(ISD::TRAP, getCurSDLoc(), MVT::Other, DAG.getRoot()));
02783 }
02784 
02785 void SelectionDAGBuilder::visitFSub(const User &I) {
02786   // -0.0 - X --> fneg
02787   Type *Ty = I.getType();
02788   if (isa<Constant>(I.getOperand(0)) &&
02789       I.getOperand(0) == ConstantFP::getZeroValueForNegation(Ty)) {
02790     SDValue Op2 = getValue(I.getOperand(1));
02791     setValue(&I, DAG.getNode(ISD::FNEG, getCurSDLoc(),
02792                              Op2.getValueType(), Op2));
02793     return;
02794   }
02795 
02796   visitBinary(I, ISD::FSUB);
02797 }
02798 
02799 void SelectionDAGBuilder::visitBinary(const User &I, unsigned OpCode) {
02800   SDValue Op1 = getValue(I.getOperand(0));
02801   SDValue Op2 = getValue(I.getOperand(1));
02802 
02803   bool nuw = false;
02804   bool nsw = false;
02805   bool exact = false;
02806   if (const OverflowingBinaryOperator *OFBinOp =
02807           dyn_cast<const OverflowingBinaryOperator>(&I)) {
02808     nuw = OFBinOp->hasNoUnsignedWrap();
02809     nsw = OFBinOp->hasNoSignedWrap();
02810   }
02811   if (const PossiblyExactOperator *ExactOp =
02812           dyn_cast<const PossiblyExactOperator>(&I))
02813     exact = ExactOp->isExact();
02814 
02815   SDValue BinNodeValue = DAG.getNode(OpCode, getCurSDLoc(), Op1.getValueType(),
02816                                      Op1, Op2, nuw, nsw, exact);
02817   setValue(&I, BinNodeValue);
02818 }
02819 
02820 void SelectionDAGBuilder::visitShift(const User &I, unsigned Opcode) {
02821   SDValue Op1 = getValue(I.getOperand(0));
02822   SDValue Op2 = getValue(I.getOperand(1));
02823 
02824   EVT ShiftTy = TM.getSubtargetImpl()->getTargetLowering()->getShiftAmountTy(
02825       Op2.getValueType());
02826 
02827   // Coerce the shift amount to the right type if we can.
02828   if (!I.getType()->isVectorTy() && Op2.getValueType() != ShiftTy) {
02829     unsigned ShiftSize = ShiftTy.getSizeInBits();
02830     unsigned Op2Size = Op2.getValueType().getSizeInBits();
02831     SDLoc DL = getCurSDLoc();
02832 
02833     // If the operand is smaller than the shift count type, promote it.
02834     if (ShiftSize > Op2Size)
02835       Op2 = DAG.getNode(ISD::ZERO_EXTEND, DL, ShiftTy, Op2);
02836 
02837     // If the operand is larger than the shift count type but the shift
02838     // count type has enough bits to represent any shift value, truncate
02839     // it now. This is a common case and it exposes the truncate to
02840     // optimization early.
02841     else if (ShiftSize >= Log2_32_Ceil(Op2.getValueType().getSizeInBits()))
02842       Op2 = DAG.getNode(ISD::TRUNCATE, DL, ShiftTy, Op2);
02843     // Otherwise we'll need to temporarily settle for some other convenient
02844     // type.  Type legalization will make adjustments once the shiftee is split.
02845     else
02846       Op2 = DAG.getZExtOrTrunc(Op2, DL, MVT::i32);
02847   }
02848 
02849   bool nuw = false;
02850   bool nsw = false;
02851   bool exact = false;
02852 
02853   if (Opcode == ISD::SRL || Opcode == ISD::SRA || Opcode == ISD::SHL) {
02854 
02855     if (const OverflowingBinaryOperator *OFBinOp =
02856             dyn_cast<const OverflowingBinaryOperator>(&I)) {
02857       nuw = OFBinOp->hasNoUnsignedWrap();
02858       nsw = OFBinOp->hasNoSignedWrap();
02859     }
02860     if (const PossiblyExactOperator *ExactOp =
02861             dyn_cast<const PossiblyExactOperator>(&I))
02862       exact = ExactOp->isExact();
02863   }
02864 
02865   SDValue Res = DAG.getNode(Opcode, getCurSDLoc(), Op1.getValueType(), Op1, Op2,
02866                             nuw, nsw, exact);
02867   setValue(&I, Res);
02868 }
02869 
02870 void SelectionDAGBuilder::visitSDiv(const User &I) {
02871   SDValue Op1 = getValue(I.getOperand(0));
02872   SDValue Op2 = getValue(I.getOperand(1));
02873 
02874   // Turn exact SDivs into multiplications.
02875   // FIXME: This should be in DAGCombiner, but it doesn't have access to the
02876   // exact bit.
02877   if (isa<BinaryOperator>(&I) && cast<BinaryOperator>(&I)->isExact() &&
02878       !isa<ConstantSDNode>(Op1) &&
02879       isa<ConstantSDNode>(Op2) && !cast<ConstantSDNode>(Op2)->isNullValue())
02880     setValue(&I, TM.getSubtargetImpl()->getTargetLowering()->BuildExactSDIV(
02881                      Op1, Op2, getCurSDLoc(), DAG));
02882   else
02883     setValue(&I, DAG.getNode(ISD::SDIV, getCurSDLoc(), Op1.getValueType(),
02884                              Op1, Op2));
02885 }
02886 
02887 void SelectionDAGBuilder::visitICmp(const User &I) {
02888   ICmpInst::Predicate predicate = ICmpInst::BAD_ICMP_PREDICATE;
02889   if (const ICmpInst *IC = dyn_cast<ICmpInst>(&I))
02890     predicate = IC->getPredicate();
02891   else if (const ConstantExpr *IC = dyn_cast<ConstantExpr>(&I))
02892     predicate = ICmpInst::Predicate(IC->getPredicate());
02893   SDValue Op1 = getValue(I.getOperand(0));
02894   SDValue Op2 = getValue(I.getOperand(1));
02895   ISD::CondCode Opcode = getICmpCondCode(predicate);
02896 
02897   EVT DestVT =
02898       TM.getSubtargetImpl()->getTargetLowering()->getValueType(I.getType());
02899   setValue(&I, DAG.getSetCC(getCurSDLoc(), DestVT, Op1, Op2, Opcode));
02900 }
02901 
02902 void SelectionDAGBuilder::visitFCmp(const User &I) {
02903   FCmpInst::Predicate predicate = FCmpInst::BAD_FCMP_PREDICATE;
02904   if (const FCmpInst *FC = dyn_cast<FCmpInst>(&I))
02905     predicate = FC->getPredicate();
02906   else if (const ConstantExpr *FC = dyn_cast<ConstantExpr>(&I))
02907     predicate = FCmpInst::Predicate(FC->getPredicate());
02908   SDValue Op1 = getValue(I.getOperand(0));
02909   SDValue Op2 = getValue(I.getOperand(1));
02910   ISD::CondCode Condition = getFCmpCondCode(predicate);
02911   if (TM.Options.NoNaNsFPMath)
02912     Condition = getFCmpCodeWithoutNaN(Condition);
02913   EVT DestVT =
02914       TM.getSubtargetImpl()->getTargetLowering()->getValueType(I.getType());
02915   setValue(&I, DAG.getSetCC(getCurSDLoc(), DestVT, Op1, Op2, Condition));
02916 }
02917 
02918 void SelectionDAGBuilder::visitSelect(const User &I) {
02919   SmallVector<EVT, 4> ValueVTs;
02920   ComputeValueVTs(*TM.getSubtargetImpl()->getTargetLowering(), I.getType(),
02921                   ValueVTs);
02922   unsigned NumValues = ValueVTs.size();
02923   if (NumValues == 0) return;
02924 
02925   SmallVector<SDValue, 4> Values(NumValues);
02926   SDValue Cond     = getValue(I.getOperand(0));
02927   SDValue TrueVal  = getValue(I.getOperand(1));
02928   SDValue FalseVal = getValue(I.getOperand(2));
02929   ISD::NodeType OpCode = Cond.getValueType().isVector() ?
02930     ISD::VSELECT : ISD::SELECT;
02931 
02932   for (unsigned i = 0; i != NumValues; ++i)
02933     Values[i] = DAG.getNode(OpCode, getCurSDLoc(),
02934                             TrueVal.getNode()->getValueType(TrueVal.getResNo()+i),
02935                             Cond,
02936                             SDValue(TrueVal.getNode(),
02937                                     TrueVal.getResNo() + i),
02938                             SDValue(FalseVal.getNode(),
02939                                     FalseVal.getResNo() + i));
02940 
02941   setValue(&I, DAG.getNode(ISD::MERGE_VALUES, getCurSDLoc(),
02942                            DAG.getVTList(ValueVTs), Values));
02943 }
02944 
02945 void SelectionDAGBuilder::visitTrunc(const User &I) {
02946   // TruncInst cannot be a no-op cast because sizeof(src) > sizeof(dest).
02947   SDValue N = getValue(I.getOperand(0));
02948   EVT DestVT =
02949       TM.getSubtargetImpl()->getTargetLowering()->getValueType(I.getType());
02950   setValue(&I, DAG.getNode(ISD::TRUNCATE, getCurSDLoc(), DestVT, N));
02951 }
02952 
02953 void SelectionDAGBuilder::visitZExt(const User &I) {
02954   // ZExt cannot be a no-op cast because sizeof(src) < sizeof(dest).
02955   // ZExt also can't be a cast to bool for same reason. So, nothing much to do
02956   SDValue N = getValue(I.getOperand(0));
02957   EVT DestVT =
02958       TM.getSubtargetImpl()->getTargetLowering()->getValueType(I.getType());
02959   setValue(&I, DAG.getNode(ISD::ZERO_EXTEND, getCurSDLoc(), DestVT, N));
02960 }
02961 
02962 void SelectionDAGBuilder::visitSExt(const User &I) {
02963   // SExt cannot be a no-op cast because sizeof(src) < sizeof(dest).
02964   // SExt also can't be a cast to bool for same reason. So, nothing much to do
02965   SDValue N = getValue(I.getOperand(0));
02966   EVT DestVT =
02967       TM.getSubtargetImpl()->getTargetLowering()->getValueType(I.getType());
02968   setValue(&I, DAG.getNode(ISD::SIGN_EXTEND, getCurSDLoc(), DestVT, N));
02969 }
02970 
02971 void SelectionDAGBuilder::visitFPTrunc(const User &I) {
02972   // FPTrunc is never a no-op cast, no need to check
02973   SDValue N = getValue(I.getOperand(0));
02974   const TargetLowering *TLI = TM.getSubtargetImpl()->getTargetLowering();
02975   EVT DestVT = TLI->getValueType(I.getType());
02976   setValue(&I, DAG.getNode(ISD::FP_ROUND, getCurSDLoc(),
02977                            DestVT, N,
02978                            DAG.getTargetConstant(0, TLI->getPointerTy())));
02979 }
02980 
02981 void SelectionDAGBuilder::visitFPExt(const User &I) {
02982   // FPExt is never a no-op cast, no need to check
02983   SDValue N = getValue(I.getOperand(0));
02984   EVT DestVT =
02985       TM.getSubtargetImpl()->getTargetLowering()->getValueType(I.getType());
02986   setValue(&I, DAG.getNode(ISD::FP_EXTEND, getCurSDLoc(), DestVT, N));
02987 }
02988 
02989 void SelectionDAGBuilder::visitFPToUI(const User &I) {
02990   // FPToUI is never a no-op cast, no need to check
02991   SDValue N = getValue(I.getOperand(0));
02992   EVT DestVT =
02993       TM.getSubtargetImpl()->getTargetLowering()->getValueType(I.getType());
02994   setValue(&I, DAG.getNode(ISD::FP_TO_UINT, getCurSDLoc(), DestVT, N));
02995 }
02996 
02997 void SelectionDAGBuilder::visitFPToSI(const User &I) {
02998   // FPToSI is never a no-op cast, no need to check
02999   SDValue N = getValue(I.getOperand(0));
03000   EVT DestVT =
03001       TM.getSubtargetImpl()->getTargetLowering()->getValueType(I.getType());
03002   setValue(&I, DAG.getNode(ISD::FP_TO_SINT, getCurSDLoc(), DestVT, N));
03003 }
03004 
03005 void SelectionDAGBuilder::visitUIToFP(const User &I) {
03006   // UIToFP is never a no-op cast, no need to check
03007   SDValue N = getValue(I.getOperand(0));
03008   EVT DestVT =
03009       TM.getSubtargetImpl()->getTargetLowering()->getValueType(I.getType());
03010   setValue(&I, DAG.getNode(ISD::UINT_TO_FP, getCurSDLoc(), DestVT, N));
03011 }
03012 
03013 void SelectionDAGBuilder::visitSIToFP(const User &I) {
03014   // SIToFP is never a no-op cast, no need to check
03015   SDValue N = getValue(I.getOperand(0));
03016   EVT DestVT =
03017       TM.getSubtargetImpl()->getTargetLowering()->getValueType(I.getType());
03018   setValue(&I, DAG.getNode(ISD::SINT_TO_FP, getCurSDLoc(), DestVT, N));
03019 }
03020 
03021 void SelectionDAGBuilder::visitPtrToInt(const User &I) {
03022   // What to do depends on the size of the integer and the size of the pointer.
03023   // We can either truncate, zero extend, or no-op, accordingly.
03024   SDValue N = getValue(I.getOperand(0));
03025   EVT DestVT =
03026       TM.getSubtargetImpl()->getTargetLowering()->getValueType(I.getType());
03027   setValue(&I, DAG.getZExtOrTrunc(N, getCurSDLoc(), DestVT));
03028 }
03029 
03030 void SelectionDAGBuilder::visitIntToPtr(const User &I) {
03031   // What to do depends on the size of the integer and the size of the pointer.
03032   // We can either truncate, zero extend, or no-op, accordingly.
03033   SDValue N = getValue(I.getOperand(0));
03034   EVT DestVT =
03035       TM.getSubtargetImpl()->getTargetLowering()->getValueType(I.getType());
03036   setValue(&I, DAG.getZExtOrTrunc(N, getCurSDLoc(), DestVT));
03037 }
03038 
03039 void SelectionDAGBuilder::visitBitCast(const User &I) {
03040   SDValue N = getValue(I.getOperand(0));
03041   EVT DestVT =
03042       TM.getSubtargetImpl()->getTargetLowering()->getValueType(I.getType());
03043 
03044   // BitCast assures us that source and destination are the same size so this is
03045   // either a BITCAST or a no-op.
03046   if (DestVT != N.getValueType())
03047     setValue(&I, DAG.getNode(ISD::BITCAST, getCurSDLoc(),
03048                              DestVT, N)); // convert types.
03049   // Check if the original LLVM IR Operand was a ConstantInt, because getValue()
03050   // might fold any kind of constant expression to an integer constant and that
03051   // is not what we are looking for. Only regcognize a bitcast of a genuine
03052   // constant integer as an opaque constant.
03053   else if(ConstantInt *C = dyn_cast<ConstantInt>(I.getOperand(0)))
03054     setValue(&I, DAG.getConstant(C->getValue(), DestVT, /*isTarget=*/false,
03055                                  /*isOpaque*/true));
03056   else
03057     setValue(&I, N);            // noop cast.
03058 }
03059 
03060 void SelectionDAGBuilder::visitAddrSpaceCast(const User &I) {
03061   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
03062   const Value *SV = I.getOperand(0);
03063   SDValue N = getValue(SV);
03064   EVT DestVT =
03065       TM.getSubtargetImpl()->getTargetLowering()->getValueType(I.getType());
03066 
03067   unsigned SrcAS = SV->getType()->getPointerAddressSpace();
03068   unsigned DestAS = I.getType()->getPointerAddressSpace();
03069 
03070   if (!TLI.isNoopAddrSpaceCast(SrcAS, DestAS))
03071     N = DAG.getAddrSpaceCast(getCurSDLoc(), DestVT, N, SrcAS, DestAS);
03072 
03073   setValue(&I, N);
03074 }
03075 
03076 void SelectionDAGBuilder::visitInsertElement(const User &I) {
03077   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
03078   SDValue InVec = getValue(I.getOperand(0));
03079   SDValue InVal = getValue(I.getOperand(1));
03080   SDValue InIdx = DAG.getSExtOrTrunc(getValue(I.getOperand(2)),
03081                                      getCurSDLoc(), TLI.getVectorIdxTy());
03082   setValue(&I,
03083            DAG.getNode(ISD::INSERT_VECTOR_ELT, getCurSDLoc(),
03084                        TM.getSubtargetImpl()->getTargetLowering()->getValueType(
03085                            I.getType()),
03086                        InVec, InVal, InIdx));
03087 }
03088 
03089 void SelectionDAGBuilder::visitExtractElement(const User &I) {
03090   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
03091   SDValue InVec = getValue(I.getOperand(0));
03092   SDValue InIdx = DAG.getSExtOrTrunc(getValue(I.getOperand(1)),
03093                                      getCurSDLoc(), TLI.getVectorIdxTy());
03094   setValue(&I,
03095            DAG.getNode(ISD::EXTRACT_VECTOR_ELT, getCurSDLoc(),
03096                        TM.getSubtargetImpl()->getTargetLowering()->getValueType(
03097                            I.getType()),
03098                        InVec, InIdx));
03099 }
03100 
03101 // Utility for visitShuffleVector - Return true if every element in Mask,
03102 // beginning from position Pos and ending in Pos+Size, falls within the
03103 // specified sequential range [L, L+Pos). or is undef.
03104 static bool isSequentialInRange(const SmallVectorImpl<int> &Mask,
03105                                 unsigned Pos, unsigned Size, int Low) {
03106   for (unsigned i = Pos, e = Pos+Size; i != e; ++i, ++Low)
03107     if (Mask[i] >= 0 && Mask[i] != Low)
03108       return false;
03109   return true;
03110 }
03111 
03112 void SelectionDAGBuilder::visitShuffleVector(const User &I) {
03113   SDValue Src1 = getValue(I.getOperand(0));
03114   SDValue Src2 = getValue(I.getOperand(1));
03115 
03116   SmallVector<int, 8> Mask;
03117   ShuffleVectorInst::getShuffleMask(cast<Constant>(I.getOperand(2)), Mask);
03118   unsigned MaskNumElts = Mask.size();
03119 
03120   const TargetLowering *TLI = TM.getSubtargetImpl()->getTargetLowering();
03121   EVT VT = TLI->getValueType(I.getType());
03122   EVT SrcVT = Src1.getValueType();
03123   unsigned SrcNumElts = SrcVT.getVectorNumElements();
03124 
03125   if (SrcNumElts == MaskNumElts) {
03126     setValue(&I, DAG.getVectorShuffle(VT, getCurSDLoc(), Src1, Src2,
03127                                       &Mask[0]));
03128     return;
03129   }
03130 
03131   // Normalize the shuffle vector since mask and vector length don't match.
03132   if (SrcNumElts < MaskNumElts && MaskNumElts % SrcNumElts == 0) {
03133     // Mask is longer than the source vectors and is a multiple of the source
03134     // vectors.  We can use concatenate vector to make the mask and vectors
03135     // lengths match.
03136     if (SrcNumElts*2 == MaskNumElts) {
03137       // First check for Src1 in low and Src2 in high
03138       if (isSequentialInRange(Mask, 0, SrcNumElts, 0) &&
03139           isSequentialInRange(Mask, SrcNumElts, SrcNumElts, SrcNumElts)) {
03140         // The shuffle is concatenating two vectors together.
03141         setValue(&I, DAG.getNode(ISD::CONCAT_VECTORS, getCurSDLoc(),
03142                                  VT, Src1, Src2));
03143         return;
03144       }
03145       // Then check for Src2 in low and Src1 in high
03146       if (isSequentialInRange(Mask, 0, SrcNumElts, SrcNumElts) &&
03147           isSequentialInRange(Mask, SrcNumElts, SrcNumElts, 0)) {
03148         // The shuffle is concatenating two vectors together.
03149         setValue(&I, DAG.getNode(ISD::CONCAT_VECTORS, getCurSDLoc(),
03150                                  VT, Src2, Src1));
03151         return;
03152       }
03153     }
03154 
03155     // Pad both vectors with undefs to make them the same length as the mask.
03156     unsigned NumConcat = MaskNumElts / SrcNumElts;
03157     bool Src1U = Src1.getOpcode() == ISD::UNDEF;
03158     bool Src2U = Src2.getOpcode() == ISD::UNDEF;
03159     SDValue UndefVal = DAG.getUNDEF(SrcVT);
03160 
03161     SmallVector<SDValue, 8> MOps1(NumConcat, UndefVal);
03162     SmallVector<SDValue, 8> MOps2(NumConcat, UndefVal);
03163     MOps1[0] = Src1;
03164     MOps2[0] = Src2;
03165 
03166     Src1 = Src1U ? DAG.getUNDEF(VT) : DAG.getNode(ISD::CONCAT_VECTORS,
03167                                                   getCurSDLoc(), VT, MOps1);
03168     Src2 = Src2U ? DAG.getUNDEF(VT) : DAG.getNode(ISD::CONCAT_VECTORS,
03169                                                   getCurSDLoc(), VT, MOps2);
03170 
03171     // Readjust mask for new input vector length.
03172     SmallVector<int, 8> MappedOps;
03173     for (unsigned i = 0; i != MaskNumElts; ++i) {
03174       int Idx = Mask[i];
03175       if (Idx >= (int)SrcNumElts)
03176         Idx -= SrcNumElts - MaskNumElts;
03177       MappedOps.push_back(Idx);
03178     }
03179 
03180     setValue(&I, DAG.getVectorShuffle(VT, getCurSDLoc(), Src1, Src2,
03181                                       &MappedOps[0]));
03182     return;
03183   }
03184 
03185   if (SrcNumElts > MaskNumElts) {
03186     // Analyze the access pattern of the vector to see if we can extract
03187     // two subvectors and do the shuffle. The analysis is done by calculating
03188     // the range of elements the mask access on both vectors.
03189     int MinRange[2] = { static_cast<int>(SrcNumElts),
03190                         static_cast<int>(SrcNumElts)};
03191     int MaxRange[2] = {-1, -1};
03192 
03193     for (unsigned i = 0; i != MaskNumElts; ++i) {
03194       int Idx = Mask[i];
03195       unsigned Input = 0;
03196       if (Idx < 0)
03197         continue;
03198 
03199       if (Idx >= (int)SrcNumElts) {
03200         Input = 1;
03201         Idx -= SrcNumElts;
03202       }
03203       if (Idx > MaxRange[Input])
03204         MaxRange[Input] = Idx;
03205       if (Idx < MinRange[Input])
03206         MinRange[Input] = Idx;
03207     }
03208 
03209     // Check if the access is smaller than the vector size and can we find
03210     // a reasonable extract index.
03211     int RangeUse[2] = { -1, -1 };  // 0 = Unused, 1 = Extract, -1 = Can not
03212                                    // Extract.
03213     int StartIdx[2];  // StartIdx to extract from
03214     for (unsigned Input = 0; Input < 2; ++Input) {
03215       if (MinRange[Input] >= (int)SrcNumElts && MaxRange[Input] < 0) {
03216         RangeUse[Input] = 0; // Unused
03217         StartIdx[Input] = 0;
03218         continue;
03219       }
03220 
03221       // Find a good start index that is a multiple of the mask length. Then
03222       // see if the rest of the elements are in range.
03223       StartIdx[Input] = (MinRange[Input]/MaskNumElts)*MaskNumElts;
03224       if (MaxRange[Input] - StartIdx[Input] < (int)MaskNumElts &&
03225           StartIdx[Input] + MaskNumElts <= SrcNumElts)
03226         RangeUse[Input] = 1; // Extract from a multiple of the mask length.
03227     }
03228 
03229     if (RangeUse[0] == 0 && RangeUse[1] == 0) {
03230       setValue(&I, DAG.getUNDEF(VT)); // Vectors are not used.
03231       return;
03232     }
03233     if (RangeUse[0] >= 0 && RangeUse[1] >= 0) {
03234       // Extract appropriate subvector and generate a vector shuffle
03235       for (unsigned Input = 0; Input < 2; ++Input) {
03236         SDValue &Src = Input == 0 ? Src1 : Src2;
03237         if (RangeUse[Input] == 0)
03238           Src = DAG.getUNDEF(VT);
03239         else
03240           Src = DAG.getNode(ISD::EXTRACT_SUBVECTOR, getCurSDLoc(), VT,
03241                             Src, DAG.getConstant(StartIdx[Input],
03242                                                  TLI->getVectorIdxTy()));
03243       }
03244 
03245       // Calculate new mask.
03246       SmallVector<int, 8> MappedOps;
03247       for (unsigned i = 0; i != MaskNumElts; ++i) {
03248         int Idx = Mask[i];
03249         if (Idx >= 0) {
03250           if (Idx < (int)SrcNumElts)
03251             Idx -= StartIdx[0];
03252           else
03253             Idx -= SrcNumElts + StartIdx[1] - MaskNumElts;
03254         }
03255         MappedOps.push_back(Idx);
03256       }
03257 
03258       setValue(&I, DAG.getVectorShuffle(VT, getCurSDLoc(), Src1, Src2,
03259                                         &MappedOps[0]));
03260       return;
03261     }
03262   }
03263 
03264   // We can't use either concat vectors or extract subvectors so fall back to
03265   // replacing the shuffle with extract and build vector.
03266   // to insert and build vector.
03267   EVT EltVT = VT.getVectorElementType();
03268   EVT IdxVT = TLI->getVectorIdxTy();
03269   SmallVector<SDValue,8> Ops;
03270   for (unsigned i = 0; i != MaskNumElts; ++i) {
03271     int Idx = Mask[i];
03272     SDValue Res;
03273 
03274     if (Idx < 0) {
03275       Res = DAG.getUNDEF(EltVT);
03276     } else {
03277       SDValue &Src = Idx < (int)SrcNumElts ? Src1 : Src2;
03278       if (Idx >= (int)SrcNumElts) Idx -= SrcNumElts;
03279 
03280       Res = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, getCurSDLoc(),
03281                         EltVT, Src, DAG.getConstant(Idx, IdxVT));
03282     }
03283 
03284     Ops.push_back(Res);
03285   }
03286 
03287   setValue(&I, DAG.getNode(ISD::BUILD_VECTOR, getCurSDLoc(), VT, Ops));
03288 }
03289 
03290 void SelectionDAGBuilder::visitInsertValue(const InsertValueInst &I) {
03291   const Value *Op0 = I.getOperand(0);
03292   const Value *Op1 = I.getOperand(1);
03293   Type *AggTy = I.getType();
03294   Type *ValTy = Op1->getType();
03295   bool IntoUndef = isa<UndefValue>(Op0);
03296   bool FromUndef = isa<UndefValue>(Op1);
03297 
03298   unsigned LinearIndex = ComputeLinearIndex(AggTy, I.getIndices());
03299 
03300   const TargetLowering *TLI = TM.getSubtargetImpl()->getTargetLowering();
03301   SmallVector<EVT, 4> AggValueVTs;
03302   ComputeValueVTs(*TLI, AggTy, AggValueVTs);
03303   SmallVector<EVT, 4> ValValueVTs;
03304   ComputeValueVTs(*TLI, ValTy, ValValueVTs);
03305 
03306   unsigned NumAggValues = AggValueVTs.size();
03307   unsigned NumValValues = ValValueVTs.size();
03308   SmallVector<SDValue, 4> Values(NumAggValues);
03309 
03310   // Ignore an insertvalue that produces an empty object
03311   if (!NumAggValues) {
03312     setValue(&I, DAG.getUNDEF(MVT(MVT::Other)));
03313     return;
03314   }
03315 
03316   SDValue Agg = getValue(Op0);
03317   unsigned i = 0;
03318   // Copy the beginning value(s) from the original aggregate.
03319   for (; i != LinearIndex; ++i)
03320     Values[i] = IntoUndef ? DAG.getUNDEF(AggValueVTs[i]) :
03321                 SDValue(Agg.getNode(), Agg.getResNo() + i);
03322   // Copy values from the inserted value(s).
03323   if (NumValValues) {
03324     SDValue Val = getValue(Op1);
03325     for (; i != LinearIndex + NumValValues; ++i)
03326       Values[i] = FromUndef ? DAG.getUNDEF(AggValueVTs[i]) :
03327                   SDValue(Val.getNode(), Val.getResNo() + i - LinearIndex);
03328   }
03329   // Copy remaining value(s) from the original aggregate.
03330   for (; i != NumAggValues; ++i)
03331     Values[i] = IntoUndef ? DAG.getUNDEF(AggValueVTs[i]) :
03332                 SDValue(Agg.getNode(), Agg.getResNo() + i);
03333 
03334   setValue(&I, DAG.getNode(ISD::MERGE_VALUES, getCurSDLoc(),
03335                            DAG.getVTList(AggValueVTs), Values));
03336 }
03337 
03338 void SelectionDAGBuilder::visitExtractValue(const ExtractValueInst &I) {
03339   const Value *Op0 = I.getOperand(0);
03340   Type *AggTy = Op0->getType();
03341   Type *ValTy = I.getType();
03342   bool OutOfUndef = isa<UndefValue>(Op0);
03343 
03344   unsigned LinearIndex = ComputeLinearIndex(AggTy, I.getIndices());
03345 
03346   const TargetLowering *TLI = TM.getSubtargetImpl()->getTargetLowering();
03347   SmallVector<EVT, 4> ValValueVTs;
03348   ComputeValueVTs(*TLI, ValTy, ValValueVTs);
03349 
03350   unsigned NumValValues = ValValueVTs.size();
03351 
03352   // Ignore a extractvalue that produces an empty object
03353   if (!NumValValues) {
03354     setValue(&I, DAG.getUNDEF(MVT(MVT::Other)));
03355     return;
03356   }
03357 
03358   SmallVector<SDValue, 4> Values(NumValValues);
03359 
03360   SDValue Agg = getValue(Op0);
03361   // Copy out the selected value(s).
03362   for (unsigned i = LinearIndex; i != LinearIndex + NumValValues; ++i)
03363     Values[i - LinearIndex] =
03364       OutOfUndef ?
03365         DAG.getUNDEF(Agg.getNode()->getValueType(Agg.getResNo() + i)) :
03366         SDValue(Agg.getNode(), Agg.getResNo() + i);
03367 
03368   setValue(&I, DAG.getNode(ISD::MERGE_VALUES, getCurSDLoc(),
03369                            DAG.getVTList(ValValueVTs), Values));
03370 }
03371 
03372 void SelectionDAGBuilder::visitGetElementPtr(const User &I) {
03373   Value *Op0 = I.getOperand(0);
03374   // Note that the pointer operand may be a vector of pointers. Take the scalar
03375   // element which holds a pointer.
03376   Type *Ty = Op0->getType()->getScalarType();
03377   unsigned AS = Ty->getPointerAddressSpace();
03378   SDValue N = getValue(Op0);
03379 
03380   for (GetElementPtrInst::const_op_iterator OI = I.op_begin()+1, E = I.op_end();
03381        OI != E; ++OI) {
03382     const Value *Idx = *OI;
03383     if (StructType *StTy = dyn_cast<StructType>(Ty)) {
03384       unsigned Field = cast<Constant>(Idx)->getUniqueInteger().getZExtValue();
03385       if (Field) {
03386         // N = N + Offset
03387         uint64_t Offset = DL->getStructLayout(StTy)->getElementOffset(Field);
03388         N = DAG.getNode(ISD::ADD, getCurSDLoc(), N.getValueType(), N,
03389                         DAG.getConstant(Offset, N.getValueType()));
03390       }
03391 
03392       Ty = StTy->getElementType(Field);
03393     } else {
03394       Ty = cast<SequentialType>(Ty)->getElementType();
03395 
03396       // If this is a constant subscript, handle it quickly.
03397       const TargetLowering *TLI = TM.getSubtargetImpl()->getTargetLowering();
03398       if (const ConstantInt *CI = dyn_cast<ConstantInt>(Idx)) {
03399         if (CI->isZero()) continue;
03400         uint64_t Offs =
03401             DL->getTypeAllocSize(Ty)*cast<ConstantInt>(CI)->getSExtValue();
03402         SDValue OffsVal;
03403         EVT PTy = TLI->getPointerTy(AS);
03404         unsigned PtrBits = PTy.getSizeInBits();
03405         if (PtrBits < 64)
03406           OffsVal = DAG.getNode(ISD::TRUNCATE, getCurSDLoc(), PTy,
03407                                 DAG.getConstant(Offs, MVT::i64));
03408         else
03409           OffsVal = DAG.getConstant(Offs, PTy);
03410 
03411         N = DAG.getNode(ISD::ADD, getCurSDLoc(), N.getValueType(), N,
03412                         OffsVal);
03413         continue;
03414       }
03415 
03416       // N = N + Idx * ElementSize;
03417       APInt ElementSize = APInt(TLI->getPointerSizeInBits(AS),
03418                                 DL->getTypeAllocSize(Ty));
03419       SDValue IdxN = getValue(Idx);
03420 
03421       // If the index is smaller or larger than intptr_t, truncate or extend
03422       // it.
03423       IdxN = DAG.getSExtOrTrunc(IdxN, getCurSDLoc(), N.getValueType());
03424 
03425       // If this is a multiply by a power of two, turn it into a shl
03426       // immediately.  This is a very common case.
03427       if (ElementSize != 1) {
03428         if (ElementSize.isPowerOf2()) {
03429           unsigned Amt = ElementSize.logBase2();
03430           IdxN = DAG.getNode(ISD::SHL, getCurSDLoc(),
03431                              N.getValueType(), IdxN,
03432                              DAG.getConstant(Amt, IdxN.getValueType()));
03433         } else {
03434           SDValue Scale = DAG.getConstant(ElementSize, IdxN.getValueType());
03435           IdxN = DAG.getNode(ISD::MUL, getCurSDLoc(),
03436                              N.getValueType(), IdxN, Scale);
03437         }
03438       }
03439 
03440       N = DAG.getNode(ISD::ADD, getCurSDLoc(),
03441                       N.getValueType(), N, IdxN);
03442     }
03443   }
03444 
03445   setValue(&I, N);
03446 }
03447 
03448 void SelectionDAGBuilder::visitAlloca(const AllocaInst &I) {
03449   // If this is a fixed sized alloca in the entry block of the function,
03450   // allocate it statically on the stack.
03451   if (FuncInfo.StaticAllocaMap.count(&I))
03452     return;   // getValue will auto-populate this.
03453 
03454   Type *Ty = I.getAllocatedType();
03455   const TargetLowering *TLI = TM.getSubtargetImpl()->getTargetLowering();
03456   uint64_t TySize = TLI->getDataLayout()->getTypeAllocSize(Ty);
03457   unsigned Align =
03458     std::max((unsigned)TLI->getDataLayout()->getPrefTypeAlignment(Ty),
03459              I.getAlignment());
03460 
03461   SDValue AllocSize = getValue(I.getArraySize());
03462 
03463   EVT IntPtr = TLI->getPointerTy();
03464   if (AllocSize.getValueType() != IntPtr)
03465     AllocSize = DAG.getZExtOrTrunc(AllocSize, getCurSDLoc(), IntPtr);
03466 
03467   AllocSize = DAG.getNode(ISD::MUL, getCurSDLoc(), IntPtr,
03468                           AllocSize,
03469                           DAG.getConstant(TySize, IntPtr));
03470 
03471   // Handle alignment.  If the requested alignment is less than or equal to
03472   // the stack alignment, ignore it.  If the size is greater than or equal to
03473   // the stack alignment, we note this in the DYNAMIC_STACKALLOC node.
03474   unsigned StackAlign =
03475       TM.getSubtargetImpl()->getFrameLowering()->getStackAlignment();
03476   if (Align <= StackAlign)
03477     Align = 0;
03478 
03479   // Round the size of the allocation up to the stack alignment size
03480   // by add SA-1 to the size.
03481   AllocSize = DAG.getNode(ISD::ADD, getCurSDLoc(),
03482                           AllocSize.getValueType(), AllocSize,
03483                           DAG.getIntPtrConstant(StackAlign-1));
03484 
03485   // Mask out the low bits for alignment purposes.
03486   AllocSize = DAG.getNode(ISD::AND, getCurSDLoc(),
03487                           AllocSize.getValueType(), AllocSize,
03488                           DAG.getIntPtrConstant(~(uint64_t)(StackAlign-1)));
03489 
03490   SDValue Ops[] = { getRoot(), AllocSize, DAG.getIntPtrConstant(Align) };
03491   SDVTList VTs = DAG.getVTList(AllocSize.getValueType(), MVT::Other);
03492   SDValue DSA = DAG.getNode(ISD::DYNAMIC_STACKALLOC, getCurSDLoc(), VTs, Ops);
03493   setValue(&I, DSA);
03494   DAG.setRoot(DSA.getValue(1));
03495 
03496   assert(FuncInfo.MF->getFrameInfo()->hasVarSizedObjects());
03497 }
03498 
03499 void SelectionDAGBuilder::visitLoad(const LoadInst &I) {
03500   if (I.isAtomic())
03501     return visitAtomicLoad(I);
03502 
03503   const Value *SV = I.getOperand(0);
03504   SDValue Ptr = getValue(SV);
03505 
03506   Type *Ty = I.getType();
03507 
03508   bool isVolatile = I.isVolatile();
03509   bool isNonTemporal = I.getMetadata("nontemporal") != nullptr;
03510   bool isInvariant = I.getMetadata("invariant.load") != nullptr;
03511   unsigned Alignment = I.getAlignment();
03512 
03513   AAMDNodes AAInfo;
03514   I.getAAMetadata(AAInfo);
03515   const MDNode *Ranges = I.getMetadata(LLVMContext::MD_range);
03516 
03517   SmallVector<EVT, 4> ValueVTs;
03518   SmallVector<uint64_t, 4> Offsets;
03519   ComputeValueVTs(*TM.getSubtargetImpl()->getTargetLowering(), Ty, ValueVTs,
03520                   &Offsets);
03521   unsigned NumValues = ValueVTs.size();
03522   if (NumValues == 0)
03523     return;
03524 
03525   SDValue Root;
03526   bool ConstantMemory = false;
03527   if (isVolatile || NumValues > MaxParallelChains)
03528     // Serialize volatile loads with other side effects.
03529     Root = getRoot();
03530   else if (AA->pointsToConstantMemory(
03531              AliasAnalysis::Location(SV, AA->getTypeStoreSize(Ty), AAInfo))) {
03532     // Do not serialize (non-volatile) loads of constant memory with anything.
03533     Root = DAG.getEntryNode();
03534     ConstantMemory = true;
03535   } else {
03536     // Do not serialize non-volatile loads against each other.
03537     Root = DAG.getRoot();
03538   }
03539 
03540   const TargetLowering *TLI = TM.getSubtargetImpl()->getTargetLowering();
03541   if (isVolatile)
03542     Root = TLI->prepareVolatileOrAtomicLoad(Root, getCurSDLoc(), DAG);
03543 
03544   SmallVector<SDValue, 4> Values(NumValues);
03545   SmallVector<SDValue, 4> Chains(std::min(unsigned(MaxParallelChains),
03546                                           NumValues));
03547   EVT PtrVT = Ptr.getValueType();
03548   unsigned ChainI = 0;
03549   for (unsigned i = 0; i != NumValues; ++i, ++ChainI) {
03550     // Serializing loads here may result in excessive register pressure, and
03551     // TokenFactor places arbitrary choke points on the scheduler. SD scheduling
03552     // could recover a bit by hoisting nodes upward in the chain by recognizing
03553     // they are side-effect free or do not alias. The optimizer should really
03554     // avoid this case by converting large object/array copies to llvm.memcpy
03555     // (MaxParallelChains should always remain as failsafe).
03556     if (ChainI == MaxParallelChains) {
03557       assert(PendingLoads.empty() && "PendingLoads must be serialized first");
03558       SDValue Chain = DAG.getNode(ISD::TokenFactor, getCurSDLoc(), MVT::Other,
03559                                   makeArrayRef(Chains.data(), ChainI));
03560       Root = Chain;
03561       ChainI = 0;
03562     }
03563     SDValue A = DAG.getNode(ISD::ADD, getCurSDLoc(),
03564                             PtrVT, Ptr,
03565                             DAG.getConstant(Offsets[i], PtrVT));
03566     SDValue L = DAG.getLoad(ValueVTs[i], getCurSDLoc(), Root,
03567                             A, MachinePointerInfo(SV, Offsets[i]), isVolatile,
03568                             isNonTemporal, isInvariant, Alignment, AAInfo,
03569                             Ranges);
03570 
03571     Values[i] = L;
03572     Chains[ChainI] = L.getValue(1);
03573   }
03574 
03575   if (!ConstantMemory) {
03576     SDValue Chain = DAG.getNode(ISD::TokenFactor, getCurSDLoc(), MVT::Other,
03577                                 makeArrayRef(Chains.data(), ChainI));
03578     if (isVolatile)
03579       DAG.setRoot(Chain);
03580     else
03581       PendingLoads.push_back(Chain);
03582   }
03583 
03584   setValue(&I, DAG.getNode(ISD::MERGE_VALUES, getCurSDLoc(),
03585                            DAG.getVTList(ValueVTs), Values));
03586 }
03587 
03588 void SelectionDAGBuilder::visitStore(const StoreInst &I) {
03589   if (I.isAtomic())
03590     return visitAtomicStore(I);
03591 
03592   const Value *SrcV = I.getOperand(0);
03593   const Value *PtrV = I.getOperand(1);
03594 
03595   SmallVector<EVT, 4> ValueVTs;
03596   SmallVector<uint64_t, 4> Offsets;
03597   ComputeValueVTs(*TM.getSubtargetImpl()->getTargetLowering(), SrcV->getType(),
03598                   ValueVTs, &Offsets);
03599   unsigned NumValues = ValueVTs.size();
03600   if (NumValues == 0)
03601     return;
03602 
03603   // Get the lowered operands. Note that we do this after
03604   // checking if NumResults is zero, because with zero results
03605   // the operands won't have values in the map.
03606   SDValue Src = getValue(SrcV);
03607   SDValue Ptr = getValue(PtrV);
03608 
03609   SDValue Root = getRoot();
03610   SmallVector<SDValue, 4> Chains(std::min(unsigned(MaxParallelChains),
03611                                           NumValues));
03612   EVT PtrVT = Ptr.getValueType();
03613   bool isVolatile = I.isVolatile();
03614   bool isNonTemporal = I.getMetadata("nontemporal") != nullptr;
03615   unsigned Alignment = I.getAlignment();
03616 
03617   AAMDNodes AAInfo;
03618   I.getAAMetadata(AAInfo);
03619 
03620   unsigned ChainI = 0;
03621   for (unsigned i = 0; i != NumValues; ++i, ++ChainI) {
03622     // See visitLoad comments.
03623     if (ChainI == MaxParallelChains) {
03624       SDValue Chain = DAG.getNode(ISD::TokenFactor, getCurSDLoc(), MVT::Other,
03625                                   makeArrayRef(Chains.data(), ChainI));
03626       Root = Chain;
03627       ChainI = 0;
03628     }
03629     SDValue Add = DAG.getNode(ISD::ADD, getCurSDLoc(), PtrVT, Ptr,
03630                               DAG.getConstant(Offsets[i], PtrVT));
03631     SDValue St = DAG.getStore(Root, getCurSDLoc(),
03632                               SDValue(Src.getNode(), Src.getResNo() + i),
03633                               Add, MachinePointerInfo(PtrV, Offsets[i]),
03634                               isVolatile, isNonTemporal, Alignment, AAInfo);
03635     Chains[ChainI] = St;
03636   }
03637 
03638   SDValue StoreNode = DAG.getNode(ISD::TokenFactor, getCurSDLoc(), MVT::Other,
03639                                   makeArrayRef(Chains.data(), ChainI));
03640   DAG.setRoot(StoreNode);
03641 }
03642 
03643 static SDValue InsertFenceForAtomic(SDValue Chain, AtomicOrdering Order,
03644                                     SynchronizationScope Scope,
03645                                     bool Before, SDLoc dl,
03646                                     SelectionDAG &DAG,
03647                                     const TargetLowering &TLI) {
03648   // Fence, if necessary
03649   if (Before) {
03650     if (Order == AcquireRelease || Order == SequentiallyConsistent)
03651       Order = Release;
03652     else if (Order == Acquire || Order == Monotonic || Order == Unordered)
03653       return Chain;
03654   } else {
03655     if (Order == AcquireRelease)
03656       Order = Acquire;
03657     else if (Order == Release || Order == Monotonic || Order == Unordered)
03658       return Chain;
03659   }
03660   SDValue Ops[3];
03661   Ops[0] = Chain;
03662   Ops[1] = DAG.getConstant(Order, TLI.getPointerTy());
03663   Ops[2] = DAG.getConstant(Scope, TLI.getPointerTy());
03664   return DAG.getNode(ISD::ATOMIC_FENCE, dl, MVT::Other, Ops);
03665 }
03666 
03667 void SelectionDAGBuilder::visitAtomicCmpXchg(const AtomicCmpXchgInst &I) {
03668   SDLoc dl = getCurSDLoc();
03669   AtomicOrdering SuccessOrder = I.getSuccessOrdering();
03670   AtomicOrdering FailureOrder = I.getFailureOrdering();
03671   SynchronizationScope Scope = I.getSynchScope();
03672 
03673   SDValue InChain = getRoot();
03674 
03675   const TargetLowering *TLI = TM.getSubtargetImpl()->getTargetLowering();
03676   if (TLI->getInsertFencesForAtomic())
03677     InChain = InsertFenceForAtomic(InChain, SuccessOrder, Scope, true, dl,
03678                                    DAG, *TLI);
03679 
03680   MVT MemVT = getValue(I.getCompareOperand()).getSimpleValueType();
03681   SDVTList VTs = DAG.getVTList(MemVT, MVT::i1, MVT::Other);
03682   SDValue L = DAG.getAtomicCmpSwap(
03683       ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS, dl, MemVT, VTs, InChain,
03684       getValue(I.getPointerOperand()), getValue(I.getCompareOperand()),
03685       getValue(I.getNewValOperand()), MachinePointerInfo(I.getPointerOperand()),
03686       0 /* Alignment */,
03687       TLI->getInsertFencesForAtomic() ? Monotonic : SuccessOrder,
03688       TLI->getInsertFencesForAtomic() ? Monotonic : FailureOrder, Scope);
03689 
03690   SDValue OutChain = L.getValue(2);
03691 
03692   if (TLI->getInsertFencesForAtomic())
03693     OutChain = InsertFenceForAtomic(OutChain, SuccessOrder, Scope, false, dl,
03694                                     DAG, *TLI);
03695 
03696   setValue(&I, L);
03697   DAG.setRoot(OutChain);
03698 }
03699 
03700 void SelectionDAGBuilder::visitAtomicRMW(const AtomicRMWInst &I) {
03701   SDLoc dl = getCurSDLoc();
03702   ISD::NodeType NT;
03703   switch (I.getOperation()) {
03704   default: llvm_unreachable("Unknown atomicrmw operation");
03705   case AtomicRMWInst::Xchg: NT = ISD::ATOMIC_SWAP; break;
03706   case AtomicRMWInst::Add:  NT = ISD::ATOMIC_LOAD_ADD; break;
03707   case AtomicRMWInst::Sub:  NT = ISD::ATOMIC_LOAD_SUB; break;
03708   case AtomicRMWInst::And:  NT = ISD::ATOMIC_LOAD_AND; break;
03709   case AtomicRMWInst::Nand: NT = ISD::ATOMIC_LOAD_NAND; break;
03710   case AtomicRMWInst::Or:   NT = ISD::ATOMIC_LOAD_OR; break;
03711   case AtomicRMWInst::Xor:  NT = ISD::ATOMIC_LOAD_XOR; break;
03712   case AtomicRMWInst::Max:  NT = ISD::ATOMIC_LOAD_MAX; break;
03713   case AtomicRMWInst::Min:  NT = ISD::ATOMIC_LOAD_MIN; break;
03714   case AtomicRMWInst::UMax: NT = ISD::ATOMIC_LOAD_UMAX; break;
03715   case AtomicRMWInst::UMin: NT = ISD::ATOMIC_LOAD_UMIN; break;
03716   }
03717   AtomicOrdering Order = I.getOrdering();
03718   SynchronizationScope Scope = I.getSynchScope();
03719 
03720   SDValue InChain = getRoot();
03721 
03722   const TargetLowering *TLI = TM.getSubtargetImpl()->getTargetLowering();
03723   if (TLI->getInsertFencesForAtomic())
03724     InChain = InsertFenceForAtomic(InChain, Order, Scope, true, dl,
03725                                    DAG, *TLI);
03726 
03727   SDValue L =
03728     DAG.getAtomic(NT, dl,
03729                   getValue(I.getValOperand()).getSimpleValueType(),
03730                   InChain,
03731                   getValue(I.getPointerOperand()),
03732                   getValue(I.getValOperand()),
03733                   I.getPointerOperand(), 0 /* Alignment */,
03734                   TLI->getInsertFencesForAtomic() ? Monotonic : Order,
03735                   Scope);
03736 
03737   SDValue OutChain = L.getValue(1);
03738 
03739   if (TLI->getInsertFencesForAtomic())
03740     OutChain = InsertFenceForAtomic(OutChain, Order, Scope, false, dl,
03741                                     DAG, *TLI);
03742 
03743   setValue(&I, L);
03744   DAG.setRoot(OutChain);
03745 }
03746 
03747 void SelectionDAGBuilder::visitFence(const FenceInst &I) {
03748   SDLoc dl = getCurSDLoc();
03749   const TargetLowering *TLI = TM.getSubtargetImpl()->getTargetLowering();
03750   SDValue Ops[3];
03751   Ops[0] = getRoot();
03752   Ops[1] = DAG.getConstant(I.getOrdering(), TLI->getPointerTy());
03753   Ops[2] = DAG.getConstant(I.getSynchScope(), TLI->getPointerTy());
03754   DAG.setRoot(DAG.getNode(ISD::ATOMIC_FENCE, dl, MVT::Other, Ops));
03755 }
03756 
03757 void SelectionDAGBuilder::visitAtomicLoad(const LoadInst &I) {
03758   SDLoc dl = getCurSDLoc();
03759   AtomicOrdering Order = I.getOrdering();
03760   SynchronizationScope Scope = I.getSynchScope();
03761 
03762   SDValue InChain = getRoot();
03763 
03764   const TargetLowering *TLI = TM.getSubtargetImpl()->getTargetLowering();
03765   EVT VT = TLI->getValueType(I.getType());
03766 
03767   if (I.getAlignment() < VT.getSizeInBits() / 8)
03768     report_fatal_error("Cannot generate unaligned atomic load");
03769 
03770   MachineMemOperand *MMO =
03771       DAG.getMachineFunction().
03772       getMachineMemOperand(MachinePointerInfo(I.getPointerOperand()),
03773                            MachineMemOperand::MOVolatile |
03774                            MachineMemOperand::MOLoad,
03775                            VT.getStoreSize(),
03776                            I.getAlignment() ? I.getAlignment() :
03777                                               DAG.getEVTAlignment(VT));
03778 
03779   InChain = TLI->prepareVolatileOrAtomicLoad(InChain, dl, DAG);
03780   SDValue L =
03781       DAG.getAtomic(ISD::ATOMIC_LOAD, dl, VT, VT, InChain,
03782                     getValue(I.getPointerOperand()), MMO,
03783                     TLI->getInsertFencesForAtomic() ? Monotonic : Order,
03784                     Scope);
03785 
03786   SDValue OutChain = L.getValue(1);
03787 
03788   if (TLI->getInsertFencesForAtomic())
03789     OutChain = InsertFenceForAtomic(OutChain, Order, Scope, false, dl,
03790                                     DAG, *TLI);
03791 
03792   setValue(&I, L);
03793   DAG.setRoot(OutChain);
03794 }
03795 
03796 void SelectionDAGBuilder::visitAtomicStore(const StoreInst &I) {
03797   SDLoc dl = getCurSDLoc();
03798 
03799   AtomicOrdering Order = I.getOrdering();
03800   SynchronizationScope Scope = I.getSynchScope();
03801 
03802   SDValue InChain = getRoot();
03803 
03804   const TargetLowering *TLI = TM.getSubtargetImpl()->getTargetLowering();
03805   EVT VT = TLI->getValueType(I.getValueOperand()->getType());
03806 
03807   if (I.getAlignment() < VT.getSizeInBits() / 8)
03808     report_fatal_error("Cannot generate unaligned atomic store");
03809 
03810   if (TLI->getInsertFencesForAtomic())
03811     InChain = InsertFenceForAtomic(InChain, Order, Scope, true, dl,
03812                                    DAG, *TLI);
03813 
03814   SDValue OutChain =
03815     DAG.getAtomic(ISD::ATOMIC_STORE, dl, VT,
03816                   InChain,
03817                   getValue(I.getPointerOperand()),
03818                   getValue(I.getValueOperand()),
03819                   I.getPointerOperand(), I.getAlignment(),
03820                   TLI->getInsertFencesForAtomic() ? Monotonic : Order,
03821                   Scope);
03822 
03823   if (TLI->getInsertFencesForAtomic())
03824     OutChain = InsertFenceForAtomic(OutChain, Order, Scope, false, dl,
03825                                     DAG, *TLI);
03826 
03827   DAG.setRoot(OutChain);
03828 }
03829 
03830 /// visitTargetIntrinsic - Lower a call of a target intrinsic to an INTRINSIC
03831 /// node.
03832 void SelectionDAGBuilder::visitTargetIntrinsic(const CallInst &I,
03833                                                unsigned Intrinsic) {
03834   bool HasChain = !I.doesNotAccessMemory();
03835   bool OnlyLoad = HasChain && I.onlyReadsMemory();
03836 
03837   // Build the operand list.
03838   SmallVector<SDValue, 8> Ops;
03839   if (HasChain) {  // If this intrinsic has side-effects, chainify it.
03840     if (OnlyLoad) {
03841       // We don't need to serialize loads against other loads.
03842       Ops.push_back(DAG.getRoot());
03843     } else {
03844       Ops.push_back(getRoot());
03845     }
03846   }
03847 
03848   // Info is set by getTgtMemInstrinsic
03849   TargetLowering::IntrinsicInfo Info;
03850   const TargetLowering *TLI = TM.getSubtargetImpl()->getTargetLowering();
03851   bool IsTgtIntrinsic = TLI->getTgtMemIntrinsic(Info, I, Intrinsic);
03852 
03853   // Add the intrinsic ID as an integer operand if it's not a target intrinsic.
03854   if (!IsTgtIntrinsic || Info.opc == ISD::INTRINSIC_VOID ||
03855       Info.opc == ISD::INTRINSIC_W_CHAIN)
03856     Ops.push_back(DAG.getTargetConstant(Intrinsic, TLI->getPointerTy()));
03857 
03858   // Add all operands of the call to the operand list.
03859   for (unsigned i = 0, e = I.getNumArgOperands(); i != e; ++i) {
03860     SDValue Op = getValue(I.getArgOperand(i));
03861     Ops.push_back(Op);
03862   }
03863 
03864   SmallVector<EVT, 4> ValueVTs;
03865   ComputeValueVTs(*TLI, I.getType(), ValueVTs);
03866 
03867   if (HasChain)
03868     ValueVTs.push_back(MVT::Other);
03869 
03870   SDVTList VTs = DAG.getVTList(ValueVTs);
03871 
03872   // Create the node.
03873   SDValue Result;
03874   if (IsTgtIntrinsic) {
03875     // This is target intrinsic that touches memory
03876     Result = DAG.getMemIntrinsicNode(Info.opc, getCurSDLoc(),
03877                                      VTs, Ops, Info.memVT,
03878                                    MachinePointerInfo(Info.ptrVal, Info.offset),
03879                                      Info.align, Info.vol,
03880                                      Info.readMem, Info.writeMem, Info.size);
03881   } else if (!HasChain) {
03882     Result = DAG.getNode(ISD::INTRINSIC_WO_CHAIN, getCurSDLoc(), VTs, Ops);
03883   } else if (!I.getType()->isVoidTy()) {
03884     Result = DAG.getNode(ISD::INTRINSIC_W_CHAIN, getCurSDLoc(), VTs, Ops);
03885   } else {
03886     Result = DAG.getNode(ISD::INTRINSIC_VOID, getCurSDLoc(), VTs, Ops);
03887   }
03888 
03889   if (HasChain) {
03890     SDValue Chain = Result.getValue(Result.getNode()->getNumValues()-1);
03891     if (OnlyLoad)
03892       PendingLoads.push_back(Chain);
03893     else
03894       DAG.setRoot(Chain);
03895   }
03896 
03897   if (!I.getType()->isVoidTy()) {
03898     if (VectorType *PTy = dyn_cast<VectorType>(I.getType())) {
03899       EVT VT = TLI->getValueType(PTy);
03900       Result = DAG.getNode(ISD::BITCAST, getCurSDLoc(), VT, Result);
03901     }
03902 
03903     setValue(&I, Result);
03904   }
03905 }
03906 
03907 /// GetSignificand - Get the significand and build it into a floating-point
03908 /// number with exponent of 1:
03909 ///
03910 ///   Op = (Op & 0x007fffff) | 0x3f800000;
03911 ///
03912 /// where Op is the hexadecimal representation of floating point value.
03913 static SDValue
03914 GetSignificand(SelectionDAG &DAG, SDValue Op, SDLoc dl) {
03915   SDValue t1 = DAG.getNode(ISD::AND, dl, MVT::i32, Op,
03916                            DAG.getConstant(0x007fffff, MVT::i32));
03917   SDValue t2 = DAG.getNode(ISD::OR, dl, MVT::i32, t1,
03918                            DAG.getConstant(0x3f800000, MVT::i32));
03919   return DAG.getNode(ISD::BITCAST, dl, MVT::f32, t2);
03920 }
03921 
03922 /// GetExponent - Get the exponent:
03923 ///
03924 ///   (float)(int)(((Op & 0x7f800000) >> 23) - 127);
03925 ///
03926 /// where Op is the hexadecimal representation of floating point value.
03927 static SDValue
03928 GetExponent(SelectionDAG &DAG, SDValue Op, const TargetLowering &TLI,
03929             SDLoc dl) {
03930   SDValue t0 = DAG.getNode(ISD::AND, dl, MVT::i32, Op,
03931                            DAG.getConstant(0x7f800000, MVT::i32));
03932   SDValue t1 = DAG.getNode(ISD::SRL, dl, MVT::i32, t0,
03933                            DAG.getConstant(23, TLI.getPointerTy()));
03934   SDValue t2 = DAG.getNode(ISD::SUB, dl, MVT::i32, t1,
03935                            DAG.getConstant(127, MVT::i32));
03936   return DAG.getNode(ISD::SINT_TO_FP, dl, MVT::f32, t2);
03937 }
03938 
03939 /// getF32Constant - Get 32-bit floating point constant.
03940 static SDValue
03941 getF32Constant(SelectionDAG &DAG, unsigned Flt) {
03942   return DAG.getConstantFP(APFloat(APFloat::IEEEsingle, APInt(32, Flt)),
03943                            MVT::f32);
03944 }
03945 
03946 /// expandExp - Lower an exp intrinsic. Handles the special sequences for
03947 /// limited-precision mode.
03948 static SDValue expandExp(SDLoc dl, SDValue Op, SelectionDAG &DAG,
03949                          const TargetLowering &TLI) {
03950   if (Op.getValueType() == MVT::f32 &&
03951       LimitFloatPrecision > 0 && LimitFloatPrecision <= 18) {
03952 
03953     // Put the exponent in the right bit position for later addition to the
03954     // final result:
03955     //
03956     //   #define LOG2OFe 1.4426950f
03957     //   IntegerPartOfX = ((int32_t)(X * LOG2OFe));
03958     SDValue t0 = DAG.getNode(ISD::FMUL, dl, MVT::f32, Op,
03959                              getF32Constant(DAG, 0x3fb8aa3b));
03960     SDValue IntegerPartOfX = DAG.getNode(ISD::FP_TO_SINT, dl, MVT::i32, t0);
03961 
03962     //   FractionalPartOfX = (X * LOG2OFe) - (float)IntegerPartOfX;
03963     SDValue t1 = DAG.getNode(ISD::SINT_TO_FP, dl, MVT::f32, IntegerPartOfX);
03964     SDValue X = DAG.getNode(ISD::FSUB, dl, MVT::f32, t0, t1);
03965 
03966     //   IntegerPartOfX <<= 23;
03967     IntegerPartOfX = DAG.getNode(ISD::SHL, dl, MVT::i32, IntegerPartOfX,
03968                                  DAG.getConstant(23, TLI.getPointerTy()));
03969 
03970     SDValue TwoToFracPartOfX;
03971     if (LimitFloatPrecision <= 6) {
03972       // For floating-point precision of 6:
03973       //
03974       //   TwoToFractionalPartOfX =
03975       //     0.997535578f +
03976       //       (0.735607626f + 0.252464424f * x) * x;
03977       //
03978       // error 0.0144103317, which is 6 bits
03979       SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X,
03980                                getF32Constant(DAG, 0x3e814304));
03981       SDValue t3 = DAG.getNode(ISD::FADD, dl, MVT::f32, t2,
03982                                getF32Constant(DAG, 0x3f3c50c8));
03983       SDValue t4 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t3, X);
03984       TwoToFracPartOfX = DAG.getNode(ISD::FADD, dl, MVT::f32, t4,
03985                                      getF32Constant(DAG, 0x3f7f5e7e));
03986     } else if (LimitFloatPrecision <= 12) {
03987       // For floating-point precision of 12:
03988       //
03989       //   TwoToFractionalPartOfX =
03990       //     0.999892986f +
03991       //       (0.696457318f +
03992       //         (0.224338339f + 0.792043434e-1f * x) * x) * x;
03993       //
03994       // 0.000107046256 error, which is 13 to 14 bits
03995       SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X,
03996                                getF32Constant(DAG, 0x3da235e3));
03997       SDValue t3 = DAG.getNode(ISD::FADD, dl, MVT::f32, t2,
03998                                getF32Constant(DAG, 0x3e65b8f3));
03999       SDValue t4 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t3, X);
04000       SDValue t5 = DAG.getNode(ISD::FADD, dl, MVT::f32, t4,
04001                                getF32Constant(DAG, 0x3f324b07));
04002       SDValue t6 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t5, X);
04003       TwoToFracPartOfX = DAG.getNode(ISD::FADD, dl, MVT::f32, t6,
04004                                      getF32Constant(DAG, 0x3f7ff8fd));
04005     } else { // LimitFloatPrecision <= 18
04006       // For floating-point precision of 18:
04007       //
04008       //   TwoToFractionalPartOfX =
04009       //     0.999999982f +
04010       //       (0.693148872f +
04011       //         (0.240227044f +
04012       //           (0.554906021e-1f +
04013       //             (0.961591928e-2f +
04014       //               (0.136028312e-2f + 0.157059148e-3f *x)*x)*x)*x)*x)*x;
04015       //
04016       // error 2.47208000*10^(-7), which is better than 18 bits
04017       SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X,
04018                                getF32Constant(DAG, 0x3924b03e));
04019       SDValue t3 = DAG.getNode(ISD::FADD, dl, MVT::f32, t2,
04020                                getF32Constant(DAG, 0x3ab24b87));
04021       SDValue t4 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t3, X);
04022       SDValue t5 = DAG.getNode(ISD::FADD, dl, MVT::f32, t4,
04023                                getF32Constant(DAG, 0x3c1d8c17));
04024       SDValue t6 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t5, X);
04025       SDValue t7 = DAG.getNode(ISD::FADD, dl, MVT::f32, t6,
04026                                getF32Constant(DAG, 0x3d634a1d));
04027       SDValue t8 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t7, X);
04028       SDValue t9 = DAG.getNode(ISD::FADD, dl, MVT::f32, t8,
04029                                getF32Constant(DAG, 0x3e75fe14));
04030       SDValue t10 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t9, X);
04031       SDValue t11 = DAG.getNode(ISD::FADD, dl, MVT::f32, t10,
04032                                 getF32Constant(DAG, 0x3f317234));
04033       SDValue t12 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t11, X);
04034       TwoToFracPartOfX = DAG.getNode(ISD::FADD, dl, MVT::f32, t12,
04035                                      getF32Constant(DAG, 0x3f800000));
04036     }
04037 
04038     // Add the exponent into the result in integer domain.
04039     SDValue t13 = DAG.getNode(ISD::BITCAST, dl, MVT::i32, TwoToFracPartOfX);
04040     return DAG.getNode(ISD::BITCAST, dl, MVT::f32,
04041                        DAG.getNode(ISD::ADD, dl, MVT::i32,
04042                                    t13, IntegerPartOfX));
04043   }
04044 
04045   // No special expansion.
04046   return DAG.getNode(ISD::FEXP, dl, Op.getValueType(), Op);
04047 }
04048 
04049 /// expandLog - Lower a log intrinsic. Handles the special sequences for
04050 /// limited-precision mode.
04051 static SDValue expandLog(SDLoc dl, SDValue Op, SelectionDAG &DAG,
04052                          const TargetLowering &TLI) {
04053   if (Op.getValueType() == MVT::f32 &&
04054       LimitFloatPrecision > 0 && LimitFloatPrecision <= 18) {
04055     SDValue Op1 = DAG.getNode(ISD::BITCAST, dl, MVT::i32, Op);
04056 
04057     // Scale the exponent by log(2) [0.69314718f].
04058     SDValue Exp = GetExponent(DAG, Op1, TLI, dl);
04059     SDValue LogOfExponent = DAG.getNode(ISD::FMUL, dl, MVT::f32, Exp,
04060                                         getF32Constant(DAG, 0x3f317218));
04061 
04062     // Get the significand and build it into a floating-point number with
04063     // exponent of 1.
04064     SDValue X = GetSignificand(DAG, Op1, dl);
04065 
04066     SDValue LogOfMantissa;
04067     if (LimitFloatPrecision <= 6) {
04068       // For floating-point precision of 6:
04069       //
04070       //   LogofMantissa =
04071       //     -1.1609546f +
04072       //       (1.4034025f - 0.23903021f * x) * x;
04073       //
04074       // error 0.0034276066, which is better than 8 bits
04075       SDValue t0 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X,
04076                                getF32Constant(DAG, 0xbe74c456));
04077       SDValue t1 = DAG.getNode(ISD::FADD, dl, MVT::f32, t0,
04078                                getF32Constant(DAG, 0x3fb3a2b1));
04079       SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t1, X);
04080       LogOfMantissa = DAG.getNode(ISD::FSUB, dl, MVT::f32, t2,
04081                                   getF32Constant(DAG, 0x3f949a29));
04082     } else if (LimitFloatPrecision <= 12) {
04083       // For floating-point precision of 12:
04084       //
04085       //   LogOfMantissa =
04086       //     -1.7417939f +
04087       //       (2.8212026f +
04088       //         (-1.4699568f +
04089       //           (0.44717955f - 0.56570851e-1f * x) * x) * x) * x;
04090       //
04091       // error 0.000061011436, which is 14 bits
04092       SDValue t0 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X,
04093                                getF32Constant(DAG, 0xbd67b6d6));
04094       SDValue t1 = DAG.getNode(ISD::FADD, dl, MVT::f32, t0,
04095                                getF32Constant(DAG, 0x3ee4f4b8));
04096       SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t1, X);
04097       SDValue t3 = DAG.getNode(ISD::FSUB, dl, MVT::f32, t2,
04098                                getF32Constant(DAG, 0x3fbc278b));
04099       SDValue t4 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t3, X);
04100       SDValue t5 = DAG.getNode(ISD::FADD, dl, MVT::f32, t4,
04101                                getF32Constant(DAG, 0x40348e95));
04102       SDValue t6 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t5, X);
04103       LogOfMantissa = DAG.getNode(ISD::FSUB, dl, MVT::f32, t6,
04104                                   getF32Constant(DAG, 0x3fdef31a));
04105     } else { // LimitFloatPrecision <= 18
04106       // For floating-point precision of 18:
04107       //
04108       //   LogOfMantissa =
04109       //     -2.1072184f +
04110       //       (4.2372794f +
04111       //         (-3.7029485f +
04112       //           (2.2781945f +
04113       //             (-0.87823314f +
04114       //               (0.19073739f - 0.17809712e-1f * x) * x) * x) * x) * x)*x;
04115       //
04116       // error 0.0000023660568, which is better than 18 bits
04117       SDValue t0 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X,
04118                                getF32Constant(DAG, 0xbc91e5ac));
04119       SDValue t1 = DAG.getNode(ISD::FADD, dl, MVT::f32, t0,
04120                                getF32Constant(DAG, 0x3e4350aa));
04121       SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t1, X);
04122       SDValue t3 = DAG.getNode(ISD::FSUB, dl, MVT::f32, t2,
04123                                getF32Constant(DAG, 0x3f60d3e3));
04124       SDValue t4 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t3, X);
04125       SDValue t5 = DAG.getNode(ISD::FADD, dl, MVT::f32, t4,
04126                                getF32Constant(DAG, 0x4011cdf0));
04127       SDValue t6 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t5, X);
04128       SDValue t7 = DAG.getNode(ISD::FSUB, dl, MVT::f32, t6,
04129                                getF32Constant(DAG, 0x406cfd1c));
04130       SDValue t8 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t7, X);
04131       SDValue t9 = DAG.getNode(ISD::FADD, dl, MVT::f32, t8,
04132                                getF32Constant(DAG, 0x408797cb));
04133       SDValue t10 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t9, X);
04134       LogOfMantissa = DAG.getNode(ISD::FSUB, dl, MVT::f32, t10,
04135                                   getF32Constant(DAG, 0x4006dcab));
04136     }
04137 
04138     return DAG.getNode(ISD::FADD, dl, MVT::f32, LogOfExponent, LogOfMantissa);
04139   }
04140 
04141   // No special expansion.
04142   return DAG.getNode(ISD::FLOG, dl, Op.getValueType(), Op);
04143 }
04144 
04145 /// expandLog2 - Lower a log2 intrinsic. Handles the special sequences for
04146 /// limited-precision mode.
04147 static SDValue expandLog2(SDLoc dl, SDValue Op, SelectionDAG &DAG,
04148                           const TargetLowering &TLI) {
04149   if (Op.getValueType() == MVT::f32 &&
04150       LimitFloatPrecision > 0 && LimitFloatPrecision <= 18) {
04151     SDValue Op1 = DAG.getNode(ISD::BITCAST, dl, MVT::i32, Op);
04152 
04153     // Get the exponent.
04154     SDValue LogOfExponent = GetExponent(DAG, Op1, TLI, dl);
04155 
04156     // Get the significand and build it into a floating-point number with
04157     // exponent of 1.
04158     SDValue X = GetSignificand(DAG, Op1, dl);
04159 
04160     // Different possible minimax approximations of significand in
04161     // floating-point for various degrees of accuracy over [1,2].
04162     SDValue Log2ofMantissa;
04163     if (LimitFloatPrecision <= 6) {
04164       // For floating-point precision of 6:
04165       //
04166       //   Log2ofMantissa = -1.6749035f + (2.0246817f - .34484768f * x) * x;
04167       //
04168       // error 0.0049451742, which is more than 7 bits
04169       SDValue t0 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X,
04170                                getF32Constant(DAG, 0xbeb08fe0));
04171       SDValue t1 = DAG.getNode(ISD::FADD, dl, MVT::f32, t0,
04172                                getF32Constant(DAG, 0x40019463));
04173       SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t1, X);
04174       Log2ofMantissa = DAG.getNode(ISD::FSUB, dl, MVT::f32, t2,
04175                                    getF32Constant(DAG, 0x3fd6633d));
04176     } else if (LimitFloatPrecision <= 12) {
04177       // For floating-point precision of 12:
04178       //
04179       //   Log2ofMantissa =
04180       //     -2.51285454f +
04181       //       (4.07009056f +
04182       //         (-2.12067489f +
04183       //           (.645142248f - 0.816157886e-1f * x) * x) * x) * x;
04184       //
04185       // error 0.0000876136000, which is better than 13 bits
04186       SDValue t0 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X,
04187                                getF32Constant(DAG, 0xbda7262e));
04188       SDValue t1 = DAG.getNode(ISD::FADD, dl, MVT::f32, t0,
04189                                getF32Constant(DAG, 0x3f25280b));
04190       SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t1, X);
04191       SDValue t3 = DAG.getNode(ISD::FSUB, dl, MVT::f32, t2,
04192                                getF32Constant(DAG, 0x4007b923));
04193       SDValue t4 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t3, X);
04194       SDValue t5 = DAG.getNode(ISD::FADD, dl, MVT::f32, t4,
04195                                getF32Constant(DAG, 0x40823e2f));
04196       SDValue t6 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t5, X);
04197       Log2ofMantissa = DAG.getNode(ISD::FSUB, dl, MVT::f32, t6,
04198                                    getF32Constant(DAG, 0x4020d29c));
04199     } else { // LimitFloatPrecision <= 18
04200       // For floating-point precision of 18:
04201       //
04202       //   Log2ofMantissa =
04203       //     -3.0400495f +
04204       //       (6.1129976f +
04205       //         (-5.3420409f +
04206       //           (3.2865683f +
04207       //             (-1.2669343f +
04208       //               (0.27515199f -
04209       //                 0.25691327e-1f * x) * x) * x) * x) * x) * x;
04210       //
04211       // error 0.0000018516, which is better than 18 bits
04212       SDValue t0 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X,
04213                                getF32Constant(DAG, 0xbcd2769e));
04214       SDValue t1 = DAG.getNode(ISD::FADD, dl, MVT::f32, t0,
04215                                getF32Constant(DAG, 0x3e8ce0b9));
04216       SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t1, X);
04217       SDValue t3 = DAG.getNode(ISD::FSUB, dl, MVT::f32, t2,
04218                                getF32Constant(DAG, 0x3fa22ae7));
04219       SDValue t4 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t3, X);
04220       SDValue t5 = DAG.getNode(ISD::FADD, dl, MVT::f32, t4,
04221                                getF32Constant(DAG, 0x40525723));
04222       SDValue t6 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t5, X);
04223       SDValue t7 = DAG.getNode(ISD::FSUB, dl, MVT::f32, t6,
04224                                getF32Constant(DAG, 0x40aaf200));
04225       SDValue t8 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t7, X);
04226       SDValue t9 = DAG.getNode(ISD::FADD, dl, MVT::f32, t8,
04227                                getF32Constant(DAG, 0x40c39dad));
04228       SDValue t10 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t9, X);
04229       Log2ofMantissa = DAG.getNode(ISD::FSUB, dl, MVT::f32, t10,
04230                                    getF32Constant(DAG, 0x4042902c));
04231     }
04232 
04233     return DAG.getNode(ISD::FADD, dl, MVT::f32, LogOfExponent, Log2ofMantissa);
04234   }
04235 
04236   // No special expansion.
04237   return DAG.getNode(ISD::FLOG2, dl, Op.getValueType(), Op);
04238 }
04239 
04240 /// expandLog10 - Lower a log10 intrinsic. Handles the special sequences for
04241 /// limited-precision mode.
04242 static SDValue expandLog10(SDLoc dl, SDValue Op, SelectionDAG &DAG,
04243                            const TargetLowering &TLI) {
04244   if (Op.getValueType() == MVT::f32 &&
04245       LimitFloatPrecision > 0 && LimitFloatPrecision <= 18) {
04246     SDValue Op1 = DAG.getNode(ISD::BITCAST, dl, MVT::i32, Op);
04247 
04248     // Scale the exponent by log10(2) [0.30102999f].
04249     SDValue Exp = GetExponent(DAG, Op1, TLI, dl);
04250     SDValue LogOfExponent = DAG.getNode(ISD::FMUL, dl, MVT::f32, Exp,
04251                                         getF32Constant(DAG, 0x3e9a209a));
04252 
04253     // Get the significand and build it into a floating-point number with
04254     // exponent of 1.
04255     SDValue X = GetSignificand(DAG, Op1, dl);
04256 
04257     SDValue Log10ofMantissa;
04258     if (LimitFloatPrecision <= 6) {
04259       // For floating-point precision of 6:
04260       //
04261       //   Log10ofMantissa =
04262       //     -0.50419619f +
04263       //       (0.60948995f - 0.10380950f * x) * x;
04264       //
04265       // error 0.0014886165, which is 6 bits
04266       SDValue t0 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X,
04267                                getF32Constant(DAG, 0xbdd49a13));
04268       SDValue t1 = DAG.getNode(ISD::FADD, dl, MVT::f32, t0,
04269                                getF32Constant(DAG, 0x3f1c0789));
04270       SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t1, X);
04271       Log10ofMantissa = DAG.getNode(ISD::FSUB, dl, MVT::f32, t2,
04272                                     getF32Constant(DAG, 0x3f011300));
04273     } else if (LimitFloatPrecision <= 12) {
04274       // For floating-point precision of 12:
04275       //
04276       //   Log10ofMantissa =
04277       //     -0.64831180f +
04278       //       (0.91751397f +
04279       //         (-0.31664806f + 0.47637168e-1f * x) * x) * x;
04280       //
04281       // error 0.00019228036, which is better than 12 bits
04282       SDValue t0 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X,
04283                                getF32Constant(DAG, 0x3d431f31));
04284       SDValue t1 = DAG.getNode(ISD::FSUB, dl, MVT::f32, t0,
04285                                getF32Constant(DAG, 0x3ea21fb2));
04286       SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t1, X);
04287       SDValue t3 = DAG.getNode(ISD::FADD, dl, MVT::f32, t2,
04288                                getF32Constant(DAG, 0x3f6ae232));
04289       SDValue t4 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t3, X);
04290       Log10ofMantissa = DAG.getNode(ISD::FSUB, dl, MVT::f32, t4,
04291                                     getF32Constant(DAG, 0x3f25f7c3));
04292     } else { // LimitFloatPrecision <= 18
04293       // For floating-point precision of 18:
04294       //
04295       //   Log10ofMantissa =
04296       //     -0.84299375f +
04297       //       (1.5327582f +
04298       //         (-1.0688956f +
04299       //           (0.49102474f +
04300       //             (-0.12539807f + 0.13508273e-1f * x) * x) * x) * x) * x;
04301       //
04302       // error 0.0000037995730, which is better than 18 bits
04303       SDValue t0 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X,
04304                                getF32Constant(DAG, 0x3c5d51ce));
04305       SDValue t1 = DAG.getNode(ISD::FSUB, dl, MVT::f32, t0,
04306                                getF32Constant(DAG, 0x3e00685a));
04307       SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t1, X);
04308       SDValue t3 = DAG.getNode(ISD::FADD, dl, MVT::f32, t2,
04309                                getF32Constant(DAG, 0x3efb6798));
04310       SDValue t4 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t3, X);
04311       SDValue t5 = DAG.getNode(ISD::FSUB, dl, MVT::f32, t4,
04312                                getF32Constant(DAG, 0x3f88d192));
04313       SDValue t6 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t5, X);
04314       SDValue t7 = DAG.getNode(ISD::FADD, dl, MVT::f32, t6,
04315                                getF32Constant(DAG, 0x3fc4316c));
04316       SDValue t8 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t7, X);
04317       Log10ofMantissa = DAG.getNode(ISD::FSUB, dl, MVT::f32, t8,
04318                                     getF32Constant(DAG, 0x3f57ce70));
04319     }
04320 
04321     return DAG.getNode(ISD::FADD, dl, MVT::f32, LogOfExponent, Log10ofMantissa);
04322   }
04323 
04324   // No special expansion.
04325   return DAG.getNode(ISD::FLOG10, dl, Op.getValueType(), Op);
04326 }
04327 
04328 /// expandExp2 - Lower an exp2 intrinsic. Handles the special sequences for
04329 /// limited-precision mode.
04330 static SDValue expandExp2(SDLoc dl, SDValue Op, SelectionDAG &DAG,
04331                           const TargetLowering &TLI) {
04332   if (Op.getValueType() == MVT::f32 &&
04333       LimitFloatPrecision > 0 && LimitFloatPrecision <= 18) {
04334     SDValue IntegerPartOfX = DAG.getNode(ISD::FP_TO_SINT, dl, MVT::i32, Op);
04335 
04336     //   FractionalPartOfX = x - (float)IntegerPartOfX;
04337     SDValue t1 = DAG.getNode(ISD::SINT_TO_FP, dl, MVT::f32, IntegerPartOfX);
04338     SDValue X = DAG.getNode(ISD::FSUB, dl, MVT::f32, Op, t1);
04339 
04340     //   IntegerPartOfX <<= 23;
04341     IntegerPartOfX = DAG.getNode(ISD::SHL, dl, MVT::i32, IntegerPartOfX,
04342                                  DAG.getConstant(23, TLI.getPointerTy()));
04343 
04344     SDValue TwoToFractionalPartOfX;
04345     if (LimitFloatPrecision <= 6) {
04346       // For floating-point precision of 6:
04347       //
04348       //   TwoToFractionalPartOfX =
04349       //     0.997535578f +
04350       //       (0.735607626f + 0.252464424f * x) * x;
04351       //
04352       // error 0.0144103317, which is 6 bits
04353       SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X,
04354                                getF32Constant(DAG, 0x3e814304));
04355       SDValue t3 = DAG.getNode(ISD::FADD, dl, MVT::f32, t2,
04356                                getF32Constant(DAG, 0x3f3c50c8));
04357       SDValue t4 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t3, X);
04358       TwoToFractionalPartOfX = DAG.getNode(ISD::FADD, dl, MVT::f32, t4,
04359                                            getF32Constant(DAG, 0x3f7f5e7e));
04360     } else if (LimitFloatPrecision <= 12) {
04361       // For floating-point precision of 12:
04362       //
04363       //   TwoToFractionalPartOfX =
04364       //     0.999892986f +
04365       //       (0.696457318f +
04366       //         (0.224338339f + 0.792043434e-1f * x) * x) * x;
04367       //
04368       // error 0.000107046256, which is 13 to 14 bits
04369       SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X,
04370                                getF32Constant(DAG, 0x3da235e3));
04371       SDValue t3 = DAG.getNode(ISD::FADD, dl, MVT::f32, t2,
04372                                getF32Constant(DAG, 0x3e65b8f3));
04373       SDValue t4 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t3, X);
04374       SDValue t5 = DAG.getNode(ISD::FADD, dl, MVT::f32, t4,
04375                                getF32Constant(DAG, 0x3f324b07));
04376       SDValue t6 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t5, X);
04377       TwoToFractionalPartOfX = DAG.getNode(ISD::FADD, dl, MVT::f32, t6,
04378                                            getF32Constant(DAG, 0x3f7ff8fd));
04379     } else { // LimitFloatPrecision <= 18
04380       // For floating-point precision of 18:
04381       //
04382       //   TwoToFractionalPartOfX =
04383       //     0.999999982f +
04384       //       (0.693148872f +
04385       //         (0.240227044f +
04386       //           (0.554906021e-1f +
04387       //             (0.961591928e-2f +
04388       //               (0.136028312e-2f + 0.157059148e-3f *x)*x)*x)*x)*x)*x;
04389       // error 2.47208000*10^(-7), which is better than 18 bits
04390       SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X,
04391                                getF32Constant(DAG, 0x3924b03e));
04392       SDValue t3 = DAG.getNode(ISD::FADD, dl, MVT::f32, t2,
04393                                getF32Constant(DAG, 0x3ab24b87));
04394       SDValue t4 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t3, X);
04395       SDValue t5 = DAG.getNode(ISD::FADD, dl, MVT::f32, t4,
04396                                getF32Constant(DAG, 0x3c1d8c17));
04397       SDValue t6 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t5, X);
04398       SDValue t7 = DAG.getNode(ISD::FADD, dl, MVT::f32, t6,
04399                                getF32Constant(DAG, 0x3d634a1d));
04400       SDValue t8 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t7, X);
04401       SDValue t9 = DAG.getNode(ISD::FADD, dl, MVT::f32, t8,
04402                                getF32Constant(DAG, 0x3e75fe14));
04403       SDValue t10 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t9, X);
04404       SDValue t11 = DAG.getNode(ISD::FADD, dl, MVT::f32, t10,
04405                                 getF32Constant(DAG, 0x3f317234));
04406       SDValue t12 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t11, X);
04407       TwoToFractionalPartOfX = DAG.getNode(ISD::FADD, dl, MVT::f32, t12,
04408                                            getF32Constant(DAG, 0x3f800000));
04409     }
04410 
04411     // Add the exponent into the result in integer domain.
04412     SDValue t13 = DAG.getNode(ISD::BITCAST, dl, MVT::i32,
04413                               TwoToFractionalPartOfX);
04414     return DAG.getNode(ISD::BITCAST, dl, MVT::f32,
04415                        DAG.getNode(ISD::ADD, dl, MVT::i32,
04416                                    t13, IntegerPartOfX));
04417   }
04418 
04419   // No special expansion.
04420   return DAG.getNode(ISD::FEXP2, dl, Op.getValueType(), Op);
04421 }
04422 
04423 /// visitPow - Lower a pow intrinsic. Handles the special sequences for
04424 /// limited-precision mode with x == 10.0f.
04425 static SDValue expandPow(SDLoc dl, SDValue LHS, SDValue RHS,
04426                          SelectionDAG &DAG, const TargetLowering &TLI) {
04427   bool IsExp10 = false;
04428   if (LHS.getValueType() == MVT::f32 && RHS.getValueType() == MVT::f32 &&
04429       LimitFloatPrecision > 0 && LimitFloatPrecision <= 18) {
04430     if (ConstantFPSDNode *LHSC = dyn_cast<ConstantFPSDNode>(LHS)) {
04431       APFloat Ten(10.0f);
04432       IsExp10 = LHSC->isExactlyValue(Ten);
04433     }
04434   }
04435 
04436   if (IsExp10) {
04437     // Put the exponent in the right bit position for later addition to the
04438     // final result:
04439     //
04440     //   #define LOG2OF10 3.3219281f
04441     //   IntegerPartOfX = (int32_t)(x * LOG2OF10);
04442     SDValue t0 = DAG.getNode(ISD::FMUL, dl, MVT::f32, RHS,
04443                              getF32Constant(DAG, 0x40549a78));
04444     SDValue IntegerPartOfX = DAG.getNode(ISD::FP_TO_SINT, dl, MVT::i32, t0);
04445 
04446     //   FractionalPartOfX = x - (float)IntegerPartOfX;
04447     SDValue t1 = DAG.getNode(ISD::SINT_TO_FP, dl, MVT::f32, IntegerPartOfX);
04448     SDValue X = DAG.getNode(ISD::FSUB, dl, MVT::f32, t0, t1);
04449 
04450     //   IntegerPartOfX <<= 23;
04451     IntegerPartOfX = DAG.getNode(ISD::SHL, dl, MVT::i32, IntegerPartOfX,
04452                                  DAG.getConstant(23, TLI.getPointerTy()));
04453 
04454     SDValue TwoToFractionalPartOfX;
04455     if (LimitFloatPrecision <= 6) {
04456       // For floating-point precision of 6:
04457       //
04458       //   twoToFractionalPartOfX =
04459       //     0.997535578f +
04460       //       (0.735607626f + 0.252464424f * x) * x;
04461       //
04462       // error 0.0144103317, which is 6 bits
04463       SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X,
04464                                getF32Constant(DAG, 0x3e814304));
04465       SDValue t3 = DAG.getNode(ISD::FADD, dl, MVT::f32, t2,
04466                                getF32Constant(DAG, 0x3f3c50c8));
04467       SDValue t4 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t3, X);
04468       TwoToFractionalPartOfX = DAG.getNode(ISD::FADD, dl, MVT::f32, t4,
04469                                            getF32Constant(DAG, 0x3f7f5e7e));
04470     } else if (LimitFloatPrecision <= 12) {
04471       // For floating-point precision of 12:
04472       //
04473       //   TwoToFractionalPartOfX =
04474       //     0.999892986f +
04475       //       (0.696457318f +
04476       //         (0.224338339f + 0.792043434e-1f * x) * x) * x;
04477       //
04478       // error 0.000107046256, which is 13 to 14 bits
04479       SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X,
04480                                getF32Constant(DAG, 0x3da235e3));
04481       SDValue t3 = DAG.getNode(ISD::FADD, dl, MVT::f32, t2,
04482                                getF32Constant(DAG, 0x3e65b8f3));
04483       SDValue t4 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t3, X);
04484       SDValue t5 = DAG.getNode(ISD::FADD, dl, MVT::f32, t4,
04485                                getF32Constant(DAG, 0x3f324b07));
04486       SDValue t6 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t5, X);
04487       TwoToFractionalPartOfX = DAG.getNode(ISD::FADD, dl, MVT::f32, t6,
04488                                            getF32Constant(DAG, 0x3f7ff8fd));
04489     } else { // LimitFloatPrecision <= 18
04490       // For floating-point precision of 18:
04491       //
04492       //   TwoToFractionalPartOfX =
04493       //     0.999999982f +
04494       //       (0.693148872f +
04495       //         (0.240227044f +
04496       //           (0.554906021e-1f +
04497       //             (0.961591928e-2f +
04498       //               (0.136028312e-2f + 0.157059148e-3f *x)*x)*x)*x)*x)*x;
04499       // error 2.47208000*10^(-7), which is better than 18 bits
04500       SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X,
04501                                getF32Constant(DAG, 0x3924b03e));
04502       SDValue t3 = DAG.getNode(ISD::FADD, dl, MVT::f32, t2,
04503                                getF32Constant(DAG, 0x3ab24b87));
04504       SDValue t4 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t3, X);
04505       SDValue t5 = DAG.getNode(ISD::FADD, dl, MVT::f32, t4,
04506                                getF32Constant(DAG, 0x3c1d8c17));
04507       SDValue t6 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t5, X);
04508       SDValue t7 = DAG.getNode(ISD::FADD, dl, MVT::f32, t6,
04509                                getF32Constant(DAG, 0x3d634a1d));
04510       SDValue t8 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t7, X);
04511       SDValue t9 = DAG.getNode(ISD::FADD, dl, MVT::f32, t8,
04512                                getF32Constant(DAG, 0x3e75fe14));
04513       SDValue t10 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t9, X);
04514       SDValue t11 = DAG.getNode(ISD::FADD, dl, MVT::f32, t10,
04515                                 getF32Constant(DAG, 0x3f317234));
04516       SDValue t12 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t11, X);
04517       TwoToFractionalPartOfX = DAG.getNode(ISD::FADD, dl, MVT::f32, t12,
04518                                            getF32Constant(DAG, 0x3f800000));
04519     }
04520 
04521     SDValue t13 = DAG.getNode(ISD::BITCAST, dl,MVT::i32,TwoToFractionalPartOfX);
04522     return DAG.getNode(ISD::BITCAST, dl, MVT::f32,
04523                        DAG.getNode(ISD::ADD, dl, MVT::i32,
04524                                    t13, IntegerPartOfX));
04525   }
04526 
04527   // No special expansion.
04528   return DAG.getNode(ISD::FPOW, dl, LHS.getValueType(), LHS, RHS);
04529 }
04530 
04531 
04532 /// ExpandPowI - Expand a llvm.powi intrinsic.
04533 static SDValue ExpandPowI(SDLoc DL, SDValue LHS, SDValue RHS,
04534                           SelectionDAG &DAG) {
04535   // If RHS is a constant, we can expand this out to a multiplication tree,
04536   // otherwise we end up lowering to a call to __powidf2 (for example).  When
04537   // optimizing for size, we only want to do this if the expansion would produce
04538   // a small number of multiplies, otherwise we do the full expansion.
04539   if (ConstantSDNode *RHSC = dyn_cast<ConstantSDNode>(RHS)) {
04540     // Get the exponent as a positive value.
04541     unsigned Val = RHSC->getSExtValue();
04542     if ((int)Val < 0) Val = -Val;
04543 
04544     // powi(x, 0) -> 1.0
04545     if (Val == 0)
04546       return DAG.getConstantFP(1.0, LHS.getValueType());
04547 
04548     const Function *F = DAG.getMachineFunction().getFunction();
04549     if (!F->getAttributes().hasAttribute(AttributeSet::FunctionIndex,
04550                                          Attribute::OptimizeForSize) ||
04551         // If optimizing for size, don't insert too many multiplies.  This
04552         // inserts up to 5 multiplies.
04553         CountPopulation_32(Val)+Log2_32(Val) < 7) {
04554       // We use the simple binary decomposition method to generate the multiply
04555       // sequence.  There are more optimal ways to do this (for example,
04556       // powi(x,15) generates one more multiply than it should), but this has
04557       // the benefit of being both really simple and much better than a libcall.
04558       SDValue Res;  // Logically starts equal to 1.0
04559       SDValue CurSquare = LHS;
04560       while (Val) {
04561         if (Val & 1) {
04562           if (Res.getNode())
04563             Res = DAG.getNode(ISD::FMUL, DL,Res.getValueType(), Res, CurSquare);
04564           else
04565             Res = CurSquare;  // 1.0*CurSquare.
04566         }
04567 
04568         CurSquare = DAG.getNode(ISD::FMUL, DL, CurSquare.getValueType(),
04569                                 CurSquare, CurSquare);
04570         Val >>= 1;
04571       }
04572 
04573       // If the original was negative, invert the result, producing 1/(x*x*x).
04574       if (RHSC->getSExtValue() < 0)
04575         Res = DAG.getNode(ISD::FDIV, DL, LHS.getValueType(),
04576                           DAG.getConstantFP(1.0, LHS.getValueType()), Res);
04577       return Res;
04578     }
04579   }
04580 
04581   // Otherwise, expand to a libcall.
04582   return DAG.getNode(ISD::FPOWI, DL, LHS.getValueType(), LHS, RHS);
04583 }
04584 
04585 // getTruncatedArgReg - Find underlying register used for an truncated
04586 // argument.
04587 static unsigned getTruncatedArgReg(const SDValue &N) {
04588   if (N.getOpcode() != ISD::TRUNCATE)
04589     return 0;
04590 
04591   const SDValue &Ext = N.getOperand(0);
04592   if (Ext.getOpcode() == ISD::AssertZext ||
04593       Ext.getOpcode() == ISD::AssertSext) {
04594     const SDValue &CFR = Ext.getOperand(0);
04595     if (CFR.getOpcode() == ISD::CopyFromReg)
04596       return cast<RegisterSDNode>(CFR.getOperand(1))->getReg();
04597     if (CFR.getOpcode() == ISD::TRUNCATE)
04598       return getTruncatedArgReg(CFR);
04599   }
04600   return 0;
04601 }
04602 
04603 /// EmitFuncArgumentDbgValue - If the DbgValueInst is a dbg_value of a function
04604 /// argument, create the corresponding DBG_VALUE machine instruction for it now.
04605 /// At the end of instruction selection, they will be inserted to the entry BB.
04606 bool
04607 SelectionDAGBuilder::EmitFuncArgumentDbgValue(const Value *V, MDNode *Variable,
04608                                               int64_t Offset, bool IsIndirect,
04609                                               const SDValue &N) {
04610   const Argument *Arg = dyn_cast<Argument>(V);
04611   if (!Arg)
04612     return false;
04613 
04614   MachineFunction &MF = DAG.getMachineFunction();
04615   const TargetInstrInfo *TII = DAG.getSubtarget().getInstrInfo();
04616 
04617   // Ignore inlined function arguments here.
04618   DIVariable DV(Variable);
04619   if (DV.isInlinedFnArgument(MF.getFunction()))
04620     return false;
04621 
04622   Optional<MachineOperand> Op;
04623   // Some arguments' frame index is recorded during argument lowering.
04624   if (int FI = FuncInfo.getArgumentFrameIndex(Arg))
04625     Op = MachineOperand::CreateFI(FI);
04626 
04627   if (!Op && N.getNode()) {
04628     unsigned Reg;
04629     if (N.getOpcode() == ISD::CopyFromReg)
04630       Reg = cast<RegisterSDNode>(N.getOperand(1))->getReg();
04631     else
04632       Reg = getTruncatedArgReg(N);
04633     if (Reg && TargetRegisterInfo::isVirtualRegister(Reg)) {
04634       MachineRegisterInfo &RegInfo = MF.getRegInfo();
04635       unsigned PR = RegInfo.getLiveInPhysReg(Reg);
04636       if (PR)
04637         Reg = PR;
04638     }
04639     if (Reg)
04640       Op = MachineOperand::CreateReg(Reg, false);
04641   }
04642 
04643   if (!Op) {
04644     // Check if ValueMap has reg number.
04645     DenseMap<const Value *, unsigned>::iterator VMI = FuncInfo.ValueMap.find(V);
04646     if (VMI != FuncInfo.ValueMap.end())
04647       Op = MachineOperand::CreateReg(VMI->second, false);
04648   }
04649 
04650   if (!Op && N.getNode())
04651     // Check if frame index is available.
04652     if (LoadSDNode *LNode = dyn_cast<LoadSDNode>(N.getNode()))
04653       if (FrameIndexSDNode *FINode =
04654           dyn_cast<FrameIndexSDNode>(LNode->getBasePtr().getNode()))
04655         Op = MachineOperand::CreateFI(FINode->getIndex());
04656 
04657   if (!Op)
04658     return false;
04659 
04660   if (Op->isReg())
04661     FuncInfo.ArgDbgValues.push_back(BuildMI(MF, getCurDebugLoc(),
04662                                             TII->get(TargetOpcode::DBG_VALUE),
04663                                             IsIndirect,
04664                                             Op->getReg(), Offset, Variable));
04665   else
04666     FuncInfo.ArgDbgValues.push_back(
04667       BuildMI(MF, getCurDebugLoc(), TII->get(TargetOpcode::DBG_VALUE))
04668           .addOperand(*Op).addImm(Offset).addMetadata(Variable));
04669 
04670   return true;
04671 }
04672 
04673 // VisualStudio defines setjmp as _setjmp
04674 #if defined(_MSC_VER) && defined(setjmp) && \
04675                          !defined(setjmp_undefined_for_msvc)
04676 #  pragma push_macro("setjmp")
04677 #  undef setjmp
04678 #  define setjmp_undefined_for_msvc
04679 #endif
04680 
04681 /// visitIntrinsicCall - Lower the call to the specified intrinsic function.  If
04682 /// we want to emit this as a call to a named external function, return the name
04683 /// otherwise lower it and return null.
04684 const char *
04685 SelectionDAGBuilder::visitIntrinsicCall(const CallInst &I, unsigned Intrinsic) {
04686   const TargetLowering *TLI = TM.getSubtargetImpl()->getTargetLowering();
04687   SDLoc sdl = getCurSDLoc();
04688   DebugLoc dl = getCurDebugLoc();
04689   SDValue Res;
04690 
04691   switch (Intrinsic) {
04692   default:
04693     // By default, turn this into a target intrinsic node.
04694     visitTargetIntrinsic(I, Intrinsic);
04695     return nullptr;
04696   case Intrinsic::vastart:  visitVAStart(I); return nullptr;
04697   case Intrinsic::vaend:    visitVAEnd(I); return nullptr;
04698   case Intrinsic::vacopy:   visitVACopy(I); return nullptr;
04699   case Intrinsic::returnaddress:
04700     setValue(&I, DAG.getNode(ISD::RETURNADDR, sdl, TLI->getPointerTy(),
04701                              getValue(I.getArgOperand(0))));
04702     return nullptr;
04703   case Intrinsic::frameaddress:
04704     setValue(&I, DAG.getNode(ISD::FRAMEADDR, sdl, TLI->getPointerTy(),
04705                              getValue(I.getArgOperand(0))));
04706     return nullptr;
04707   case Intrinsic::read_register: {
04708     Value *Reg = I.getArgOperand(0);
04709     SDValue RegName = DAG.getMDNode(cast<MDNode>(Reg));
04710     EVT VT =
04711         TM.getSubtargetImpl()->getTargetLowering()->getValueType(I.getType());
04712     setValue(&I, DAG.getNode(ISD::READ_REGISTER, sdl, VT, RegName));
04713     return nullptr;
04714   }
04715   case Intrinsic::write_register: {
04716     Value *Reg = I.getArgOperand(0);
04717     Value *RegValue = I.getArgOperand(1);
04718     SDValue Chain = getValue(RegValue).getOperand(0);
04719     SDValue RegName = DAG.getMDNode(cast<MDNode>(Reg));
04720     DAG.setRoot(DAG.getNode(ISD::WRITE_REGISTER, sdl, MVT::Other, Chain,
04721                             RegName, getValue(RegValue)));
04722     return nullptr;
04723   }
04724   case Intrinsic::setjmp:
04725     return &"_setjmp"[!TLI->usesUnderscoreSetJmp()];
04726   case Intrinsic::longjmp:
04727     return &"_longjmp"[!TLI->usesUnderscoreLongJmp()];
04728   case Intrinsic::memcpy: {
04729     // Assert for address < 256 since we support only user defined address
04730     // spaces.
04731     assert(cast<PointerType>(I.getArgOperand(0)->getType())->getAddressSpace()
04732            < 256 &&
04733            cast<PointerType>(I.getArgOperand(1)->getType())->getAddressSpace()
04734            < 256 &&
04735            "Unknown address space");
04736     SDValue Op1 = getValue(I.getArgOperand(0));
04737     SDValue Op2 = getValue(I.getArgOperand(1));
04738     SDValue Op3 = getValue(I.getArgOperand(2));
04739     unsigned Align = cast<ConstantInt>(I.getArgOperand(3))->getZExtValue();
04740     if (!Align)
04741       Align = 1; // @llvm.memcpy defines 0 and 1 to both mean no alignment.
04742     bool isVol = cast<ConstantInt>(I.getArgOperand(4))->getZExtValue();
04743     DAG.setRoot(DAG.getMemcpy(getRoot(), sdl, Op1, Op2, Op3, Align, isVol, false,
04744                               MachinePointerInfo(I.getArgOperand(0)),
04745                               MachinePointerInfo(I.getArgOperand(1))));
04746     return nullptr;
04747   }
04748   case Intrinsic::memset: {
04749     // Assert for address < 256 since we support only user defined address
04750     // spaces.
04751     assert(cast<PointerType>(I.getArgOperand(0)->getType())->getAddressSpace()
04752            < 256 &&
04753            "Unknown address space");
04754     SDValue Op1 = getValue(I.getArgOperand(0));
04755     SDValue Op2 = getValue(I.getArgOperand(1));
04756     SDValue Op3 = getValue(I.getArgOperand(2));
04757     unsigned Align = cast<ConstantInt>(I.getArgOperand(3))->getZExtValue();
04758     if (!Align)
04759       Align = 1; // @llvm.memset defines 0 and 1 to both mean no alignment.
04760     bool isVol = cast<ConstantInt>(I.getArgOperand(4))->getZExtValue();
04761     DAG.setRoot(DAG.getMemset(getRoot(), sdl, Op1, Op2, Op3, Align, isVol,
04762                               MachinePointerInfo(I.getArgOperand(0))));
04763     return nullptr;
04764   }
04765   case Intrinsic::memmove: {
04766     // Assert for address < 256 since we support only user defined address
04767     // spaces.
04768     assert(cast<PointerType>(I.getArgOperand(0)->getType())->getAddressSpace()
04769            < 256 &&
04770            cast<PointerType>(I.getArgOperand(1)->getType())->getAddressSpace()
04771            < 256 &&
04772            "Unknown address space");
04773     SDValue Op1 = getValue(I.getArgOperand(0));
04774     SDValue Op2 = getValue(I.getArgOperand(1));
04775     SDValue Op3 = getValue(I.getArgOperand(2));
04776     unsigned Align = cast<ConstantInt>(I.getArgOperand(3))->getZExtValue();
04777     if (!Align)
04778       Align = 1; // @llvm.memmove defines 0 and 1 to both mean no alignment.
04779     bool isVol = cast<ConstantInt>(I.getArgOperand(4))->getZExtValue();
04780     DAG.setRoot(DAG.getMemmove(getRoot(), sdl, Op1, Op2, Op3, Align, isVol,
04781                                MachinePointerInfo(I.getArgOperand(0)),
04782                                MachinePointerInfo(I.getArgOperand(1))));
04783     return nullptr;
04784   }
04785   case Intrinsic::dbg_declare: {
04786     const DbgDeclareInst &DI = cast<DbgDeclareInst>(I);
04787     MDNode *Variable = DI.getVariable();
04788     const Value *Address = DI.getAddress();
04789     DIVariable DIVar(Variable);
04790     assert((!DIVar || DIVar.isVariable()) &&
04791       "Variable in DbgDeclareInst should be either null or a DIVariable.");
04792     if (!Address || !DIVar) {
04793       DEBUG(dbgs() << "Dropping debug info for " << DI << "\n");
04794       return nullptr;
04795     }
04796 
04797     // Check if address has undef value.
04798     if (isa<UndefValue>(Address) ||
04799         (Address->use_empty() && !isa<Argument>(Address))) {
04800       DEBUG(dbgs() << "Dropping debug info for " << DI << "\n");
04801       return nullptr;
04802     }
04803 
04804     SDValue &N = NodeMap[Address];
04805     if (!N.getNode() && isa<Argument>(Address))
04806       // Check unused arguments map.
04807       N = UnusedArgNodeMap[Address];
04808     SDDbgValue *SDV;
04809     if (N.getNode()) {
04810       if (const BitCastInst *BCI = dyn_cast<BitCastInst>(Address))
04811         Address = BCI->getOperand(0);
04812       // Parameters are handled specially.
04813       bool isParameter =
04814         (DIVariable(Variable).getTag() == dwarf::DW_TAG_arg_variable ||
04815          isa<Argument>(Address));
04816 
04817       const AllocaInst *AI = dyn_cast<AllocaInst>(Address);
04818 
04819       if (isParameter && !AI) {
04820         FrameIndexSDNode *FINode = dyn_cast<FrameIndexSDNode>(N.getNode());
04821         if (FINode)
04822           // Byval parameter.  We have a frame index at this point.
04823           SDV = DAG.getFrameIndexDbgValue(Variable, FINode->getIndex(),
04824             0, dl, SDNodeOrder);
04825         else {
04826           // Address is an argument, so try to emit its dbg value using
04827           // virtual register info from the FuncInfo.ValueMap.
04828           EmitFuncArgumentDbgValue(Address, Variable, 0, false, N);
04829           return nullptr;
04830         }
04831       } else if (AI)
04832         SDV = DAG.getDbgValue(Variable, N.getNode(), N.getResNo(),
04833                               true, 0, dl, SDNodeOrder);
04834       else {
04835         // Can't do anything with other non-AI cases yet.
04836         DEBUG(dbgs() << "Dropping debug info for " << DI << "\n");
04837         DEBUG(dbgs() << "non-AllocaInst issue for Address: \n\t");
04838         DEBUG(Address->dump());
04839         return nullptr;
04840       }
04841       DAG.AddDbgValue(SDV, N.getNode(), isParameter);
04842     } else {
04843       // If Address is an argument then try to emit its dbg value using
04844       // virtual register info from the FuncInfo.ValueMap.
04845       if (!EmitFuncArgumentDbgValue(Address, Variable, 0, false, N)) {
04846         // If variable is pinned by a alloca in dominating bb then
04847         // use StaticAllocaMap.
04848         if (const AllocaInst *AI = dyn_cast<AllocaInst>(Address)) {
04849           if (AI->getParent() != DI.getParent()) {
04850             DenseMap<const AllocaInst*, int>::iterator SI =
04851               FuncInfo.StaticAllocaMap.find(AI);
04852             if (SI != FuncInfo.StaticAllocaMap.end()) {
04853               SDV = DAG.getFrameIndexDbgValue(Variable, SI->second,
04854                                               0, dl, SDNodeOrder);
04855               DAG.AddDbgValue(SDV, nullptr, false);
04856               return nullptr;
04857             }
04858           }
04859         }
04860         DEBUG(dbgs() << "Dropping debug info for " << DI << "\n");
04861       }
04862     }
04863     return nullptr;
04864   }
04865   case Intrinsic::dbg_value: {
04866     const DbgValueInst &DI = cast<DbgValueInst>(I);
04867     DIVariable DIVar(DI.getVariable());
04868     assert((!DIVar || DIVar.isVariable()) &&
04869       "Variable in DbgValueInst should be either null or a DIVariable.");
04870     if (!DIVar)
04871       return nullptr;
04872 
04873     MDNode *Variable = DI.getVariable();
04874     uint64_t Offset = DI.getOffset();
04875     const Value *V = DI.getValue();
04876     if (!V)
04877       return nullptr;
04878 
04879     SDDbgValue *SDV;
04880     if (isa<ConstantInt>(V) || isa<ConstantFP>(V) || isa<UndefValue>(V)) {
04881       SDV = DAG.getConstantDbgValue(Variable, V, Offset, dl, SDNodeOrder);
04882       DAG.AddDbgValue(SDV, nullptr, false);
04883     } else {
04884       // Do not use getValue() in here; we don't want to generate code at
04885       // this point if it hasn't been done yet.
04886       SDValue N = NodeMap[V];
04887       if (!N.getNode() && isa<Argument>(V))
04888         // Check unused arguments map.
04889         N = UnusedArgNodeMap[V];
04890       if (N.getNode()) {
04891         // A dbg.value for an alloca is always indirect.
04892         bool IsIndirect = isa<AllocaInst>(V) || Offset != 0;
04893         if (!EmitFuncArgumentDbgValue(V, Variable, Offset, IsIndirect, N)) {
04894           SDV = DAG.getDbgValue(Variable, N.getNode(),
04895                                 N.getResNo(), IsIndirect,
04896         Offset, dl, SDNodeOrder);
04897           DAG.AddDbgValue(SDV, N.getNode(), false);
04898         }
04899       } else if (!V->use_empty() ) {
04900         // Do not call getValue(V) yet, as we don't want to generate code.
04901         // Remember it for later.
04902         DanglingDebugInfo DDI(&DI, dl, SDNodeOrder);
04903         DanglingDebugInfoMap[V] = DDI;
04904       } else {
04905         // We may expand this to cover more cases.  One case where we have no
04906         // data available is an unreferenced parameter.
04907         DEBUG(dbgs() << "Dropping debug info for " << DI << "\n");
04908       }
04909     }
04910 
04911     // Build a debug info table entry.
04912     if (const BitCastInst *BCI = dyn_cast<BitCastInst>(V))
04913       V = BCI->getOperand(0);
04914     const AllocaInst *AI = dyn_cast<AllocaInst>(V);
04915     // Don't handle byval struct arguments or VLAs, for example.
04916     if (!AI) {
04917       DEBUG(dbgs() << "Dropping debug location info for:\n  " << DI << "\n");
04918       DEBUG(dbgs() << "  Last seen at:\n    " << *V << "\n");
04919       return nullptr;
04920     }
04921     DenseMap<const AllocaInst*, int>::iterator SI =
04922       FuncInfo.StaticAllocaMap.find(AI);
04923     if (SI == FuncInfo.StaticAllocaMap.end())
04924       return nullptr; // VLAs.
04925     return nullptr;
04926   }
04927 
04928   case Intrinsic::eh_typeid_for: {
04929     // Find the type id for the given typeinfo.
04930     GlobalVariable *GV = ExtractTypeInfo(I.getArgOperand(0));
04931     unsigned TypeID = DAG.getMachineFunction().getMMI().getTypeIDFor(GV);
04932     Res = DAG.getConstant(TypeID, MVT::i32);
04933     setValue(&I, Res);
04934     return nullptr;
04935   }
04936 
04937   case Intrinsic::eh_return_i32:
04938   case Intrinsic::eh_return_i64:
04939     DAG.getMachineFunction().getMMI().setCallsEHReturn(true);
04940     DAG.setRoot(DAG.getNode(ISD::EH_RETURN, sdl,
04941                             MVT::Other,
04942                             getControlRoot(),
04943                             getValue(I.getArgOperand(0)),
04944                             getValue(I.getArgOperand(1))));
04945     return nullptr;
04946   case Intrinsic::eh_unwind_init:
04947     DAG.getMachineFunction().getMMI().setCallsUnwindInit(true);
04948     return nullptr;
04949   case Intrinsic::eh_dwarf_cfa: {
04950     SDValue CfaArg = DAG.getSExtOrTrunc(getValue(I.getArgOperand(0)), sdl,
04951                                         TLI->getPointerTy());
04952     SDValue Offset = DAG.getNode(ISD::ADD, sdl,
04953                                  CfaArg.getValueType(),
04954                                  DAG.getNode(ISD::FRAME_TO_ARGS_OFFSET, sdl,
04955                                              CfaArg.getValueType()),
04956                                  CfaArg);
04957     SDValue FA = DAG.getNode(ISD::FRAMEADDR, sdl,
04958                              TLI->getPointerTy(),
04959                              DAG.getConstant(0, TLI->getPointerTy()));
04960     setValue(&I, DAG.getNode(ISD::ADD, sdl, FA.getValueType(),
04961                              FA, Offset));
04962     return nullptr;
04963   }
04964   case Intrinsic::eh_sjlj_callsite: {
04965     MachineModuleInfo &MMI = DAG.getMachineFunction().getMMI();
04966     ConstantInt *CI = dyn_cast<ConstantInt>(I.getArgOperand(0));
04967     assert(CI && "Non-constant call site value in eh.sjlj.callsite!");
04968     assert(MMI.getCurrentCallSite() == 0 && "Overlapping call sites!");
04969 
04970     MMI.setCurrentCallSite(CI->getZExtValue());
04971     return nullptr;
04972   }
04973   case Intrinsic::eh_sjlj_functioncontext: {
04974     // Get and store the index of the function context.
04975     MachineFrameInfo *MFI = DAG.getMachineFunction().getFrameInfo();
04976     AllocaInst *FnCtx =
04977       cast<AllocaInst>(I.getArgOperand(0)->stripPointerCasts());
04978     int FI = FuncInfo.StaticAllocaMap[FnCtx];
04979     MFI->setFunctionContextIndex(FI);
04980     return nullptr;
04981   }
04982   case Intrinsic::eh_sjlj_setjmp: {
04983     SDValue Ops[2];
04984     Ops[0] = getRoot();
04985     Ops[1] = getValue(I.getArgOperand(0));
04986     SDValue Op = DAG.getNode(ISD::EH_SJLJ_SETJMP, sdl,
04987                              DAG.getVTList(MVT::i32, MVT::Other), Ops);
04988     setValue(&I, Op.getValue(0));
04989     DAG.setRoot(Op.getValue(1));
04990     return nullptr;
04991   }
04992   case Intrinsic::eh_sjlj_longjmp: {
04993     DAG.setRoot(DAG.getNode(ISD::EH_SJLJ_LONGJMP, sdl, MVT::Other,
04994                             getRoot(), getValue(I.getArgOperand(0))));
04995     return nullptr;
04996   }
04997 
04998   case Intrinsic::x86_mmx_pslli_w:
04999   case Intrinsic::x86_mmx_pslli_d:
05000   case Intrinsic::x86_mmx_pslli_q:
05001   case Intrinsic::x86_mmx_psrli_w:
05002   case Intrinsic::x86_mmx_psrli_d:
05003   case Intrinsic::x86_mmx_psrli_q:
05004   case Intrinsic::x86_mmx_psrai_w:
05005   case Intrinsic::x86_mmx_psrai_d: {
05006     SDValue ShAmt = getValue(I.getArgOperand(1));
05007     if (isa<ConstantSDNode>(ShAmt)) {
05008       visitTargetIntrinsic(I, Intrinsic);
05009       return nullptr;
05010     }
05011     unsigned NewIntrinsic = 0;
05012     EVT ShAmtVT = MVT::v2i32;
05013     switch (Intrinsic) {
05014     case Intrinsic::x86_mmx_pslli_w:
05015       NewIntrinsic = Intrinsic::x86_mmx_psll_w;
05016       break;
05017     case Intrinsic::x86_mmx_pslli_d:
05018       NewIntrinsic = Intrinsic::x86_mmx_psll_d;
05019       break;
05020     case Intrinsic::x86_mmx_pslli_q:
05021       NewIntrinsic = Intrinsic::x86_mmx_psll_q;
05022       break;
05023     case Intrinsic::x86_mmx_psrli_w:
05024       NewIntrinsic = Intrinsic::x86_mmx_psrl_w;
05025       break;
05026     case Intrinsic::x86_mmx_psrli_d:
05027       NewIntrinsic = Intrinsic::x86_mmx_psrl_d;
05028       break;
05029     case Intrinsic::x86_mmx_psrli_q:
05030       NewIntrinsic = Intrinsic::x86_mmx_psrl_q;
05031       break;
05032     case Intrinsic::x86_mmx_psrai_w:
05033       NewIntrinsic = Intrinsic::x86_mmx_psra_w;
05034       break;
05035     case Intrinsic::x86_mmx_psrai_d:
05036       NewIntrinsic = Intrinsic::x86_mmx_psra_d;
05037       break;
05038     default: llvm_unreachable("Impossible intrinsic");  // Can't reach here.
05039     }
05040 
05041     // The vector shift intrinsics with scalars uses 32b shift amounts but
05042     // the sse2/mmx shift instructions reads 64 bits. Set the upper 32 bits
05043     // to be zero.
05044     // We must do this early because v2i32 is not a legal type.
05045     SDValue ShOps[2];
05046     ShOps[0] = ShAmt;
05047     ShOps[1] = DAG.getConstant(0, MVT::i32);
05048     ShAmt =  DAG.getNode(ISD::BUILD_VECTOR, sdl, ShAmtVT, ShOps);
05049     EVT DestVT = TLI->getValueType(I.getType());
05050     ShAmt = DAG.getNode(ISD::BITCAST, sdl, DestVT, ShAmt);
05051     Res = DAG.getNode(ISD::INTRINSIC_WO_CHAIN, sdl, DestVT,
05052                        DAG.getConstant(NewIntrinsic, MVT::i32),
05053                        getValue(I.getArgOperand(0)), ShAmt);
05054     setValue(&I, Res);
05055     return nullptr;
05056   }
05057   case Intrinsic::x86_avx_vinsertf128_pd_256:
05058   case Intrinsic::x86_avx_vinsertf128_ps_256:
05059   case Intrinsic::x86_avx_vinsertf128_si_256:
05060   case Intrinsic::x86_avx2_vinserti128: {
05061     EVT DestVT = TLI->getValueType(I.getType());
05062     EVT ElVT = TLI->getValueType(I.getArgOperand(1)->getType());
05063     uint64_t Idx = (cast<ConstantInt>(I.getArgOperand(2))->getZExtValue() & 1) *
05064                    ElVT.getVectorNumElements();
05065     Res = DAG.getNode(ISD::INSERT_SUBVECTOR, sdl, DestVT,
05066                       getValue(I.getArgOperand(0)),
05067                       getValue(I.getArgOperand(1)),
05068                       DAG.getConstant(Idx, TLI->getVectorIdxTy()));
05069     setValue(&I, Res);
05070     return nullptr;
05071   }
05072   case Intrinsic::x86_avx_vextractf128_pd_256:
05073   case Intrinsic::x86_avx_vextractf128_ps_256:
05074   case Intrinsic::x86_avx_vextractf128_si_256:
05075   case Intrinsic::x86_avx2_vextracti128: {
05076     EVT DestVT = TLI->getValueType(I.getType());
05077     uint64_t Idx = (cast<ConstantInt>(I.getArgOperand(1))->getZExtValue() & 1) *
05078                    DestVT.getVectorNumElements();
05079     Res = DAG.getNode(ISD::EXTRACT_SUBVECTOR, sdl, DestVT,
05080                       getValue(I.getArgOperand(0)),
05081                       DAG.getConstant(Idx, TLI->getVectorIdxTy()));
05082     setValue(&I, Res);
05083     return nullptr;
05084   }
05085   case Intrinsic::convertff:
05086   case Intrinsic::convertfsi:
05087   case Intrinsic::convertfui:
05088   case Intrinsic::convertsif:
05089   case Intrinsic::convertuif:
05090   case Intrinsic::convertss:
05091   case Intrinsic::convertsu:
05092   case Intrinsic::convertus:
05093   case Intrinsic::convertuu: {
05094     ISD::CvtCode Code = ISD::CVT_INVALID;
05095     switch (Intrinsic) {
05096     default: llvm_unreachable("Impossible intrinsic");  // Can't reach here.
05097     case Intrinsic::convertff:  Code = ISD::CVT_FF; break;
05098     case Intrinsic::convertfsi: Code = ISD::CVT_FS; break;
05099     case Intrinsic::convertfui: Code = ISD::CVT_FU; break;
05100     case Intrinsic::convertsif: Code = ISD::CVT_SF; break;
05101     case Intrinsic::convertuif: Code = ISD::CVT_UF; break;
05102     case Intrinsic::convertss:  Code = ISD::CVT_SS; break;
05103     case Intrinsic::convertsu:  Code = ISD::CVT_SU; break;
05104     case Intrinsic::convertus:  Code = ISD::CVT_US; break;
05105     case Intrinsic::convertuu:  Code = ISD::CVT_UU; break;
05106     }
05107     EVT DestVT = TLI->getValueType(I.getType());
05108     const Value *Op1 = I.getArgOperand(0);
05109     Res = DAG.getConvertRndSat(DestVT, sdl, getValue(Op1),
05110                                DAG.getValueType(DestVT),
05111                                DAG.getValueType(getValue(Op1).getValueType()),
05112                                getValue(I.getArgOperand(1)),
05113                                getValue(I.getArgOperand(2)),
05114                                Code);
05115     setValue(&I, Res);
05116     return nullptr;
05117   }
05118   case Intrinsic::powi:
05119     setValue(&I, ExpandPowI(sdl, getValue(I.getArgOperand(0)),
05120                             getValue(I.getArgOperand(1)), DAG));
05121     return nullptr;
05122   case Intrinsic::log:
05123     setValue(&I, expandLog(sdl, getValue(I.getArgOperand(0)), DAG, *TLI));
05124     return nullptr;
05125   case Intrinsic::log2:
05126     setValue(&I, expandLog2(sdl, getValue(I.getArgOperand(0)), DAG, *TLI));
05127     return nullptr;
05128   case Intrinsic::log10:
05129     setValue(&I, expandLog10(sdl, getValue(I.getArgOperand(0)), DAG, *TLI));
05130     return nullptr;
05131   case Intrinsic::exp:
05132     setValue(&I, expandExp(sdl, getValue(I.getArgOperand(0)), DAG, *TLI));
05133     return nullptr;
05134   case Intrinsic::exp2:
05135     setValue(&I, expandExp2(sdl, getValue(I.getArgOperand(0)), DAG, *TLI));
05136     return nullptr;
05137   case Intrinsic::pow:
05138     setValue(&I, expandPow(sdl, getValue(I.getArgOperand(0)),
05139                            getValue(I.getArgOperand(1)), DAG, *TLI));
05140     return nullptr;
05141   case Intrinsic::sqrt:
05142   case Intrinsic::fabs:
05143   case Intrinsic::sin:
05144   case Intrinsic::cos:
05145   case Intrinsic::floor:
05146   case Intrinsic::ceil:
05147   case Intrinsic::trunc:
05148   case Intrinsic::rint:
05149   case Intrinsic::nearbyint:
05150   case Intrinsic::round: {
05151     unsigned Opcode;
05152     switch (Intrinsic) {
05153     default: llvm_unreachable("Impossible intrinsic");  // Can't reach here.
05154     case Intrinsic::sqrt:      Opcode = ISD::FSQRT;      break;
05155     case Intrinsic::fabs:      Opcode = ISD::FABS;       break;
05156     case Intrinsic::sin:       Opcode = ISD::FSIN;       break;
05157     case Intrinsic::cos:       Opcode = ISD::FCOS;       break;
05158     case Intrinsic::floor:     Opcode = ISD::FFLOOR;     break;
05159     case Intrinsic::ceil:      Opcode = ISD::FCEIL;      break;
05160     case Intrinsic::trunc:     Opcode = ISD::FTRUNC;     break;
05161     case Intrinsic::rint:      Opcode = ISD::FRINT;      break;
05162     case Intrinsic::nearbyint: Opcode = ISD::FNEARBYINT; break;
05163     case Intrinsic::round:     Opcode = ISD::FROUND;     break;
05164     }
05165 
05166     setValue(&I, DAG.getNode(Opcode, sdl,
05167                              getValue(I.getArgOperand(0)).getValueType(),
05168                              getValue(I.getArgOperand(0))));
05169     return nullptr;
05170   }
05171   case Intrinsic::copysign:
05172     setValue(&I, DAG.getNode(ISD::FCOPYSIGN, sdl,
05173                              getValue(I.getArgOperand(0)).getValueType(),
05174                              getValue(I.getArgOperand(0)),
05175                              getValue(I.getArgOperand(1))));
05176     return nullptr;
05177   case Intrinsic::fma:
05178     setValue(&I, DAG.getNode(ISD::FMA, sdl,
05179                              getValue(I.getArgOperand(0)).getValueType(),
05180                              getValue(I.getArgOperand(0)),
05181                              getValue(I.getArgOperand(1)),
05182                              getValue(I.getArgOperand(2))));
05183     return nullptr;
05184   case Intrinsic::fmuladd: {
05185     EVT VT = TLI->getValueType(I.getType());
05186     if (TM.Options.AllowFPOpFusion != FPOpFusion::Strict &&
05187         TLI->isFMAFasterThanFMulAndFAdd(VT)) {
05188       setValue(&I, DAG.getNode(ISD::FMA, sdl,
05189                                getValue(I.getArgOperand(0)).getValueType(),
05190                                getValue(I.getArgOperand(0)),
05191                                getValue(I.getArgOperand(1)),
05192                                getValue(I.getArgOperand(2))));
05193     } else {
05194       SDValue Mul = DAG.getNode(ISD::FMUL, sdl,
05195                                 getValue(I.getArgOperand(0)).getValueType(),
05196                                 getValue(I.getArgOperand(0)),
05197                                 getValue(I.getArgOperand(1)));
05198       SDValue Add = DAG.getNode(ISD::FADD, sdl,
05199                                 getValue(I.getArgOperand(0)).getValueType(),
05200                                 Mul,
05201                                 getValue(I.getArgOperand(2)));
05202       setValue(&I, Add);
05203     }
05204     return nullptr;
05205   }
05206   case Intrinsic::convert_to_fp16:
05207     setValue(&I, DAG.getNode(ISD::BITCAST, sdl, MVT::i16,
05208                              DAG.getNode(ISD::FP_ROUND, sdl, MVT::f16,
05209                                          getValue(I.getArgOperand(0)),
05210                                          DAG.getTargetConstant(0, MVT::i32))));
05211     return nullptr;
05212   case Intrinsic::convert_from_fp16:
05213     setValue(&I,
05214              DAG.getNode(ISD::FP_EXTEND, sdl, TLI->getValueType(I.getType()),
05215                          DAG.getNode(ISD::BITCAST, sdl, MVT::f16,
05216                                      getValue(I.getArgOperand(0)))));
05217     return nullptr;
05218   case Intrinsic::pcmarker: {
05219     SDValue Tmp = getValue(I.getArgOperand(0));
05220     DAG.setRoot(DAG.getNode(ISD::PCMARKER, sdl, MVT::Other, getRoot(), Tmp));
05221     return nullptr;
05222   }
05223   case Intrinsic::readcyclecounter: {
05224     SDValue Op = getRoot();
05225     Res = DAG.getNode(ISD::READCYCLECOUNTER, sdl,
05226                       DAG.getVTList(MVT::i64, MVT::Other), Op);
05227     setValue(&I, Res);
05228     DAG.setRoot(Res.getValue(1));
05229     return nullptr;
05230   }
05231   case Intrinsic::bswap:
05232     setValue(&I, DAG.getNode(ISD::BSWAP, sdl,
05233                              getValue(I.getArgOperand(0)).getValueType(),
05234                              getValue(I.getArgOperand(0))));
05235     return nullptr;
05236   case Intrinsic::cttz: {
05237     SDValue Arg = getValue(I.getArgOperand(0));
05238     ConstantInt *CI = cast<ConstantInt>(I.getArgOperand(1));
05239     EVT Ty = Arg.getValueType();
05240     setValue(&I, DAG.getNode(CI->isZero() ? ISD::CTTZ : ISD::CTTZ_ZERO_UNDEF,
05241                              sdl, Ty, Arg));
05242     return nullptr;
05243   }
05244   case Intrinsic::ctlz: {
05245     SDValue Arg = getValue(I.getArgOperand(0));
05246     ConstantInt *CI = cast<ConstantInt>(I.getArgOperand(1));
05247     EVT Ty = Arg.getValueType();
05248     setValue(&I, DAG.getNode(CI->isZero() ? ISD::CTLZ : ISD::CTLZ_ZERO_UNDEF,
05249                              sdl, Ty, Arg));
05250     return nullptr;
05251   }
05252   case Intrinsic::ctpop: {
05253     SDValue Arg = getValue(I.getArgOperand(0));
05254     EVT Ty = Arg.getValueType();
05255     setValue(&I, DAG.getNode(ISD::CTPOP, sdl, Ty, Arg));
05256     return nullptr;
05257   }
05258   case Intrinsic::stacksave: {
05259     SDValue Op = getRoot();
05260     Res = DAG.getNode(ISD::STACKSAVE, sdl,
05261                       DAG.getVTList(TLI->getPointerTy(), MVT::Other), Op);
05262     setValue(&I, Res);
05263     DAG.setRoot(Res.getValue(1));
05264     return nullptr;
05265   }
05266   case Intrinsic::stackrestore: {
05267     Res = getValue(I.getArgOperand(0));
05268     DAG.setRoot(DAG.getNode(ISD::STACKRESTORE, sdl, MVT::Other, getRoot(), Res));
05269     return nullptr;
05270   }
05271   case Intrinsic::stackprotector: {
05272     // Emit code into the DAG to store the stack guard onto the stack.
05273     MachineFunction &MF = DAG.getMachineFunction();
05274     MachineFrameInfo *MFI = MF.getFrameInfo();
05275     EVT PtrTy = TLI->getPointerTy();
05276     SDValue Src, Chain = getRoot();
05277     const Value *Ptr = cast<LoadInst>(I.getArgOperand(0))->getPointerOperand();
05278     const GlobalVariable *GV = dyn_cast<GlobalVariable>(Ptr);
05279 
05280     // See if Ptr is a bitcast. If it is, look through it and see if we can get
05281     // global variable __stack_chk_guard.
05282     if (!GV)
05283       if (const Operator *BC = dyn_cast<Operator>(Ptr))
05284         if (BC->getOpcode() == Instruction::BitCast)
05285           GV = dyn_cast<GlobalVariable>(BC->getOperand(0));
05286 
05287     if (GV && TLI->useLoadStackGuardNode()) {
05288       // Emit a LOAD_STACK_GUARD node.
05289       MachineSDNode *Node = DAG.getMachineNode(TargetOpcode::LOAD_STACK_GUARD,
05290                                                sdl, PtrTy, Chain);
05291       MachinePointerInfo MPInfo(GV);
05292       MachineInstr::mmo_iterator MemRefs = MF.allocateMemRefsArray(1);
05293       unsigned Flags = MachineMemOperand::MOLoad |
05294                        MachineMemOperand::MOInvariant;
05295       *MemRefs = MF.getMachineMemOperand(MPInfo, Flags,
05296                                          PtrTy.getSizeInBits() / 8,
05297                                          DAG.getEVTAlignment(PtrTy));
05298       Node->setMemRefs(MemRefs, MemRefs + 1);
05299 
05300       // Copy the guard value to a virtual register so that it can be
05301       // retrieved in the epilogue.
05302       Src = SDValue(Node, 0);
05303       const TargetRegisterClass *RC =
05304           TLI->getRegClassFor(Src.getSimpleValueType());
05305       unsigned Reg = MF.getRegInfo().createVirtualRegister(RC);
05306 
05307       SPDescriptor.setGuardReg(Reg);
05308       Chain = DAG.getCopyToReg(Chain, sdl, Reg, Src);
05309     } else {
05310       Src = getValue(I.getArgOperand(0));   // The guard's value.
05311     }
05312 
05313     AllocaInst *Slot = cast<AllocaInst>(I.getArgOperand(1));
05314 
05315     int FI = FuncInfo.StaticAllocaMap[Slot];
05316     MFI->setStackProtectorIndex(FI);
05317 
05318     SDValue FIN = DAG.getFrameIndex(FI, PtrTy);
05319 
05320     // Store the stack protector onto the stack.
05321     Res = DAG.getStore(Chain, sdl, Src, FIN,
05322                        MachinePointerInfo::getFixedStack(FI),
05323                        true, false, 0);
05324     setValue(&I, Res);
05325     DAG.setRoot(Res);
05326     return nullptr;
05327   }
05328   case Intrinsic::objectsize: {
05329     // If we don't know by now, we're never going to know.
05330     ConstantInt *CI = dyn_cast<ConstantInt>(I.getArgOperand(1));
05331 
05332     assert(CI && "Non-constant type in __builtin_object_size?");
05333 
05334     SDValue Arg = getValue(I.getCalledValue());
05335     EVT Ty = Arg.getValueType();
05336 
05337     if (CI->isZero())
05338       Res = DAG.getConstant(-1ULL, Ty);
05339     else
05340       Res = DAG.getConstant(0, Ty);
05341 
05342     setValue(&I, Res);
05343     return nullptr;
05344   }
05345   case Intrinsic::annotation:
05346   case Intrinsic::ptr_annotation:
05347     // Drop the intrinsic, but forward the value
05348     setValue(&I, getValue(I.getOperand(0)));
05349     return nullptr;
05350   case Intrinsic::assume:
05351   case Intrinsic::var_annotation:
05352     // Discard annotate attributes and assumptions
05353     return nullptr;
05354 
05355   case Intrinsic::init_trampoline: {
05356     const Function *F = cast<Function>(I.getArgOperand(1)->stripPointerCasts());
05357 
05358     SDValue Ops[6];
05359     Ops[0] = getRoot();
05360     Ops[1] = getValue(I.getArgOperand(0));
05361     Ops[2] = getValue(I.getArgOperand(1));
05362     Ops[3] = getValue(I.getArgOperand(2));
05363     Ops[4] = DAG.getSrcValue(I.getArgOperand(0));
05364     Ops[5] = DAG.getSrcValue(F);
05365 
05366     Res = DAG.getNode(ISD::INIT_TRAMPOLINE, sdl, MVT::Other, Ops);
05367 
05368     DAG.setRoot(Res);
05369     return nullptr;
05370   }
05371   case Intrinsic::adjust_trampoline: {
05372     setValue(&I, DAG.getNode(ISD::ADJUST_TRAMPOLINE, sdl,
05373                              TLI->getPointerTy(),
05374                              getValue(I.getArgOperand(0))));
05375     return nullptr;
05376   }
05377   case Intrinsic::gcroot:
05378     if (GFI) {
05379       const Value *Alloca = I.getArgOperand(0)->stripPointerCasts();
05380       const Constant *TypeMap = cast<Constant>(I.getArgOperand(1));
05381 
05382       FrameIndexSDNode *FI = cast<FrameIndexSDNode>(getValue(Alloca).getNode());
05383       GFI->addStackRoot(FI->getIndex(), TypeMap);
05384     }
05385     return nullptr;
05386   case Intrinsic::gcread:
05387   case Intrinsic::gcwrite:
05388     llvm_unreachable("GC failed to lower gcread/gcwrite intrinsics!");
05389   case Intrinsic::flt_rounds:
05390     setValue(&I, DAG.getNode(ISD::FLT_ROUNDS_, sdl, MVT::i32));
05391     return nullptr;
05392 
05393   case Intrinsic::expect: {
05394     // Just replace __builtin_expect(exp, c) with EXP.
05395     setValue(&I, getValue(I.getArgOperand(0)));
05396     return nullptr;
05397   }
05398 
05399   case Intrinsic::debugtrap:
05400   case Intrinsic::trap: {
05401     StringRef TrapFuncName = TM.Options.getTrapFunctionName();
05402     if (TrapFuncName.empty()) {
05403       ISD::NodeType Op = (Intrinsic == Intrinsic::trap) ?
05404         ISD::TRAP : ISD::DEBUGTRAP;
05405       DAG.setRoot(DAG.getNode(Op, sdl,MVT::Other, getRoot()));
05406       return nullptr;
05407     }
05408     TargetLowering::ArgListTy Args;
05409 
05410     TargetLowering::CallLoweringInfo CLI(DAG);
05411     CLI.setDebugLoc(sdl).setChain(getRoot())
05412       .setCallee(CallingConv::C, I.getType(),
05413                  DAG.getExternalSymbol(TrapFuncName.data(), TLI->getPointerTy()),
05414                  std::move(Args), 0);
05415 
05416     std::pair<SDValue, SDValue> Result = TLI->LowerCallTo(CLI);
05417     DAG.setRoot(Result.second);
05418     return nullptr;
05419   }
05420 
05421   case Intrinsic::uadd_with_overflow:
05422   case Intrinsic::sadd_with_overflow:
05423   case Intrinsic::usub_with_overflow:
05424   case Intrinsic::ssub_with_overflow:
05425   case Intrinsic::umul_with_overflow:
05426   case Intrinsic::smul_with_overflow: {
05427     ISD::NodeType Op;
05428     switch (Intrinsic) {
05429     default: llvm_unreachable("Impossible intrinsic");  // Can't reach here.
05430     case Intrinsic::uadd_with_overflow: Op = ISD::UADDO; break;
05431     case Intrinsic::sadd_with_overflow: Op = ISD::SADDO; break;
05432     case Intrinsic::usub_with_overflow: Op = ISD::USUBO; break;
05433     case Intrinsic::ssub_with_overflow: Op = ISD::SSUBO; break;
05434     case Intrinsic::umul_with_overflow: Op = ISD::UMULO; break;
05435     case Intrinsic::smul_with_overflow: Op = ISD::SMULO; break;
05436     }
05437     SDValue Op1 = getValue(I.getArgOperand(0));
05438     SDValue Op2 = getValue(I.getArgOperand(1));
05439 
05440     SDVTList VTs = DAG.getVTList(Op1.getValueType(), MVT::i1);
05441     setValue(&I, DAG.getNode(Op, sdl, VTs, Op1, Op2));
05442     return nullptr;
05443   }
05444   case Intrinsic::prefetch: {
05445     SDValue Ops[5];
05446     unsigned rw = cast<ConstantInt>(I.getArgOperand(1))->getZExtValue();
05447     Ops[0] = getRoot();
05448     Ops[1] = getValue(I.getArgOperand(0));
05449     Ops[2] = getValue(I.getArgOperand(1));
05450     Ops[3] = getValue(I.getArgOperand(2));
05451     Ops[4] = getValue(I.getArgOperand(3));
05452     DAG.setRoot(DAG.getMemIntrinsicNode(ISD::PREFETCH, sdl,
05453                                         DAG.getVTList(MVT::Other), Ops,
05454                                         EVT::getIntegerVT(*Context, 8),
05455                                         MachinePointerInfo(I.getArgOperand(0)),
05456                                         0, /* align */
05457                                         false, /* volatile */
05458                                         rw==0, /* read */
05459                                         rw==1)); /* write */
05460     return nullptr;
05461   }
05462   case Intrinsic::lifetime_start:
05463   case Intrinsic::lifetime_end: {
05464     bool IsStart = (Intrinsic == Intrinsic::lifetime_start);
05465     // Stack coloring is not enabled in O0, discard region information.
05466     if (TM.getOptLevel() == CodeGenOpt::None)
05467       return nullptr;
05468 
05469     SmallVector<Value *, 4> Allocas;
05470     GetUnderlyingObjects(I.getArgOperand(1), Allocas, DL);
05471 
05472     for (SmallVectorImpl<Value*>::iterator Object = Allocas.begin(),
05473            E = Allocas.end(); Object != E; ++Object) {
05474       AllocaInst *LifetimeObject = dyn_cast_or_null<AllocaInst>(*Object);
05475 
05476       // Could not find an Alloca.
05477       if (!LifetimeObject)
05478         continue;
05479 
05480       int FI = FuncInfo.StaticAllocaMap[LifetimeObject];
05481 
05482       SDValue Ops[2];
05483       Ops[0] = getRoot();
05484       Ops[1] = DAG.getFrameIndex(FI, TLI->getPointerTy(), true);
05485       unsigned Opcode = (IsStart ? ISD::LIFETIME_START : ISD::LIFETIME_END);
05486 
05487       Res = DAG.getNode(Opcode, sdl, MVT::Other, Ops);
05488       DAG.setRoot(Res);
05489     }
05490     return nullptr;
05491   }
05492   case Intrinsic::invariant_start:
05493     // Discard region information.
05494     setValue(&I, DAG.getUNDEF(TLI->getPointerTy()));
05495     return nullptr;
05496   case Intrinsic::invariant_end:
05497     // Discard region information.
05498     return nullptr;
05499   case Intrinsic::stackprotectorcheck: {
05500     // Do not actually emit anything for this basic block. Instead we initialize
05501     // the stack protector descriptor and export the guard variable so we can
05502     // access it in FinishBasicBlock.
05503     const BasicBlock *BB = I.getParent();
05504     SPDescriptor.initialize(BB, FuncInfo.MBBMap[BB], I);
05505     ExportFromCurrentBlock(SPDescriptor.getGuard());
05506 
05507     // Flush our exports since we are going to process a terminator.
05508     (void)getControlRoot();
05509     return nullptr;
05510   }
05511   case Intrinsic::clear_cache:
05512     return TLI->getClearCacheBuiltinName();
05513   case Intrinsic::donothing:
05514     // ignore
05515     return nullptr;
05516   case Intrinsic::experimental_stackmap: {
05517     visitStackmap(I);
05518     return nullptr;
05519   }
05520   case Intrinsic::experimental_patchpoint_void:
05521   case Intrinsic::experimental_patchpoint_i64: {
05522     visitPatchpoint(I);
05523     return nullptr;
05524   }
05525   }
05526 }
05527 
05528 void SelectionDAGBuilder::LowerCallTo(ImmutableCallSite CS, SDValue Callee,
05529                                       bool isTailCall,
05530                                       MachineBasicBlock *LandingPad) {
05531   const TargetLowering *TLI = TM.getSubtargetImpl()->getTargetLowering();
05532   PointerType *PT = cast<PointerType>(CS.getCalledValue()->getType());
05533   FunctionType *FTy = cast<FunctionType>(PT->getElementType());
05534   Type *RetTy = FTy->getReturnType();
05535   MachineModuleInfo &MMI = DAG.getMachineFunction().getMMI();
05536   MCSymbol *BeginLabel = nullptr;
05537 
05538   TargetLowering::ArgListTy Args;
05539   TargetLowering::ArgListEntry Entry;
05540   Args.reserve(CS.arg_size());
05541 
05542   for (ImmutableCallSite::arg_iterator i = CS.arg_begin(), e = CS.arg_end();
05543        i != e; ++i) {
05544     const Value *V = *i;
05545 
05546     // Skip empty types
05547     if (V->getType()->isEmptyTy())
05548       continue;
05549 
05550     SDValue ArgNode = getValue(V);
05551     Entry.Node = ArgNode; Entry.Ty = V->getType();
05552 
05553     // Skip the first return-type Attribute to get to params.
05554     Entry.setAttributes(&CS, i - CS.arg_begin() + 1);
05555     Args.push_back(Entry);
05556   }
05557 
05558   if (LandingPad) {
05559     // Insert a label before the invoke call to mark the try range.  This can be
05560     // used to detect deletion of the invoke via the MachineModuleInfo.
05561     BeginLabel = MMI.getContext().CreateTempSymbol();
05562 
05563     // For SjLj, keep track of which landing pads go with which invokes
05564     // so as to maintain the ordering of pads in the LSDA.
05565     unsigned CallSiteIndex = MMI.getCurrentCallSite();
05566     if (CallSiteIndex) {
05567       MMI.setCallSiteBeginLabel(BeginLabel, CallSiteIndex);
05568       LPadToCallSiteMap[LandingPad].push_back(CallSiteIndex);
05569 
05570       // Now that the call site is handled, stop tracking it.
05571       MMI.setCurrentCallSite(0);
05572     }
05573 
05574     // Both PendingLoads and PendingExports must be flushed here;
05575     // this call might not return.
05576     (void)getRoot();
05577     DAG.setRoot(DAG.getEHLabel(getCurSDLoc(), getControlRoot(), BeginLabel));
05578   }
05579 
05580   // Check if target-independent constraints permit a tail call here.
05581   // Target-dependent constraints are checked within TLI->LowerCallTo.
05582   if (isTailCall && !isInTailCallPosition(CS, DAG.getTarget()))
05583     isTailCall = false;
05584 
05585   TargetLowering::CallLoweringInfo CLI(DAG);
05586   CLI.setDebugLoc(getCurSDLoc()).setChain(getRoot())
05587     .setCallee(RetTy, FTy, Callee, std::move(Args), CS).setTailCall(isTailCall);
05588 
05589   std::pair<SDValue,SDValue> Result = TLI->LowerCallTo(CLI);
05590   assert((isTailCall || Result.second.getNode()) &&
05591          "Non-null chain expected with non-tail call!");
05592   assert((Result.second.getNode() || !Result.first.getNode()) &&
05593          "Null value expected with tail call!");
05594   if (Result.first.getNode())
05595     setValue(CS.getInstruction(), Result.first);
05596 
05597   if (!Result.second.getNode()) {
05598     // As a special case, a null chain means that a tail call has been emitted
05599     // and the DAG root is already updated.
05600     HasTailCall = true;
05601 
05602     // Since there's no actual continuation from this block, nothing can be
05603     // relying on us setting vregs for them.
05604     PendingExports.clear();
05605   } else {
05606     DAG.setRoot(Result.second);
05607   }
05608 
05609   if (LandingPad) {
05610     // Insert a label at the end of the invoke call to mark the try range.  This
05611     // can be used to detect deletion of the invoke via the MachineModuleInfo.
05612     MCSymbol *EndLabel = MMI.getContext().CreateTempSymbol();
05613     DAG.setRoot(DAG.getEHLabel(getCurSDLoc(), getRoot(), EndLabel));
05614 
05615     // Inform MachineModuleInfo of range.
05616     MMI.addInvoke(LandingPad, BeginLabel, EndLabel);
05617   }
05618 }
05619 
05620 /// IsOnlyUsedInZeroEqualityComparison - Return true if it only matters that the
05621 /// value is equal or not-equal to zero.
05622 static bool IsOnlyUsedInZeroEqualityComparison(const Value *V) {
05623   for (const User *U : V->users()) {
05624     if (const ICmpInst *IC = dyn_cast<ICmpInst>(U))
05625       if (IC->isEquality())
05626         if (const Constant *C = dyn_cast<Constant>(IC->getOperand(1)))
05627           if (C->isNullValue())
05628             continue;
05629     // Unknown instruction.
05630     return false;
05631   }
05632   return true;
05633 }
05634 
05635 static SDValue getMemCmpLoad(const Value *PtrVal, MVT LoadVT,
05636                              Type *LoadTy,
05637                              SelectionDAGBuilder &Builder) {
05638 
05639   // Check to see if this load can be trivially constant folded, e.g. if the
05640   // input is from a string literal.
05641   if (const Constant *LoadInput = dyn_cast<Constant>(PtrVal)) {
05642     // Cast pointer to the type we really want to load.
05643     LoadInput = ConstantExpr::getBitCast(const_cast<Constant *>(LoadInput),
05644                                          PointerType::getUnqual(LoadTy));
05645 
05646     if (const Constant *LoadCst =
05647           ConstantFoldLoadFromConstPtr(const_cast<Constant *>(LoadInput),
05648                                        Builder.DL))
05649       return Builder.getValue(LoadCst);
05650   }
05651 
05652   // Otherwise, we have to emit the load.  If the pointer is to unfoldable but
05653   // still constant memory, the input chain can be the entry node.
05654   SDValue Root;
05655   bool ConstantMemory = false;
05656 
05657   // Do not serialize (non-volatile) loads of constant memory with anything.
05658   if (Builder.AA->pointsToConstantMemory(PtrVal)) {
05659     Root = Builder.DAG.getEntryNode();
05660     ConstantMemory = true;
05661   } else {
05662     // Do not serialize non-volatile loads against each other.
05663     Root = Builder.DAG.getRoot();
05664   }
05665 
05666   SDValue Ptr = Builder.getValue(PtrVal);
05667   SDValue LoadVal = Builder.DAG.getLoad(LoadVT, Builder.getCurSDLoc(), Root,
05668                                         Ptr, MachinePointerInfo(PtrVal),
05669                                         false /*volatile*/,
05670                                         false /*nontemporal*/,
05671                                         false /*isinvariant*/, 1 /* align=1 */);
05672 
05673   if (!ConstantMemory)
05674     Builder.PendingLoads.push_back(LoadVal.getValue(1));
05675   return LoadVal;
05676 }
05677 
05678 /// processIntegerCallValue - Record the value for an instruction that
05679 /// produces an integer result, converting the type where necessary.
05680 void SelectionDAGBuilder::processIntegerCallValue(const Instruction &I,
05681                                                   SDValue Value,
05682                                                   bool IsSigned) {
05683   EVT VT = TM.getSubtargetImpl()->getTargetLowering()->getValueType(I.getType(),
05684                                                                     true);
05685   if (IsSigned)
05686     Value = DAG.getSExtOrTrunc(Value, getCurSDLoc(), VT);
05687   else
05688     Value = DAG.getZExtOrTrunc(Value, getCurSDLoc(), VT);
05689   setValue(&I, Value);
05690 }
05691 
05692 /// visitMemCmpCall - See if we can lower a call to memcmp in an optimized form.
05693 /// If so, return true and lower it, otherwise return false and it will be
05694 /// lowered like a normal call.
05695 bool SelectionDAGBuilder::visitMemCmpCall(const CallInst &I) {
05696   // Verify that the prototype makes sense.  int memcmp(void*,void*,size_t)
05697   if (I.getNumArgOperands() != 3)
05698     return false;
05699 
05700   const Value *LHS = I.getArgOperand(0), *RHS = I.getArgOperand(1);
05701   if (!LHS->getType()->isPointerTy() || !RHS->getType()->isPointerTy() ||
05702       !I.getArgOperand(2)->getType()->isIntegerTy() ||
05703       !I.getType()->isIntegerTy())
05704     return false;
05705 
05706   const Value *Size = I.getArgOperand(2);
05707   const ConstantInt *CSize = dyn_cast<ConstantInt>(Size);
05708   if (CSize && CSize->getZExtValue() == 0) {
05709     EVT CallVT = TM.getSubtargetImpl()->getTargetLowering()->getValueType(
05710         I.getType(), true);
05711     setValue(&I, DAG.getConstant(0, CallVT));
05712     return true;
05713   }
05714 
05715   const TargetSelectionDAGInfo &TSI = DAG.getSelectionDAGInfo();
05716   std::pair<SDValue, SDValue> Res =
05717     TSI.EmitTargetCodeForMemcmp(DAG, getCurSDLoc(), DAG.getRoot(),
05718                                 getValue(LHS), getValue(RHS), getValue(Size),
05719                                 MachinePointerInfo(LHS),
05720                                 MachinePointerInfo(RHS));
05721   if (Res.first.getNode()) {
05722     processIntegerCallValue(I, Res.first, true);
05723     PendingLoads.push_back(Res.second);
05724     return true;
05725   }
05726 
05727   // memcmp(S1,S2,2) != 0 -> (*(short*)LHS != *(short*)RHS)  != 0
05728   // memcmp(S1,S2,4) != 0 -> (*(int*)LHS != *(int*)RHS)  != 0
05729   if (CSize && IsOnlyUsedInZeroEqualityComparison(&I)) {
05730     bool ActuallyDoIt = true;
05731     MVT LoadVT;
05732     Type *LoadTy;
05733     switch (CSize->getZExtValue()) {
05734     default:
05735       LoadVT = MVT::Other;
05736       LoadTy = nullptr;
05737       ActuallyDoIt = false;
05738       break;
05739     case 2:
05740       LoadVT = MVT::i16;
05741       LoadTy = Type::getInt16Ty(CSize->getContext());
05742       break;
05743     case 4:
05744       LoadVT = MVT::i32;
05745       LoadTy = Type::getInt32Ty(CSize->getContext());
05746       break;
05747     case 8:
05748       LoadVT = MVT::i64;
05749       LoadTy = Type::getInt64Ty(CSize->getContext());
05750       break;
05751         /*
05752     case 16:
05753       LoadVT = MVT::v4i32;
05754       LoadTy = Type::getInt32Ty(CSize->getContext());
05755       LoadTy = VectorType::get(LoadTy, 4);
05756       break;
05757          */
05758     }
05759 
05760     // This turns into unaligned loads.  We only do this if the target natively
05761     // supports the MVT we'll be loading or if it is small enough (<= 4) that
05762     // we'll only produce a small number of byte loads.
05763 
05764     // Require that we can find a legal MVT, and only do this if the target
05765     // supports unaligned loads of that type.  Expanding into byte loads would
05766     // bloat the code.
05767     const TargetLowering *TLI = TM.getSubtargetImpl()->getTargetLowering();
05768     if (ActuallyDoIt && CSize->getZExtValue() > 4) {
05769       unsigned DstAS = LHS->getType()->getPointerAddressSpace();
05770       unsigned SrcAS = RHS->getType()->getPointerAddressSpace();
05771       // TODO: Handle 5 byte compare as 4-byte + 1 byte.
05772       // TODO: Handle 8 byte compare on x86-32 as two 32-bit loads.
05773       // TODO: Check alignment of src and dest ptrs.
05774       if (!TLI->isTypeLegal(LoadVT) ||
05775           !TLI->allowsMisalignedMemoryAccesses(LoadVT, SrcAS) ||
05776           !TLI->allowsMisalignedMemoryAccesses(LoadVT, DstAS))
05777         ActuallyDoIt = false;
05778     }
05779 
05780     if (ActuallyDoIt) {
05781       SDValue LHSVal = getMemCmpLoad(LHS, LoadVT, LoadTy, *this);
05782       SDValue RHSVal = getMemCmpLoad(RHS, LoadVT, LoadTy, *this);
05783 
05784       SDValue Res = DAG.getSetCC(getCurSDLoc(), MVT::i1, LHSVal, RHSVal,
05785                                  ISD::SETNE);
05786       processIntegerCallValue(I, Res, false);
05787       return true;
05788     }
05789   }
05790 
05791 
05792   return false;
05793 }
05794 
05795 /// visitMemChrCall -- See if we can lower a memchr call into an optimized
05796 /// form.  If so, return true and lower it, otherwise return false and it
05797 /// will be lowered like a normal call.
05798 bool SelectionDAGBuilder::visitMemChrCall(const CallInst &I) {
05799   // Verify that the prototype makes sense.  void *memchr(void *, int, size_t)
05800   if (I.getNumArgOperands() != 3)
05801     return false;
05802 
05803   const Value *Src = I.getArgOperand(0);
05804   const Value *Char = I.getArgOperand(1);
05805   const Value *Length = I.getArgOperand(2);
05806   if (!Src->getType()->isPointerTy() ||
05807       !Char->getType()->isIntegerTy() ||
05808       !Length->getType()->isIntegerTy() ||
05809       !I.getType()->isPointerTy())
05810     return false;
05811 
05812   const TargetSelectionDAGInfo &TSI = DAG.getSelectionDAGInfo();
05813   std::pair<SDValue, SDValue> Res =
05814     TSI.EmitTargetCodeForMemchr(DAG, getCurSDLoc(), DAG.getRoot(),
05815                                 getValue(Src), getValue(Char), getValue(Length),
05816                                 MachinePointerInfo(Src));
05817   if (Res.first.getNode()) {
05818     setValue(&I, Res.first);
05819     PendingLoads.push_back(Res.second);
05820     return true;
05821   }
05822 
05823   return false;
05824 }
05825 
05826 /// visitStrCpyCall -- See if we can lower a strcpy or stpcpy call into an
05827 /// optimized form.  If so, return true and lower it, otherwise return false
05828 /// and it will be lowered like a normal call.
05829 bool SelectionDAGBuilder::visitStrCpyCall(const CallInst &I, bool isStpcpy) {
05830   // Verify that the prototype makes sense.  char *strcpy(char *, char *)
05831   if (I.getNumArgOperands() != 2)
05832     return false;
05833 
05834   const Value *Arg0 = I.getArgOperand(0), *Arg1 = I.getArgOperand(1);
05835   if (!Arg0->getType()->isPointerTy() ||
05836       !Arg1->getType()->isPointerTy() ||
05837       !I.getType()->isPointerTy())
05838     return false;
05839 
05840   const TargetSelectionDAGInfo &TSI = DAG.getSelectionDAGInfo();
05841   std::pair<SDValue, SDValue> Res =
05842     TSI.EmitTargetCodeForStrcpy(DAG, getCurSDLoc(), getRoot(),
05843                                 getValue(Arg0), getValue(Arg1),
05844                                 MachinePointerInfo(Arg0),
05845                                 MachinePointerInfo(Arg1), isStpcpy);
05846   if (Res.first.getNode()) {
05847     setValue(&I, Res.first);
05848     DAG.setRoot(Res.second);
05849     return true;
05850   }
05851 
05852   return false;
05853 }
05854 
05855 /// visitStrCmpCall - See if we can lower a call to strcmp in an optimized form.
05856 /// If so, return true and lower it, otherwise return false and it will be
05857 /// lowered like a normal call.
05858 bool SelectionDAGBuilder::visitStrCmpCall(const CallInst &I) {
05859   // Verify that the prototype makes sense.  int strcmp(void*,void*)
05860   if (I.getNumArgOperands() != 2)
05861     return false;
05862 
05863   const Value *Arg0 = I.getArgOperand(0), *Arg1 = I.getArgOperand(1);
05864   if (!Arg0->getType()->isPointerTy() ||
05865       !Arg1->getType()->isPointerTy() ||
05866       !I.getType()->isIntegerTy())
05867     return false;
05868 
05869   const TargetSelectionDAGInfo &TSI = DAG.getSelectionDAGInfo();
05870   std::pair<SDValue, SDValue> Res =
05871     TSI.EmitTargetCodeForStrcmp(DAG, getCurSDLoc(), DAG.getRoot(),
05872                                 getValue(Arg0), getValue(Arg1),
05873                                 MachinePointerInfo(Arg0),
05874                                 MachinePointerInfo(Arg1));
05875   if (Res.first.getNode()) {
05876     processIntegerCallValue(I, Res.first, true);
05877     PendingLoads.push_back(Res.second);
05878     return true;
05879   }
05880 
05881   return false;
05882 }
05883 
05884 /// visitStrLenCall -- See if we can lower a strlen call into an optimized
05885 /// form.  If so, return true and lower it, otherwise return false and it
05886 /// will be lowered like a normal call.
05887 bool SelectionDAGBuilder::visitStrLenCall(const CallInst &I) {
05888   // Verify that the prototype makes sense.  size_t strlen(char *)
05889   if (I.getNumArgOperands() != 1)
05890     return false;
05891 
05892   const Value *Arg0 = I.getArgOperand(0);
05893   if (!Arg0->getType()->isPointerTy() || !I.getType()->isIntegerTy())
05894     return false;
05895 
05896   const TargetSelectionDAGInfo &TSI = DAG.getSelectionDAGInfo();
05897   std::pair<SDValue, SDValue> Res =
05898     TSI.EmitTargetCodeForStrlen(DAG, getCurSDLoc(), DAG.getRoot(),
05899                                 getValue(Arg0), MachinePointerInfo(Arg0));
05900   if (Res.first.getNode()) {
05901     processIntegerCallValue(I, Res.first, false);
05902     PendingLoads.push_back(Res.second);
05903     return true;
05904   }
05905 
05906   return false;
05907 }
05908 
05909 /// visitStrNLenCall -- See if we can lower a strnlen call into an optimized
05910 /// form.  If so, return true and lower it, otherwise return false and it
05911 /// will be lowered like a normal call.
05912 bool SelectionDAGBuilder::visitStrNLenCall(const CallInst &I) {
05913   // Verify that the prototype makes sense.  size_t strnlen(char *, size_t)
05914   if (I.getNumArgOperands() != 2)
05915     return false;
05916 
05917   const Value *Arg0 = I.getArgOperand(0), *Arg1 = I.getArgOperand(1);
05918   if (!Arg0->getType()->isPointerTy() ||
05919       !Arg1->getType()->isIntegerTy() ||
05920       !I.getType()->isIntegerTy())
05921     return false;
05922 
05923   const TargetSelectionDAGInfo &TSI = DAG.getSelectionDAGInfo();
05924   std::pair<SDValue, SDValue> Res =
05925     TSI.EmitTargetCodeForStrnlen(DAG, getCurSDLoc(), DAG.getRoot(),
05926                                  getValue(Arg0), getValue(Arg1),
05927                                  MachinePointerInfo(Arg0));
05928   if (Res.first.getNode()) {
05929     processIntegerCallValue(I, Res.first, false);
05930     PendingLoads.push_back(Res.second);
05931     return true;
05932   }
05933 
05934   return false;
05935 }
05936 
05937 /// visitUnaryFloatCall - If a call instruction is a unary floating-point
05938 /// operation (as expected), translate it to an SDNode with the specified opcode
05939 /// and return true.
05940 bool SelectionDAGBuilder::visitUnaryFloatCall(const CallInst &I,
05941                                               unsigned Opcode) {
05942   // Sanity check that it really is a unary floating-point call.
05943   if (I.getNumArgOperands() != 1 ||
05944       !I.getArgOperand(0)->getType()->isFloatingPointTy() ||
05945       I.getType() != I.getArgOperand(0)->getType() ||
05946       !I.onlyReadsMemory())
05947     return false;
05948 
05949   SDValue Tmp = getValue(I.getArgOperand(0));
05950   setValue(&I, DAG.getNode(Opcode, getCurSDLoc(), Tmp.getValueType(), Tmp));
05951   return true;
05952 }
05953 
05954 void SelectionDAGBuilder::visitCall(const CallInst &I) {
05955   // Handle inline assembly differently.
05956   if (isa<InlineAsm>(I.getCalledValue())) {
05957     visitInlineAsm(&I);
05958     return;
05959   }
05960 
05961   MachineModuleInfo &MMI = DAG.getMachineFunction().getMMI();
05962   ComputeUsesVAFloatArgument(I, &MMI);
05963 
05964   const char *RenameFn = nullptr;
05965   if (Function *F = I.getCalledFunction()) {
05966     if (F->isDeclaration()) {
05967       if (const TargetIntrinsicInfo *II = TM.getIntrinsicInfo()) {
05968         if (unsigned IID = II->getIntrinsicID(F)) {
05969           RenameFn = visitIntrinsicCall(I, IID);
05970           if (!RenameFn)
05971             return;
05972         }
05973       }
05974       if (unsigned IID = F->getIntrinsicID()) {
05975         RenameFn = visitIntrinsicCall(I, IID);
05976         if (!RenameFn)
05977           return;
05978       }
05979     }
05980 
05981     // Check for well-known libc/libm calls.  If the function is internal, it
05982     // can't be a library call.
05983     LibFunc::Func Func;
05984     if (!F->hasLocalLinkage() && F->hasName() &&
05985         LibInfo->getLibFunc(F->getName(), Func) &&
05986         LibInfo->hasOptimizedCodeGen(Func)) {
05987       switch (Func) {
05988       default: break;
05989       case LibFunc::copysign:
05990       case LibFunc::copysignf:
05991       case LibFunc::copysignl:
05992         if (I.getNumArgOperands() == 2 &&   // Basic sanity checks.
05993             I.getArgOperand(0)->getType()->isFloatingPointTy() &&
05994             I.getType() == I.getArgOperand(0)->getType() &&
05995             I.getType() == I.getArgOperand(1)->getType() &&
05996             I.onlyReadsMemory()) {
05997           SDValue LHS = getValue(I.getArgOperand(0));
05998           SDValue RHS = getValue(I.getArgOperand(1));
05999           setValue(&I, DAG.getNode(ISD::FCOPYSIGN, getCurSDLoc(),
06000                                    LHS.getValueType(), LHS, RHS));
06001           return;
06002         }
06003         break;
06004       case LibFunc::fabs:
06005       case LibFunc::fabsf:
06006       case LibFunc::fabsl:
06007         if (visitUnaryFloatCall(I, ISD::FABS))
06008           return;
06009         break;
06010       case LibFunc::sin:
06011       case LibFunc::sinf:
06012       case LibFunc::sinl:
06013         if (visitUnaryFloatCall(I, ISD::FSIN))
06014           return;
06015         break;
06016       case LibFunc::cos:
06017       case LibFunc::cosf:
06018       case LibFunc::cosl:
06019         if (visitUnaryFloatCall(I, ISD::FCOS))
06020           return;
06021         break;
06022       case LibFunc::sqrt:
06023       case LibFunc::sqrtf:
06024       case LibFunc::sqrtl:
06025       case LibFunc::sqrt_finite:
06026       case LibFunc::sqrtf_finite:
06027       case LibFunc::sqrtl_finite:
06028         if (visitUnaryFloatCall(I, ISD::FSQRT))
06029           return;
06030         break;
06031       case LibFunc::floor:
06032       case LibFunc::floorf:
06033       case LibFunc::floorl:
06034         if (visitUnaryFloatCall(I, ISD::FFLOOR))
06035           return;
06036         break;
06037       case LibFunc::nearbyint:
06038       case LibFunc::nearbyintf:
06039       case LibFunc::nearbyintl:
06040         if (visitUnaryFloatCall(I, ISD::FNEARBYINT))
06041           return;
06042         break;
06043       case LibFunc::ceil:
06044       case LibFunc::ceilf:
06045       case LibFunc::ceill:
06046         if (visitUnaryFloatCall(I, ISD::FCEIL))
06047           return;
06048         break;
06049       case LibFunc::rint:
06050       case LibFunc::rintf:
06051       case LibFunc::rintl:
06052         if (visitUnaryFloatCall(I, ISD::FRINT))
06053           return;
06054         break;
06055       case LibFunc::round:
06056       case LibFunc::roundf:
06057       case LibFunc::roundl:
06058         if (visitUnaryFloatCall(I, ISD::FROUND))
06059           return;
06060         break;
06061       case LibFunc::trunc:
06062       case LibFunc::truncf:
06063       case LibFunc::truncl:
06064         if (visitUnaryFloatCall(I, ISD::FTRUNC))
06065           return;
06066         break;
06067       case LibFunc::log2:
06068       case LibFunc::log2f:
06069       case LibFunc::log2l:
06070         if (visitUnaryFloatCall(I, ISD::FLOG2))
06071           return;
06072         break;
06073       case LibFunc::exp2:
06074       case LibFunc::exp2f:
06075       case LibFunc::exp2l:
06076         if (visitUnaryFloatCall(I, ISD::FEXP2))
06077           return;
06078         break;
06079       case LibFunc::memcmp:
06080         if (visitMemCmpCall(I))
06081           return;
06082         break;
06083       case LibFunc::memchr:
06084         if (visitMemChrCall(I))
06085           return;
06086         break;
06087       case LibFunc::strcpy:
06088         if (visitStrCpyCall(I, false))
06089           return;
06090         break;
06091       case LibFunc::stpcpy:
06092         if (visitStrCpyCall(I, true))
06093           return;
06094         break;
06095       case LibFunc::strcmp:
06096         if (visitStrCmpCall(I))
06097           return;
06098         break;
06099       case LibFunc::strlen:
06100         if (visitStrLenCall(I))
06101           return;
06102         break;
06103       case LibFunc::strnlen:
06104         if (visitStrNLenCall(I))
06105           return;
06106         break;
06107       }
06108     }
06109   }
06110 
06111   SDValue Callee;
06112   if (!RenameFn)
06113     Callee = getValue(I.getCalledValue());
06114   else
06115     Callee = DAG.getExternalSymbol(
06116         RenameFn, TM.getSubtargetImpl()->getTargetLowering()->getPointerTy());
06117 
06118   // Check if we can potentially perform a tail call. More detailed checking is
06119   // be done within LowerCallTo, after more information about the call is known.
06120   LowerCallTo(&I, Callee, I.isTailCall());
06121 }
06122 
06123 namespace {
06124 
06125 /// AsmOperandInfo - This contains information for each constraint that we are
06126 /// lowering.
06127 class SDISelAsmOperandInfo : public TargetLowering::AsmOperandInfo {
06128 public:
06129   /// CallOperand - If this is the result output operand or a clobber
06130   /// this is null, otherwise it is the incoming operand to the CallInst.
06131   /// This gets modified as the asm is processed.
06132   SDValue CallOperand;
06133 
06134   /// AssignedRegs - If this is a register or register class operand, this
06135   /// contains the set of register corresponding to the operand.
06136   RegsForValue AssignedRegs;
06137 
06138   explicit SDISelAsmOperandInfo(const TargetLowering::AsmOperandInfo &info)
06139     : TargetLowering::AsmOperandInfo(info), CallOperand(nullptr,0) {
06140   }
06141 
06142   /// getCallOperandValEVT - Return the EVT of the Value* that this operand
06143   /// corresponds to.  If there is no Value* for this operand, it returns
06144   /// MVT::Other.
06145   EVT getCallOperandValEVT(LLVMContext &Context,
06146                            const TargetLowering &TLI,
06147                            const DataLayout *DL) const {
06148     if (!CallOperandVal) return MVT::Other;
06149 
06150     if (isa<BasicBlock>(CallOperandVal))
06151       return TLI.getPointerTy();
06152 
06153     llvm::Type *OpTy = CallOperandVal->getType();
06154 
06155     // FIXME: code duplicated from TargetLowering::ParseConstraints().
06156     // If this is an indirect operand, the operand is a pointer to the
06157     // accessed type.
06158     if (isIndirect) {
06159       llvm::PointerType *PtrTy = dyn_cast<PointerType>(OpTy);
06160       if (!PtrTy)
06161         report_fatal_error("Indirect operand for inline asm not a pointer!");
06162       OpTy = PtrTy->getElementType();
06163     }
06164 
06165     // Look for vector wrapped in a struct. e.g. { <16 x i8> }.
06166     if (StructType *STy = dyn_cast<StructType>(OpTy))
06167       if (STy->getNumElements() == 1)
06168         OpTy = STy->getElementType(0);
06169 
06170     // If OpTy is not a single value, it may be a struct/union that we
06171     // can tile with integers.
06172     if (!OpTy->isSingleValueType() && OpTy->isSized()) {
06173       unsigned BitSize = DL->getTypeSizeInBits(OpTy);
06174       switch (BitSize) {
06175       default: break;
06176       case 1:
06177       case 8:
06178       case 16:
06179       case 32:
06180       case 64:
06181       case 128:
06182         OpTy = IntegerType::get(Context, BitSize);
06183         break;
06184       }
06185     }
06186 
06187     return TLI.getValueType(OpTy, true);
06188   }
06189 };
06190 
06191 typedef SmallVector<SDISelAsmOperandInfo,16> SDISelAsmOperandInfoVector;
06192 
06193 } // end anonymous namespace
06194 
06195 /// GetRegistersForValue - Assign registers (virtual or physical) for the
06196 /// specified operand.  We prefer to assi