LLVM API Documentation

SelectionDAGBuilder.cpp
Go to the documentation of this file.
00001 //===-- SelectionDAGBuilder.cpp - Selection-DAG building ------------------===//
00002 //
00003 //                     The LLVM Compiler Infrastructure
00004 //
00005 // This file is distributed under the University of Illinois Open Source
00006 // License. See LICENSE.TXT for details.
00007 //
00008 //===----------------------------------------------------------------------===//
00009 //
00010 // This implements routines for translating from LLVM IR into SelectionDAG IR.
00011 //
00012 //===----------------------------------------------------------------------===//
00013 
00014 #include "SelectionDAGBuilder.h"
00015 #include "SDNodeDbgValue.h"
00016 #include "llvm/ADT/BitVector.h"
00017 #include "llvm/ADT/Optional.h"
00018 #include "llvm/ADT/SmallSet.h"
00019 #include "llvm/Analysis/AliasAnalysis.h"
00020 #include "llvm/Analysis/BranchProbabilityInfo.h"
00021 #include "llvm/Analysis/ConstantFolding.h"
00022 #include "llvm/Analysis/ValueTracking.h"
00023 #include "llvm/CodeGen/Analysis.h"
00024 #include "llvm/CodeGen/FastISel.h"
00025 #include "llvm/CodeGen/FunctionLoweringInfo.h"
00026 #include "llvm/CodeGen/GCMetadata.h"
00027 #include "llvm/CodeGen/GCStrategy.h"
00028 #include "llvm/CodeGen/MachineFrameInfo.h"
00029 #include "llvm/CodeGen/MachineFunction.h"
00030 #include "llvm/CodeGen/MachineInstrBuilder.h"
00031 #include "llvm/CodeGen/MachineJumpTableInfo.h"
00032 #include "llvm/CodeGen/MachineModuleInfo.h"
00033 #include "llvm/CodeGen/MachineRegisterInfo.h"
00034 #include "llvm/CodeGen/SelectionDAG.h"
00035 #include "llvm/CodeGen/StackMaps.h"
00036 #include "llvm/IR/CallingConv.h"
00037 #include "llvm/IR/Constants.h"
00038 #include "llvm/IR/DataLayout.h"
00039 #include "llvm/IR/DebugInfo.h"
00040 #include "llvm/IR/DerivedTypes.h"
00041 #include "llvm/IR/Function.h"
00042 #include "llvm/IR/GlobalVariable.h"
00043 #include "llvm/IR/InlineAsm.h"
00044 #include "llvm/IR/Instructions.h"
00045 #include "llvm/IR/IntrinsicInst.h"
00046 #include "llvm/IR/Intrinsics.h"
00047 #include "llvm/IR/LLVMContext.h"
00048 #include "llvm/IR/Module.h"
00049 #include "llvm/Support/CommandLine.h"
00050 #include "llvm/Support/Debug.h"
00051 #include "llvm/Support/ErrorHandling.h"
00052 #include "llvm/Support/MathExtras.h"
00053 #include "llvm/Support/raw_ostream.h"
00054 #include "llvm/Target/TargetFrameLowering.h"
00055 #include "llvm/Target/TargetInstrInfo.h"
00056 #include "llvm/Target/TargetIntrinsicInfo.h"
00057 #include "llvm/Target/TargetLibraryInfo.h"
00058 #include "llvm/Target/TargetLowering.h"
00059 #include "llvm/Target/TargetOptions.h"
00060 #include "llvm/Target/TargetSelectionDAGInfo.h"
00061 #include <algorithm>
00062 using namespace llvm;
00063 
00064 #define DEBUG_TYPE "isel"
00065 
00066 /// LimitFloatPrecision - Generate low-precision inline sequences for
00067 /// some float libcalls (6, 8 or 12 bits).
00068 static unsigned LimitFloatPrecision;
00069 
00070 static cl::opt<unsigned, true>
00071 LimitFPPrecision("limit-float-precision",
00072                  cl::desc("Generate low-precision inline sequences "
00073                           "for some float libcalls"),
00074                  cl::location(LimitFloatPrecision),
00075                  cl::init(0));
00076 
00077 // Limit the width of DAG chains. This is important in general to prevent
00078 // prevent DAG-based analysis from blowing up. For example, alias analysis and
00079 // load clustering may not complete in reasonable time. It is difficult to
00080 // recognize and avoid this situation within each individual analysis, and
00081 // future analyses are likely to have the same behavior. Limiting DAG width is
00082 // the safe approach, and will be especially important with global DAGs.
00083 //
00084 // MaxParallelChains default is arbitrarily high to avoid affecting
00085 // optimization, but could be lowered to improve compile time. Any ld-ld-st-st
00086 // sequence over this should have been converted to llvm.memcpy by the
00087 // frontend. It easy to induce this behavior with .ll code such as:
00088 // %buffer = alloca [4096 x i8]
00089 // %data = load [4096 x i8]* %argPtr
00090 // store [4096 x i8] %data, [4096 x i8]* %buffer
00091 static const unsigned MaxParallelChains = 64;
00092 
00093 static SDValue getCopyFromPartsVector(SelectionDAG &DAG, SDLoc DL,
00094                                       const SDValue *Parts, unsigned NumParts,
00095                                       MVT PartVT, EVT ValueVT, const Value *V);
00096 
00097 /// getCopyFromParts - Create a value that contains the specified legal parts
00098 /// combined into the value they represent.  If the parts combine to a type
00099 /// larger then ValueVT then AssertOp can be used to specify whether the extra
00100 /// bits are known to be zero (ISD::AssertZext) or sign extended from ValueVT
00101 /// (ISD::AssertSext).
00102 static SDValue getCopyFromParts(SelectionDAG &DAG, SDLoc DL,
00103                                 const SDValue *Parts,
00104                                 unsigned NumParts, MVT PartVT, EVT ValueVT,
00105                                 const Value *V,
00106                                 ISD::NodeType AssertOp = ISD::DELETED_NODE) {
00107   if (ValueVT.isVector())
00108     return getCopyFromPartsVector(DAG, DL, Parts, NumParts,
00109                                   PartVT, ValueVT, V);
00110 
00111   assert(NumParts > 0 && "No parts to assemble!");
00112   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
00113   SDValue Val = Parts[0];
00114 
00115   if (NumParts > 1) {
00116     // Assemble the value from multiple parts.
00117     if (ValueVT.isInteger()) {
00118       unsigned PartBits = PartVT.getSizeInBits();
00119       unsigned ValueBits = ValueVT.getSizeInBits();
00120 
00121       // Assemble the power of 2 part.
00122       unsigned RoundParts = NumParts & (NumParts - 1) ?
00123         1 << Log2_32(NumParts) : NumParts;
00124       unsigned RoundBits = PartBits * RoundParts;
00125       EVT RoundVT = RoundBits == ValueBits ?
00126         ValueVT : EVT::getIntegerVT(*DAG.getContext(), RoundBits);
00127       SDValue Lo, Hi;
00128 
00129       EVT HalfVT = EVT::getIntegerVT(*DAG.getContext(), RoundBits/2);
00130 
00131       if (RoundParts > 2) {
00132         Lo = getCopyFromParts(DAG, DL, Parts, RoundParts / 2,
00133                               PartVT, HalfVT, V);
00134         Hi = getCopyFromParts(DAG, DL, Parts + RoundParts / 2,
00135                               RoundParts / 2, PartVT, HalfVT, V);
00136       } else {
00137         Lo = DAG.getNode(ISD::BITCAST, DL, HalfVT, Parts[0]);
00138         Hi = DAG.getNode(ISD::BITCAST, DL, HalfVT, Parts[1]);
00139       }
00140 
00141       if (TLI.isBigEndian())
00142         std::swap(Lo, Hi);
00143 
00144       Val = DAG.getNode(ISD::BUILD_PAIR, DL, RoundVT, Lo, Hi);
00145 
00146       if (RoundParts < NumParts) {
00147         // Assemble the trailing non-power-of-2 part.
00148         unsigned OddParts = NumParts - RoundParts;
00149         EVT OddVT = EVT::getIntegerVT(*DAG.getContext(), OddParts * PartBits);
00150         Hi = getCopyFromParts(DAG, DL,
00151                               Parts + RoundParts, OddParts, PartVT, OddVT, V);
00152 
00153         // Combine the round and odd parts.
00154         Lo = Val;
00155         if (TLI.isBigEndian())
00156           std::swap(Lo, Hi);
00157         EVT TotalVT = EVT::getIntegerVT(*DAG.getContext(), NumParts * PartBits);
00158         Hi = DAG.getNode(ISD::ANY_EXTEND, DL, TotalVT, Hi);
00159         Hi = DAG.getNode(ISD::SHL, DL, TotalVT, Hi,
00160                          DAG.getConstant(Lo.getValueType().getSizeInBits(),
00161                                          TLI.getPointerTy()));
00162         Lo = DAG.getNode(ISD::ZERO_EXTEND, DL, TotalVT, Lo);
00163         Val = DAG.getNode(ISD::OR, DL, TotalVT, Lo, Hi);
00164       }
00165     } else if (PartVT.isFloatingPoint()) {
00166       // FP split into multiple FP parts (for ppcf128)
00167       assert(ValueVT == EVT(MVT::ppcf128) && PartVT == MVT::f64 &&
00168              "Unexpected split");
00169       SDValue Lo, Hi;
00170       Lo = DAG.getNode(ISD::BITCAST, DL, EVT(MVT::f64), Parts[0]);
00171       Hi = DAG.getNode(ISD::BITCAST, DL, EVT(MVT::f64), Parts[1]);
00172       if (TLI.hasBigEndianPartOrdering(ValueVT))
00173         std::swap(Lo, Hi);
00174       Val = DAG.getNode(ISD::BUILD_PAIR, DL, ValueVT, Lo, Hi);
00175     } else {
00176       // FP split into integer parts (soft fp)
00177       assert(ValueVT.isFloatingPoint() && PartVT.isInteger() &&
00178              !PartVT.isVector() && "Unexpected split");
00179       EVT IntVT = EVT::getIntegerVT(*DAG.getContext(), ValueVT.getSizeInBits());
00180       Val = getCopyFromParts(DAG, DL, Parts, NumParts, PartVT, IntVT, V);
00181     }
00182   }
00183 
00184   // There is now one part, held in Val.  Correct it to match ValueVT.
00185   EVT PartEVT = Val.getValueType();
00186 
00187   if (PartEVT == ValueVT)
00188     return Val;
00189 
00190   if (PartEVT.isInteger() && ValueVT.isInteger()) {
00191     if (ValueVT.bitsLT(PartEVT)) {
00192       // For a truncate, see if we have any information to
00193       // indicate whether the truncated bits will always be
00194       // zero or sign-extension.
00195       if (AssertOp != ISD::DELETED_NODE)
00196         Val = DAG.getNode(AssertOp, DL, PartEVT, Val,
00197                           DAG.getValueType(ValueVT));
00198       return DAG.getNode(ISD::TRUNCATE, DL, ValueVT, Val);
00199     }
00200     return DAG.getNode(ISD::ANY_EXTEND, DL, ValueVT, Val);
00201   }
00202 
00203   if (PartEVT.isFloatingPoint() && ValueVT.isFloatingPoint()) {
00204     // FP_ROUND's are always exact here.
00205     if (ValueVT.bitsLT(Val.getValueType()))
00206       return DAG.getNode(ISD::FP_ROUND, DL, ValueVT, Val,
00207                          DAG.getTargetConstant(1, TLI.getPointerTy()));
00208 
00209     return DAG.getNode(ISD::FP_EXTEND, DL, ValueVT, Val);
00210   }
00211 
00212   if (PartEVT.getSizeInBits() == ValueVT.getSizeInBits())
00213     return DAG.getNode(ISD::BITCAST, DL, ValueVT, Val);
00214 
00215   llvm_unreachable("Unknown mismatch!");
00216 }
00217 
00218 static void diagnosePossiblyInvalidConstraint(LLVMContext &Ctx, const Value *V,
00219                                               const Twine &ErrMsg) {
00220   const Instruction *I = dyn_cast_or_null<Instruction>(V);
00221   if (!V)
00222     return Ctx.emitError(ErrMsg);
00223 
00224   const char *AsmError = ", possible invalid constraint for vector type";
00225   if (const CallInst *CI = dyn_cast<CallInst>(I))
00226     if (isa<InlineAsm>(CI->getCalledValue()))
00227       return Ctx.emitError(I, ErrMsg + AsmError);
00228 
00229   return Ctx.emitError(I, ErrMsg);
00230 }
00231 
00232 /// getCopyFromPartsVector - Create a value that contains the specified legal
00233 /// parts combined into the value they represent.  If the parts combine to a
00234 /// type larger then ValueVT then AssertOp can be used to specify whether the
00235 /// extra bits are known to be zero (ISD::AssertZext) or sign extended from
00236 /// ValueVT (ISD::AssertSext).
00237 static SDValue getCopyFromPartsVector(SelectionDAG &DAG, SDLoc DL,
00238                                       const SDValue *Parts, unsigned NumParts,
00239                                       MVT PartVT, EVT ValueVT, const Value *V) {
00240   assert(ValueVT.isVector() && "Not a vector value");
00241   assert(NumParts > 0 && "No parts to assemble!");
00242   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
00243   SDValue Val = Parts[0];
00244 
00245   // Handle a multi-element vector.
00246   if (NumParts > 1) {
00247     EVT IntermediateVT;
00248     MVT RegisterVT;
00249     unsigned NumIntermediates;
00250     unsigned NumRegs =
00251     TLI.getVectorTypeBreakdown(*DAG.getContext(), ValueVT, IntermediateVT,
00252                                NumIntermediates, RegisterVT);
00253     assert(NumRegs == NumParts && "Part count doesn't match vector breakdown!");
00254     NumParts = NumRegs; // Silence a compiler warning.
00255     assert(RegisterVT == PartVT && "Part type doesn't match vector breakdown!");
00256     assert(RegisterVT == Parts[0].getSimpleValueType() &&
00257            "Part type doesn't match part!");
00258 
00259     // Assemble the parts into intermediate operands.
00260     SmallVector<SDValue, 8> Ops(NumIntermediates);
00261     if (NumIntermediates == NumParts) {
00262       // If the register was not expanded, truncate or copy the value,
00263       // as appropriate.
00264       for (unsigned i = 0; i != NumParts; ++i)
00265         Ops[i] = getCopyFromParts(DAG, DL, &Parts[i], 1,
00266                                   PartVT, IntermediateVT, V);
00267     } else if (NumParts > 0) {
00268       // If the intermediate type was expanded, build the intermediate
00269       // operands from the parts.
00270       assert(NumParts % NumIntermediates == 0 &&
00271              "Must expand into a divisible number of parts!");
00272       unsigned Factor = NumParts / NumIntermediates;
00273       for (unsigned i = 0; i != NumIntermediates; ++i)
00274         Ops[i] = getCopyFromParts(DAG, DL, &Parts[i * Factor], Factor,
00275                                   PartVT, IntermediateVT, V);
00276     }
00277 
00278     // Build a vector with BUILD_VECTOR or CONCAT_VECTORS from the
00279     // intermediate operands.
00280     Val = DAG.getNode(IntermediateVT.isVector() ? ISD::CONCAT_VECTORS
00281                                                 : ISD::BUILD_VECTOR,
00282                       DL, ValueVT, Ops);
00283   }
00284 
00285   // There is now one part, held in Val.  Correct it to match ValueVT.
00286   EVT PartEVT = Val.getValueType();
00287 
00288   if (PartEVT == ValueVT)
00289     return Val;
00290 
00291   if (PartEVT.isVector()) {
00292     // If the element type of the source/dest vectors are the same, but the
00293     // parts vector has more elements than the value vector, then we have a
00294     // vector widening case (e.g. <2 x float> -> <4 x float>).  Extract the
00295     // elements we want.
00296     if (PartEVT.getVectorElementType() == ValueVT.getVectorElementType()) {
00297       assert(PartEVT.getVectorNumElements() > ValueVT.getVectorNumElements() &&
00298              "Cannot narrow, it would be a lossy transformation");
00299       return DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, ValueVT, Val,
00300                          DAG.getConstant(0, TLI.getVectorIdxTy()));
00301     }
00302 
00303     // Vector/Vector bitcast.
00304     if (ValueVT.getSizeInBits() == PartEVT.getSizeInBits())
00305       return DAG.getNode(ISD::BITCAST, DL, ValueVT, Val);
00306 
00307     assert(PartEVT.getVectorNumElements() == ValueVT.getVectorNumElements() &&
00308       "Cannot handle this kind of promotion");
00309     // Promoted vector extract
00310     bool Smaller = ValueVT.bitsLE(PartEVT);
00311     return DAG.getNode((Smaller ? ISD::TRUNCATE : ISD::ANY_EXTEND),
00312                        DL, ValueVT, Val);
00313 
00314   }
00315 
00316   // Trivial bitcast if the types are the same size and the destination
00317   // vector type is legal.
00318   if (PartEVT.getSizeInBits() == ValueVT.getSizeInBits() &&
00319       TLI.isTypeLegal(ValueVT))
00320     return DAG.getNode(ISD::BITCAST, DL, ValueVT, Val);
00321 
00322   // Handle cases such as i8 -> <1 x i1>
00323   if (ValueVT.getVectorNumElements() != 1) {
00324     diagnosePossiblyInvalidConstraint(*DAG.getContext(), V,
00325                                       "non-trivial scalar-to-vector conversion");
00326     return DAG.getUNDEF(ValueVT);
00327   }
00328 
00329   if (ValueVT.getVectorNumElements() == 1 &&
00330       ValueVT.getVectorElementType() != PartEVT) {
00331     bool Smaller = ValueVT.bitsLE(PartEVT);
00332     Val = DAG.getNode((Smaller ? ISD::TRUNCATE : ISD::ANY_EXTEND),
00333                        DL, ValueVT.getScalarType(), Val);
00334   }
00335 
00336   return DAG.getNode(ISD::BUILD_VECTOR, DL, ValueVT, Val);
00337 }
00338 
00339 static void getCopyToPartsVector(SelectionDAG &DAG, SDLoc dl,
00340                                  SDValue Val, SDValue *Parts, unsigned NumParts,
00341                                  MVT PartVT, const Value *V);
00342 
00343 /// getCopyToParts - Create a series of nodes that contain the specified value
00344 /// split into legal parts.  If the parts contain more bits than Val, then, for
00345 /// integers, ExtendKind can be used to specify how to generate the extra bits.
00346 static void getCopyToParts(SelectionDAG &DAG, SDLoc DL,
00347                            SDValue Val, SDValue *Parts, unsigned NumParts,
00348                            MVT PartVT, const Value *V,
00349                            ISD::NodeType ExtendKind = ISD::ANY_EXTEND) {
00350   EVT ValueVT = Val.getValueType();
00351 
00352   // Handle the vector case separately.
00353   if (ValueVT.isVector())
00354     return getCopyToPartsVector(DAG, DL, Val, Parts, NumParts, PartVT, V);
00355 
00356   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
00357   unsigned PartBits = PartVT.getSizeInBits();
00358   unsigned OrigNumParts = NumParts;
00359   assert(TLI.isTypeLegal(PartVT) && "Copying to an illegal type!");
00360 
00361   if (NumParts == 0)
00362     return;
00363 
00364   assert(!ValueVT.isVector() && "Vector case handled elsewhere");
00365   EVT PartEVT = PartVT;
00366   if (PartEVT == ValueVT) {
00367     assert(NumParts == 1 && "No-op copy with multiple parts!");
00368     Parts[0] = Val;
00369     return;
00370   }
00371 
00372   if (NumParts * PartBits > ValueVT.getSizeInBits()) {
00373     // If the parts cover more bits than the value has, promote the value.
00374     if (PartVT.isFloatingPoint() && ValueVT.isFloatingPoint()) {
00375       assert(NumParts == 1 && "Do not know what to promote to!");
00376       Val = DAG.getNode(ISD::FP_EXTEND, DL, PartVT, Val);
00377     } else {
00378       assert((PartVT.isInteger() || PartVT == MVT::x86mmx) &&
00379              ValueVT.isInteger() &&
00380              "Unknown mismatch!");
00381       ValueVT = EVT::getIntegerVT(*DAG.getContext(), NumParts * PartBits);
00382       Val = DAG.getNode(ExtendKind, DL, ValueVT, Val);
00383       if (PartVT == MVT::x86mmx)
00384         Val = DAG.getNode(ISD::BITCAST, DL, PartVT, Val);
00385     }
00386   } else if (PartBits == ValueVT.getSizeInBits()) {
00387     // Different types of the same size.
00388     assert(NumParts == 1 && PartEVT != ValueVT);
00389     Val = DAG.getNode(ISD::BITCAST, DL, PartVT, Val);
00390   } else if (NumParts * PartBits < ValueVT.getSizeInBits()) {
00391     // If the parts cover less bits than value has, truncate the value.
00392     assert((PartVT.isInteger() || PartVT == MVT::x86mmx) &&
00393            ValueVT.isInteger() &&
00394            "Unknown mismatch!");
00395     ValueVT = EVT::getIntegerVT(*DAG.getContext(), NumParts * PartBits);
00396     Val = DAG.getNode(ISD::TRUNCATE, DL, ValueVT, Val);
00397     if (PartVT == MVT::x86mmx)
00398       Val = DAG.getNode(ISD::BITCAST, DL, PartVT, Val);
00399   }
00400 
00401   // The value may have changed - recompute ValueVT.
00402   ValueVT = Val.getValueType();
00403   assert(NumParts * PartBits == ValueVT.getSizeInBits() &&
00404          "Failed to tile the value with PartVT!");
00405 
00406   if (NumParts == 1) {
00407     if (PartEVT != ValueVT)
00408       diagnosePossiblyInvalidConstraint(*DAG.getContext(), V,
00409                                         "scalar-to-vector conversion failed");
00410 
00411     Parts[0] = Val;
00412     return;
00413   }
00414 
00415   // Expand the value into multiple parts.
00416   if (NumParts & (NumParts - 1)) {
00417     // The number of parts is not a power of 2.  Split off and copy the tail.
00418     assert(PartVT.isInteger() && ValueVT.isInteger() &&
00419            "Do not know what to expand to!");
00420     unsigned RoundParts = 1 << Log2_32(NumParts);
00421     unsigned RoundBits = RoundParts * PartBits;
00422     unsigned OddParts = NumParts - RoundParts;
00423     SDValue OddVal = DAG.getNode(ISD::SRL, DL, ValueVT, Val,
00424                                  DAG.getIntPtrConstant(RoundBits));
00425     getCopyToParts(DAG, DL, OddVal, Parts + RoundParts, OddParts, PartVT, V);
00426 
00427     if (TLI.isBigEndian())
00428       // The odd parts were reversed by getCopyToParts - unreverse them.
00429       std::reverse(Parts + RoundParts, Parts + NumParts);
00430 
00431     NumParts = RoundParts;
00432     ValueVT = EVT::getIntegerVT(*DAG.getContext(), NumParts * PartBits);
00433     Val = DAG.getNode(ISD::TRUNCATE, DL, ValueVT, Val);
00434   }
00435 
00436   // The number of parts is a power of 2.  Repeatedly bisect the value using
00437   // EXTRACT_ELEMENT.
00438   Parts[0] = DAG.getNode(ISD::BITCAST, DL,
00439                          EVT::getIntegerVT(*DAG.getContext(),
00440                                            ValueVT.getSizeInBits()),
00441                          Val);
00442 
00443   for (unsigned StepSize = NumParts; StepSize > 1; StepSize /= 2) {
00444     for (unsigned i = 0; i < NumParts; i += StepSize) {
00445       unsigned ThisBits = StepSize * PartBits / 2;
00446       EVT ThisVT = EVT::getIntegerVT(*DAG.getContext(), ThisBits);
00447       SDValue &Part0 = Parts[i];
00448       SDValue &Part1 = Parts[i+StepSize/2];
00449 
00450       Part1 = DAG.getNode(ISD::EXTRACT_ELEMENT, DL,
00451                           ThisVT, Part0, DAG.getIntPtrConstant(1));
00452       Part0 = DAG.getNode(ISD::EXTRACT_ELEMENT, DL,
00453                           ThisVT, Part0, DAG.getIntPtrConstant(0));
00454 
00455       if (ThisBits == PartBits && ThisVT != PartVT) {
00456         Part0 = DAG.getNode(ISD::BITCAST, DL, PartVT, Part0);
00457         Part1 = DAG.getNode(ISD::BITCAST, DL, PartVT, Part1);
00458       }
00459     }
00460   }
00461 
00462   if (TLI.isBigEndian())
00463     std::reverse(Parts, Parts + OrigNumParts);
00464 }
00465 
00466 
00467 /// getCopyToPartsVector - Create a series of nodes that contain the specified
00468 /// value split into legal parts.
00469 static void getCopyToPartsVector(SelectionDAG &DAG, SDLoc DL,
00470                                  SDValue Val, SDValue *Parts, unsigned NumParts,
00471                                  MVT PartVT, const Value *V) {
00472   EVT ValueVT = Val.getValueType();
00473   assert(ValueVT.isVector() && "Not a vector");
00474   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
00475 
00476   if (NumParts == 1) {
00477     EVT PartEVT = PartVT;
00478     if (PartEVT == ValueVT) {
00479       // Nothing to do.
00480     } else if (PartVT.getSizeInBits() == ValueVT.getSizeInBits()) {
00481       // Bitconvert vector->vector case.
00482       Val = DAG.getNode(ISD::BITCAST, DL, PartVT, Val);
00483     } else if (PartVT.isVector() &&
00484                PartEVT.getVectorElementType() == ValueVT.getVectorElementType() &&
00485                PartEVT.getVectorNumElements() > ValueVT.getVectorNumElements()) {
00486       EVT ElementVT = PartVT.getVectorElementType();
00487       // Vector widening case, e.g. <2 x float> -> <4 x float>.  Shuffle in
00488       // undef elements.
00489       SmallVector<SDValue, 16> Ops;
00490       for (unsigned i = 0, e = ValueVT.getVectorNumElements(); i != e; ++i)
00491         Ops.push_back(DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL,
00492                                   ElementVT, Val, DAG.getConstant(i,
00493                                                   TLI.getVectorIdxTy())));
00494 
00495       for (unsigned i = ValueVT.getVectorNumElements(),
00496            e = PartVT.getVectorNumElements(); i != e; ++i)
00497         Ops.push_back(DAG.getUNDEF(ElementVT));
00498 
00499       Val = DAG.getNode(ISD::BUILD_VECTOR, DL, PartVT, Ops);
00500 
00501       // FIXME: Use CONCAT for 2x -> 4x.
00502 
00503       //SDValue UndefElts = DAG.getUNDEF(VectorTy);
00504       //Val = DAG.getNode(ISD::CONCAT_VECTORS, DL, PartVT, Val, UndefElts);
00505     } else if (PartVT.isVector() &&
00506                PartEVT.getVectorElementType().bitsGE(
00507                  ValueVT.getVectorElementType()) &&
00508                PartEVT.getVectorNumElements() == ValueVT.getVectorNumElements()) {
00509 
00510       // Promoted vector extract
00511       bool Smaller = PartEVT.bitsLE(ValueVT);
00512       Val = DAG.getNode((Smaller ? ISD::TRUNCATE : ISD::ANY_EXTEND),
00513                         DL, PartVT, Val);
00514     } else{
00515       // Vector -> scalar conversion.
00516       assert(ValueVT.getVectorNumElements() == 1 &&
00517              "Only trivial vector-to-scalar conversions should get here!");
00518       Val = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL,
00519                         PartVT, Val, DAG.getConstant(0, TLI.getVectorIdxTy()));
00520 
00521       bool Smaller = ValueVT.bitsLE(PartVT);
00522       Val = DAG.getNode((Smaller ? ISD::TRUNCATE : ISD::ANY_EXTEND),
00523                          DL, PartVT, Val);
00524     }
00525 
00526     Parts[0] = Val;
00527     return;
00528   }
00529 
00530   // Handle a multi-element vector.
00531   EVT IntermediateVT;
00532   MVT RegisterVT;
00533   unsigned NumIntermediates;
00534   unsigned NumRegs = TLI.getVectorTypeBreakdown(*DAG.getContext(), ValueVT,
00535                                                 IntermediateVT,
00536                                                 NumIntermediates, RegisterVT);
00537   unsigned NumElements = ValueVT.getVectorNumElements();
00538 
00539   assert(NumRegs == NumParts && "Part count doesn't match vector breakdown!");
00540   NumParts = NumRegs; // Silence a compiler warning.
00541   assert(RegisterVT == PartVT && "Part type doesn't match vector breakdown!");
00542 
00543   // Split the vector into intermediate operands.
00544   SmallVector<SDValue, 8> Ops(NumIntermediates);
00545   for (unsigned i = 0; i != NumIntermediates; ++i) {
00546     if (IntermediateVT.isVector())
00547       Ops[i] = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL,
00548                            IntermediateVT, Val,
00549                    DAG.getConstant(i * (NumElements / NumIntermediates),
00550                                    TLI.getVectorIdxTy()));
00551     else
00552       Ops[i] = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL,
00553                            IntermediateVT, Val,
00554                            DAG.getConstant(i, TLI.getVectorIdxTy()));
00555   }
00556 
00557   // Split the intermediate operands into legal parts.
00558   if (NumParts == NumIntermediates) {
00559     // If the register was not expanded, promote or copy the value,
00560     // as appropriate.
00561     for (unsigned i = 0; i != NumParts; ++i)
00562       getCopyToParts(DAG, DL, Ops[i], &Parts[i], 1, PartVT, V);
00563   } else if (NumParts > 0) {
00564     // If the intermediate type was expanded, split each the value into
00565     // legal parts.
00566     assert(NumParts % NumIntermediates == 0 &&
00567            "Must expand into a divisible number of parts!");
00568     unsigned Factor = NumParts / NumIntermediates;
00569     for (unsigned i = 0; i != NumIntermediates; ++i)
00570       getCopyToParts(DAG, DL, Ops[i], &Parts[i*Factor], Factor, PartVT, V);
00571   }
00572 }
00573 
00574 namespace {
00575   /// RegsForValue - This struct represents the registers (physical or virtual)
00576   /// that a particular set of values is assigned, and the type information
00577   /// about the value. The most common situation is to represent one value at a
00578   /// time, but struct or array values are handled element-wise as multiple
00579   /// values.  The splitting of aggregates is performed recursively, so that we
00580   /// never have aggregate-typed registers. The values at this point do not
00581   /// necessarily have legal types, so each value may require one or more
00582   /// registers of some legal type.
00583   ///
00584   struct RegsForValue {
00585     /// ValueVTs - The value types of the values, which may not be legal, and
00586     /// may need be promoted or synthesized from one or more registers.
00587     ///
00588     SmallVector<EVT, 4> ValueVTs;
00589 
00590     /// RegVTs - The value types of the registers. This is the same size as
00591     /// ValueVTs and it records, for each value, what the type of the assigned
00592     /// register or registers are. (Individual values are never synthesized
00593     /// from more than one type of register.)
00594     ///
00595     /// With virtual registers, the contents of RegVTs is redundant with TLI's
00596     /// getRegisterType member function, however when with physical registers
00597     /// it is necessary to have a separate record of the types.
00598     ///
00599     SmallVector<MVT, 4> RegVTs;
00600 
00601     /// Regs - This list holds the registers assigned to the values.
00602     /// Each legal or promoted value requires one register, and each
00603     /// expanded value requires multiple registers.
00604     ///
00605     SmallVector<unsigned, 4> Regs;
00606 
00607     RegsForValue() {}
00608 
00609     RegsForValue(const SmallVector<unsigned, 4> &regs,
00610                  MVT regvt, EVT valuevt)
00611       : ValueVTs(1, valuevt), RegVTs(1, regvt), Regs(regs) {}
00612 
00613     RegsForValue(LLVMContext &Context, const TargetLowering &tli,
00614                  unsigned Reg, Type *Ty) {
00615       ComputeValueVTs(tli, Ty, ValueVTs);
00616 
00617       for (unsigned Value = 0, e = ValueVTs.size(); Value != e; ++Value) {
00618         EVT ValueVT = ValueVTs[Value];
00619         unsigned NumRegs = tli.getNumRegisters(Context, ValueVT);
00620         MVT RegisterVT = tli.getRegisterType(Context, ValueVT);
00621         for (unsigned i = 0; i != NumRegs; ++i)
00622           Regs.push_back(Reg + i);
00623         RegVTs.push_back(RegisterVT);
00624         Reg += NumRegs;
00625       }
00626     }
00627 
00628     /// append - Add the specified values to this one.
00629     void append(const RegsForValue &RHS) {
00630       ValueVTs.append(RHS.ValueVTs.begin(), RHS.ValueVTs.end());
00631       RegVTs.append(RHS.RegVTs.begin(), RHS.RegVTs.end());
00632       Regs.append(RHS.Regs.begin(), RHS.Regs.end());
00633     }
00634 
00635     /// getCopyFromRegs - Emit a series of CopyFromReg nodes that copies from
00636     /// this value and returns the result as a ValueVTs value.  This uses
00637     /// Chain/Flag as the input and updates them for the output Chain/Flag.
00638     /// If the Flag pointer is NULL, no flag is used.
00639     SDValue getCopyFromRegs(SelectionDAG &DAG, FunctionLoweringInfo &FuncInfo,
00640                             SDLoc dl,
00641                             SDValue &Chain, SDValue *Flag,
00642                             const Value *V = nullptr) const;
00643 
00644     /// getCopyToRegs - Emit a series of CopyToReg nodes that copies the
00645     /// specified value into the registers specified by this object.  This uses
00646     /// Chain/Flag as the input and updates them for the output Chain/Flag.
00647     /// If the Flag pointer is NULL, no flag is used.
00648     void getCopyToRegs(SDValue Val, SelectionDAG &DAG, SDLoc dl,
00649                        SDValue &Chain, SDValue *Flag, const Value *V) const;
00650 
00651     /// AddInlineAsmOperands - Add this value to the specified inlineasm node
00652     /// operand list.  This adds the code marker, matching input operand index
00653     /// (if applicable), and includes the number of values added into it.
00654     void AddInlineAsmOperands(unsigned Kind,
00655                               bool HasMatching, unsigned MatchingIdx,
00656                               SelectionDAG &DAG,
00657                               std::vector<SDValue> &Ops) const;
00658   };
00659 }
00660 
00661 /// getCopyFromRegs - Emit a series of CopyFromReg nodes that copies from
00662 /// this value and returns the result as a ValueVT value.  This uses
00663 /// Chain/Flag as the input and updates them for the output Chain/Flag.
00664 /// If the Flag pointer is NULL, no flag is used.
00665 SDValue RegsForValue::getCopyFromRegs(SelectionDAG &DAG,
00666                                       FunctionLoweringInfo &FuncInfo,
00667                                       SDLoc dl,
00668                                       SDValue &Chain, SDValue *Flag,
00669                                       const Value *V) const {
00670   // A Value with type {} or [0 x %t] needs no registers.
00671   if (ValueVTs.empty())
00672     return SDValue();
00673 
00674   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
00675 
00676   // Assemble the legal parts into the final values.
00677   SmallVector<SDValue, 4> Values(ValueVTs.size());
00678   SmallVector<SDValue, 8> Parts;
00679   for (unsigned Value = 0, Part = 0, e = ValueVTs.size(); Value != e; ++Value) {
00680     // Copy the legal parts from the registers.
00681     EVT ValueVT = ValueVTs[Value];
00682     unsigned NumRegs = TLI.getNumRegisters(*DAG.getContext(), ValueVT);
00683     MVT RegisterVT = RegVTs[Value];
00684 
00685     Parts.resize(NumRegs);
00686     for (unsigned i = 0; i != NumRegs; ++i) {
00687       SDValue P;
00688       if (!Flag) {
00689         P = DAG.getCopyFromReg(Chain, dl, Regs[Part+i], RegisterVT);
00690       } else {
00691         P = DAG.getCopyFromReg(Chain, dl, Regs[Part+i], RegisterVT, *Flag);
00692         *Flag = P.getValue(2);
00693       }
00694 
00695       Chain = P.getValue(1);
00696       Parts[i] = P;
00697 
00698       // If the source register was virtual and if we know something about it,
00699       // add an assert node.
00700       if (!TargetRegisterInfo::isVirtualRegister(Regs[Part+i]) ||
00701           !RegisterVT.isInteger() || RegisterVT.isVector())
00702         continue;
00703 
00704       const FunctionLoweringInfo::LiveOutInfo *LOI =
00705         FuncInfo.GetLiveOutRegInfo(Regs[Part+i]);
00706       if (!LOI)
00707         continue;
00708 
00709       unsigned RegSize = RegisterVT.getSizeInBits();
00710       unsigned NumSignBits = LOI->NumSignBits;
00711       unsigned NumZeroBits = LOI->KnownZero.countLeadingOnes();
00712 
00713       if (NumZeroBits == RegSize) {
00714         // The current value is a zero.
00715         // Explicitly express that as it would be easier for
00716         // optimizations to kick in.
00717         Parts[i] = DAG.getConstant(0, RegisterVT);
00718         continue;
00719       }
00720 
00721       // FIXME: We capture more information than the dag can represent.  For
00722       // now, just use the tightest assertzext/assertsext possible.
00723       bool isSExt = true;
00724       EVT FromVT(MVT::Other);
00725       if (NumSignBits == RegSize)
00726         isSExt = true, FromVT = MVT::i1;   // ASSERT SEXT 1
00727       else if (NumZeroBits >= RegSize-1)
00728         isSExt = false, FromVT = MVT::i1;  // ASSERT ZEXT 1
00729       else if (NumSignBits > RegSize-8)
00730         isSExt = true, FromVT = MVT::i8;   // ASSERT SEXT 8
00731       else if (NumZeroBits >= RegSize-8)
00732         isSExt = false, FromVT = MVT::i8;  // ASSERT ZEXT 8
00733       else if (NumSignBits > RegSize-16)
00734         isSExt = true, FromVT = MVT::i16;  // ASSERT SEXT 16
00735       else if (NumZeroBits >= RegSize-16)
00736         isSExt = false, FromVT = MVT::i16; // ASSERT ZEXT 16
00737       else if (NumSignBits > RegSize-32)
00738         isSExt = true, FromVT = MVT::i32;  // ASSERT SEXT 32
00739       else if (NumZeroBits >= RegSize-32)
00740         isSExt = false, FromVT = MVT::i32; // ASSERT ZEXT 32
00741       else
00742         continue;
00743 
00744       // Add an assertion node.
00745       assert(FromVT != MVT::Other);
00746       Parts[i] = DAG.getNode(isSExt ? ISD::AssertSext : ISD::AssertZext, dl,
00747                              RegisterVT, P, DAG.getValueType(FromVT));
00748     }
00749 
00750     Values[Value] = getCopyFromParts(DAG, dl, Parts.begin(),
00751                                      NumRegs, RegisterVT, ValueVT, V);
00752     Part += NumRegs;
00753     Parts.clear();
00754   }
00755 
00756   return DAG.getNode(ISD::MERGE_VALUES, dl, DAG.getVTList(ValueVTs), Values);
00757 }
00758 
00759 /// getCopyToRegs - Emit a series of CopyToReg nodes that copies the
00760 /// specified value into the registers specified by this object.  This uses
00761 /// Chain/Flag as the input and updates them for the output Chain/Flag.
00762 /// If the Flag pointer is NULL, no flag is used.
00763 void RegsForValue::getCopyToRegs(SDValue Val, SelectionDAG &DAG, SDLoc dl,
00764                                  SDValue &Chain, SDValue *Flag,
00765                                  const Value *V) const {
00766   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
00767 
00768   // Get the list of the values's legal parts.
00769   unsigned NumRegs = Regs.size();
00770   SmallVector<SDValue, 8> Parts(NumRegs);
00771   for (unsigned Value = 0, Part = 0, e = ValueVTs.size(); Value != e; ++Value) {
00772     EVT ValueVT = ValueVTs[Value];
00773     unsigned NumParts = TLI.getNumRegisters(*DAG.getContext(), ValueVT);
00774     MVT RegisterVT = RegVTs[Value];
00775     ISD::NodeType ExtendKind =
00776       TLI.isZExtFree(Val, RegisterVT)? ISD::ZERO_EXTEND: ISD::ANY_EXTEND;
00777 
00778     getCopyToParts(DAG, dl, Val.getValue(Val.getResNo() + Value),
00779                    &Parts[Part], NumParts, RegisterVT, V, ExtendKind);
00780     Part += NumParts;
00781   }
00782 
00783   // Copy the parts into the registers.
00784   SmallVector<SDValue, 8> Chains(NumRegs);
00785   for (unsigned i = 0; i != NumRegs; ++i) {
00786     SDValue Part;
00787     if (!Flag) {
00788       Part = DAG.getCopyToReg(Chain, dl, Regs[i], Parts[i]);
00789     } else {
00790       Part = DAG.getCopyToReg(Chain, dl, Regs[i], Parts[i], *Flag);
00791       *Flag = Part.getValue(1);
00792     }
00793 
00794     Chains[i] = Part.getValue(0);
00795   }
00796 
00797   if (NumRegs == 1 || Flag)
00798     // If NumRegs > 1 && Flag is used then the use of the last CopyToReg is
00799     // flagged to it. That is the CopyToReg nodes and the user are considered
00800     // a single scheduling unit. If we create a TokenFactor and return it as
00801     // chain, then the TokenFactor is both a predecessor (operand) of the
00802     // user as well as a successor (the TF operands are flagged to the user).
00803     // c1, f1 = CopyToReg
00804     // c2, f2 = CopyToReg
00805     // c3     = TokenFactor c1, c2
00806     // ...
00807     //        = op c3, ..., f2
00808     Chain = Chains[NumRegs-1];
00809   else
00810     Chain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Chains);
00811 }
00812 
00813 /// AddInlineAsmOperands - Add this value to the specified inlineasm node
00814 /// operand list.  This adds the code marker and includes the number of
00815 /// values added into it.
00816 void RegsForValue::AddInlineAsmOperands(unsigned Code, bool HasMatching,
00817                                         unsigned MatchingIdx,
00818                                         SelectionDAG &DAG,
00819                                         std::vector<SDValue> &Ops) const {
00820   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
00821 
00822   unsigned Flag = InlineAsm::getFlagWord(Code, Regs.size());
00823   if (HasMatching)
00824     Flag = InlineAsm::getFlagWordForMatchingOp(Flag, MatchingIdx);
00825   else if (!Regs.empty() &&
00826            TargetRegisterInfo::isVirtualRegister(Regs.front())) {
00827     // Put the register class of the virtual registers in the flag word.  That
00828     // way, later passes can recompute register class constraints for inline
00829     // assembly as well as normal instructions.
00830     // Don't do this for tied operands that can use the regclass information
00831     // from the def.
00832     const MachineRegisterInfo &MRI = DAG.getMachineFunction().getRegInfo();
00833     const TargetRegisterClass *RC = MRI.getRegClass(Regs.front());
00834     Flag = InlineAsm::getFlagWordForRegClass(Flag, RC->getID());
00835   }
00836 
00837   SDValue Res = DAG.getTargetConstant(Flag, MVT::i32);
00838   Ops.push_back(Res);
00839 
00840   unsigned SP = TLI.getStackPointerRegisterToSaveRestore();
00841   for (unsigned Value = 0, Reg = 0, e = ValueVTs.size(); Value != e; ++Value) {
00842     unsigned NumRegs = TLI.getNumRegisters(*DAG.getContext(), ValueVTs[Value]);
00843     MVT RegisterVT = RegVTs[Value];
00844     for (unsigned i = 0; i != NumRegs; ++i) {
00845       assert(Reg < Regs.size() && "Mismatch in # registers expected");
00846       unsigned TheReg = Regs[Reg++];
00847       Ops.push_back(DAG.getRegister(TheReg, RegisterVT));
00848 
00849       if (TheReg == SP && Code == InlineAsm::Kind_Clobber) {
00850         // If we clobbered the stack pointer, MFI should know about it.
00851         assert(DAG.getMachineFunction().getFrameInfo()->
00852             hasInlineAsmWithSPAdjust());
00853       }
00854     }
00855   }
00856 }
00857 
00858 void SelectionDAGBuilder::init(GCFunctionInfo *gfi, AliasAnalysis &aa,
00859                                const TargetLibraryInfo *li) {
00860   AA = &aa;
00861   GFI = gfi;
00862   LibInfo = li;
00863   DL = DAG.getTarget().getDataLayout();
00864   Context = DAG.getContext();
00865   LPadToCallSiteMap.clear();
00866 }
00867 
00868 /// clear - Clear out the current SelectionDAG and the associated
00869 /// state and prepare this SelectionDAGBuilder object to be used
00870 /// for a new block. This doesn't clear out information about
00871 /// additional blocks that are needed to complete switch lowering
00872 /// or PHI node updating; that information is cleared out as it is
00873 /// consumed.
00874 void SelectionDAGBuilder::clear() {
00875   NodeMap.clear();
00876   UnusedArgNodeMap.clear();
00877   PendingLoads.clear();
00878   PendingExports.clear();
00879   CurInst = nullptr;
00880   HasTailCall = false;
00881   SDNodeOrder = LowestSDNodeOrder;
00882 }
00883 
00884 /// clearDanglingDebugInfo - Clear the dangling debug information
00885 /// map. This function is separated from the clear so that debug
00886 /// information that is dangling in a basic block can be properly
00887 /// resolved in a different basic block. This allows the
00888 /// SelectionDAG to resolve dangling debug information attached
00889 /// to PHI nodes.
00890 void SelectionDAGBuilder::clearDanglingDebugInfo() {
00891   DanglingDebugInfoMap.clear();
00892 }
00893 
00894 /// getRoot - Return the current virtual root of the Selection DAG,
00895 /// flushing any PendingLoad items. This must be done before emitting
00896 /// a store or any other node that may need to be ordered after any
00897 /// prior load instructions.
00898 ///
00899 SDValue SelectionDAGBuilder::getRoot() {
00900   if (PendingLoads.empty())
00901     return DAG.getRoot();
00902 
00903   if (PendingLoads.size() == 1) {
00904     SDValue Root = PendingLoads[0];
00905     DAG.setRoot(Root);
00906     PendingLoads.clear();
00907     return Root;
00908   }
00909 
00910   // Otherwise, we have to make a token factor node.
00911   SDValue Root = DAG.getNode(ISD::TokenFactor, getCurSDLoc(), MVT::Other,
00912                              PendingLoads);
00913   PendingLoads.clear();
00914   DAG.setRoot(Root);
00915   return Root;
00916 }
00917 
00918 /// getControlRoot - Similar to getRoot, but instead of flushing all the
00919 /// PendingLoad items, flush all the PendingExports items. It is necessary
00920 /// to do this before emitting a terminator instruction.
00921 ///
00922 SDValue SelectionDAGBuilder::getControlRoot() {
00923   SDValue Root = DAG.getRoot();
00924 
00925   if (PendingExports.empty())
00926     return Root;
00927 
00928   // Turn all of the CopyToReg chains into one factored node.
00929   if (Root.getOpcode() != ISD::EntryToken) {
00930     unsigned i = 0, e = PendingExports.size();
00931     for (; i != e; ++i) {
00932       assert(PendingExports[i].getNode()->getNumOperands() > 1);
00933       if (PendingExports[i].getNode()->getOperand(0) == Root)
00934         break;  // Don't add the root if we already indirectly depend on it.
00935     }
00936 
00937     if (i == e)
00938       PendingExports.push_back(Root);
00939   }
00940 
00941   Root = DAG.getNode(ISD::TokenFactor, getCurSDLoc(), MVT::Other,
00942                      PendingExports);
00943   PendingExports.clear();
00944   DAG.setRoot(Root);
00945   return Root;
00946 }
00947 
00948 void SelectionDAGBuilder::visit(const Instruction &I) {
00949   // Set up outgoing PHI node register values before emitting the terminator.
00950   if (isa<TerminatorInst>(&I))
00951     HandlePHINodesInSuccessorBlocks(I.getParent());
00952 
00953   ++SDNodeOrder;
00954 
00955   CurInst = &I;
00956 
00957   visit(I.getOpcode(), I);
00958 
00959   if (!isa<TerminatorInst>(&I) && !HasTailCall)
00960     CopyToExportRegsIfNeeded(&I);
00961 
00962   CurInst = nullptr;
00963 }
00964 
00965 void SelectionDAGBuilder::visitPHI(const PHINode &) {
00966   llvm_unreachable("SelectionDAGBuilder shouldn't visit PHI nodes!");
00967 }
00968 
00969 void SelectionDAGBuilder::visit(unsigned Opcode, const User &I) {
00970   // Note: this doesn't use InstVisitor, because it has to work with
00971   // ConstantExpr's in addition to instructions.
00972   switch (Opcode) {
00973   default: llvm_unreachable("Unknown instruction type encountered!");
00974     // Build the switch statement using the Instruction.def file.
00975 #define HANDLE_INST(NUM, OPCODE, CLASS) \
00976     case Instruction::OPCODE: visit##OPCODE((const CLASS&)I); break;
00977 #include "llvm/IR/Instruction.def"
00978   }
00979 }
00980 
00981 // resolveDanglingDebugInfo - if we saw an earlier dbg_value referring to V,
00982 // generate the debug data structures now that we've seen its definition.
00983 void SelectionDAGBuilder::resolveDanglingDebugInfo(const Value *V,
00984                                                    SDValue Val) {
00985   DanglingDebugInfo &DDI = DanglingDebugInfoMap[V];
00986   if (DDI.getDI()) {
00987     const DbgValueInst *DI = DDI.getDI();
00988     DebugLoc dl = DDI.getdl();
00989     unsigned DbgSDNodeOrder = DDI.getSDNodeOrder();
00990     MDNode *Variable = DI->getVariable();
00991     uint64_t Offset = DI->getOffset();
00992     // A dbg.value for an alloca is always indirect.
00993     bool IsIndirect = isa<AllocaInst>(V) || Offset != 0;
00994     SDDbgValue *SDV;
00995     if (Val.getNode()) {
00996       if (!EmitFuncArgumentDbgValue(V, Variable, Offset, IsIndirect, Val)) {
00997         SDV = DAG.getDbgValue(Variable, Val.getNode(),
00998                               Val.getResNo(), IsIndirect,
00999             Offset, dl, DbgSDNodeOrder);
01000         DAG.AddDbgValue(SDV, Val.getNode(), false);
01001       }
01002     } else
01003       DEBUG(dbgs() << "Dropping debug info for " << *DI << "\n");
01004     DanglingDebugInfoMap[V] = DanglingDebugInfo();
01005   }
01006 }
01007 
01008 /// getValue - Return an SDValue for the given Value.
01009 SDValue SelectionDAGBuilder::getValue(const Value *V) {
01010   // If we already have an SDValue for this value, use it. It's important
01011   // to do this first, so that we don't create a CopyFromReg if we already
01012   // have a regular SDValue.
01013   SDValue &N = NodeMap[V];
01014   if (N.getNode()) return N;
01015 
01016   // If there's a virtual register allocated and initialized for this
01017   // value, use it.
01018   DenseMap<const Value *, unsigned>::iterator It = FuncInfo.ValueMap.find(V);
01019   if (It != FuncInfo.ValueMap.end()) {
01020     unsigned InReg = It->second;
01021     RegsForValue RFV(*DAG.getContext(), *TM.getTargetLowering(),
01022                      InReg, V->getType());
01023     SDValue Chain = DAG.getEntryNode();
01024     N = RFV.getCopyFromRegs(DAG, FuncInfo, getCurSDLoc(), Chain, nullptr, V);
01025     resolveDanglingDebugInfo(V, N);
01026     return N;
01027   }
01028 
01029   // Otherwise create a new SDValue and remember it.
01030   SDValue Val = getValueImpl(V);
01031   NodeMap[V] = Val;
01032   resolveDanglingDebugInfo(V, Val);
01033   return Val;
01034 }
01035 
01036 /// getNonRegisterValue - Return an SDValue for the given Value, but
01037 /// don't look in FuncInfo.ValueMap for a virtual register.
01038 SDValue SelectionDAGBuilder::getNonRegisterValue(const Value *V) {
01039   // If we already have an SDValue for this value, use it.
01040   SDValue &N = NodeMap[V];
01041   if (N.getNode()) return N;
01042 
01043   // Otherwise create a new SDValue and remember it.
01044   SDValue Val = getValueImpl(V);
01045   NodeMap[V] = Val;
01046   resolveDanglingDebugInfo(V, Val);
01047   return Val;
01048 }
01049 
01050 /// getValueImpl - Helper function for getValue and getNonRegisterValue.
01051 /// Create an SDValue for the given value.
01052 SDValue SelectionDAGBuilder::getValueImpl(const Value *V) {
01053   const TargetLowering *TLI = TM.getTargetLowering();
01054 
01055   if (const Constant *C = dyn_cast<Constant>(V)) {
01056     EVT VT = TLI->getValueType(V->getType(), true);
01057 
01058     if (const ConstantInt *CI = dyn_cast<ConstantInt>(C))
01059       return DAG.getConstant(*CI, VT);
01060 
01061     if (const GlobalValue *GV = dyn_cast<GlobalValue>(C))
01062       return DAG.getGlobalAddress(GV, getCurSDLoc(), VT);
01063 
01064     if (isa<ConstantPointerNull>(C)) {
01065       unsigned AS = V->getType()->getPointerAddressSpace();
01066       return DAG.getConstant(0, TLI->getPointerTy(AS));
01067     }
01068 
01069     if (const ConstantFP *CFP = dyn_cast<ConstantFP>(C))
01070       return DAG.getConstantFP(*CFP, VT);
01071 
01072     if (isa<UndefValue>(C) && !V->getType()->isAggregateType())
01073       return DAG.getUNDEF(VT);
01074 
01075     if (const ConstantExpr *CE = dyn_cast<ConstantExpr>(C)) {
01076       visit(CE->getOpcode(), *CE);
01077       SDValue N1 = NodeMap[V];
01078       assert(N1.getNode() && "visit didn't populate the NodeMap!");
01079       return N1;
01080     }
01081 
01082     if (isa<ConstantStruct>(C) || isa<ConstantArray>(C)) {
01083       SmallVector<SDValue, 4> Constants;
01084       for (User::const_op_iterator OI = C->op_begin(), OE = C->op_end();
01085            OI != OE; ++OI) {
01086         SDNode *Val = getValue(*OI).getNode();
01087         // If the operand is an empty aggregate, there are no values.
01088         if (!Val) continue;
01089         // Add each leaf value from the operand to the Constants list
01090         // to form a flattened list of all the values.
01091         for (unsigned i = 0, e = Val->getNumValues(); i != e; ++i)
01092           Constants.push_back(SDValue(Val, i));
01093       }
01094 
01095       return DAG.getMergeValues(Constants, getCurSDLoc());
01096     }
01097 
01098     if (const ConstantDataSequential *CDS =
01099           dyn_cast<ConstantDataSequential>(C)) {
01100       SmallVector<SDValue, 4> Ops;
01101       for (unsigned i = 0, e = CDS->getNumElements(); i != e; ++i) {
01102         SDNode *Val = getValue(CDS->getElementAsConstant(i)).getNode();
01103         // Add each leaf value from the operand to the Constants list
01104         // to form a flattened list of all the values.
01105         for (unsigned i = 0, e = Val->getNumValues(); i != e; ++i)
01106           Ops.push_back(SDValue(Val, i));
01107       }
01108 
01109       if (isa<ArrayType>(CDS->getType()))
01110         return DAG.getMergeValues(Ops, getCurSDLoc());
01111       return NodeMap[V] = DAG.getNode(ISD::BUILD_VECTOR, getCurSDLoc(),
01112                                       VT, Ops);
01113     }
01114 
01115     if (C->getType()->isStructTy() || C->getType()->isArrayTy()) {
01116       assert((isa<ConstantAggregateZero>(C) || isa<UndefValue>(C)) &&
01117              "Unknown struct or array constant!");
01118 
01119       SmallVector<EVT, 4> ValueVTs;
01120       ComputeValueVTs(*TLI, C->getType(), ValueVTs);
01121       unsigned NumElts = ValueVTs.size();
01122       if (NumElts == 0)
01123         return SDValue(); // empty struct
01124       SmallVector<SDValue, 4> Constants(NumElts);
01125       for (unsigned i = 0; i != NumElts; ++i) {
01126         EVT EltVT = ValueVTs[i];
01127         if (isa<UndefValue>(C))
01128           Constants[i] = DAG.getUNDEF(EltVT);
01129         else if (EltVT.isFloatingPoint())
01130           Constants[i] = DAG.getConstantFP(0, EltVT);
01131         else
01132           Constants[i] = DAG.getConstant(0, EltVT);
01133       }
01134 
01135       return DAG.getMergeValues(Constants, getCurSDLoc());
01136     }
01137 
01138     if (const BlockAddress *BA = dyn_cast<BlockAddress>(C))
01139       return DAG.getBlockAddress(BA, VT);
01140 
01141     VectorType *VecTy = cast<VectorType>(V->getType());
01142     unsigned NumElements = VecTy->getNumElements();
01143 
01144     // Now that we know the number and type of the elements, get that number of
01145     // elements into the Ops array based on what kind of constant it is.
01146     SmallVector<SDValue, 16> Ops;
01147     if (const ConstantVector *CV = dyn_cast<ConstantVector>(C)) {
01148       for (unsigned i = 0; i != NumElements; ++i)
01149         Ops.push_back(getValue(CV->getOperand(i)));
01150     } else {
01151       assert(isa<ConstantAggregateZero>(C) && "Unknown vector constant!");
01152       EVT EltVT = TLI->getValueType(VecTy->getElementType());
01153 
01154       SDValue Op;
01155       if (EltVT.isFloatingPoint())
01156         Op = DAG.getConstantFP(0, EltVT);
01157       else
01158         Op = DAG.getConstant(0, EltVT);
01159       Ops.assign(NumElements, Op);
01160     }
01161 
01162     // Create a BUILD_VECTOR node.
01163     return NodeMap[V] = DAG.getNode(ISD::BUILD_VECTOR, getCurSDLoc(), VT, Ops);
01164   }
01165 
01166   // If this is a static alloca, generate it as the frameindex instead of
01167   // computation.
01168   if (const AllocaInst *AI = dyn_cast<AllocaInst>(V)) {
01169     DenseMap<const AllocaInst*, int>::iterator SI =
01170       FuncInfo.StaticAllocaMap.find(AI);
01171     if (SI != FuncInfo.StaticAllocaMap.end())
01172       return DAG.getFrameIndex(SI->second, TLI->getPointerTy());
01173   }
01174 
01175   // If this is an instruction which fast-isel has deferred, select it now.
01176   if (const Instruction *Inst = dyn_cast<Instruction>(V)) {
01177     unsigned InReg = FuncInfo.InitializeRegForValue(Inst);
01178     RegsForValue RFV(*DAG.getContext(), *TLI, InReg, Inst->getType());
01179     SDValue Chain = DAG.getEntryNode();
01180     return RFV.getCopyFromRegs(DAG, FuncInfo, getCurSDLoc(), Chain, nullptr, V);
01181   }
01182 
01183   llvm_unreachable("Can't get register for value!");
01184 }
01185 
01186 void SelectionDAGBuilder::visitRet(const ReturnInst &I) {
01187   const TargetLowering *TLI = TM.getTargetLowering();
01188   SDValue Chain = getControlRoot();
01189   SmallVector<ISD::OutputArg, 8> Outs;
01190   SmallVector<SDValue, 8> OutVals;
01191 
01192   if (!FuncInfo.CanLowerReturn) {
01193     unsigned DemoteReg = FuncInfo.DemoteRegister;
01194     const Function *F = I.getParent()->getParent();
01195 
01196     // Emit a store of the return value through the virtual register.
01197     // Leave Outs empty so that LowerReturn won't try to load return
01198     // registers the usual way.
01199     SmallVector<EVT, 1> PtrValueVTs;
01200     ComputeValueVTs(*TLI, PointerType::getUnqual(F->getReturnType()),
01201                     PtrValueVTs);
01202 
01203     SDValue RetPtr = DAG.getRegister(DemoteReg, PtrValueVTs[0]);
01204     SDValue RetOp = getValue(I.getOperand(0));
01205 
01206     SmallVector<EVT, 4> ValueVTs;
01207     SmallVector<uint64_t, 4> Offsets;
01208     ComputeValueVTs(*TLI, I.getOperand(0)->getType(), ValueVTs, &Offsets);
01209     unsigned NumValues = ValueVTs.size();
01210 
01211     SmallVector<SDValue, 4> Chains(NumValues);
01212     for (unsigned i = 0; i != NumValues; ++i) {
01213       SDValue Add = DAG.getNode(ISD::ADD, getCurSDLoc(),
01214                                 RetPtr.getValueType(), RetPtr,
01215                                 DAG.getIntPtrConstant(Offsets[i]));
01216       Chains[i] =
01217         DAG.getStore(Chain, getCurSDLoc(),
01218                      SDValue(RetOp.getNode(), RetOp.getResNo() + i),
01219                      // FIXME: better loc info would be nice.
01220                      Add, MachinePointerInfo(), false, false, 0);
01221     }
01222 
01223     Chain = DAG.getNode(ISD::TokenFactor, getCurSDLoc(),
01224                         MVT::Other, Chains);
01225   } else if (I.getNumOperands() != 0) {
01226     SmallVector<EVT, 4> ValueVTs;
01227     ComputeValueVTs(*TLI, I.getOperand(0)->getType(), ValueVTs);
01228     unsigned NumValues = ValueVTs.size();
01229     if (NumValues) {
01230       SDValue RetOp = getValue(I.getOperand(0));
01231       for (unsigned j = 0, f = NumValues; j != f; ++j) {
01232         EVT VT = ValueVTs[j];
01233 
01234         ISD::NodeType ExtendKind = ISD::ANY_EXTEND;
01235 
01236         const Function *F = I.getParent()->getParent();
01237         if (F->getAttributes().hasAttribute(AttributeSet::ReturnIndex,
01238                                             Attribute::SExt))
01239           ExtendKind = ISD::SIGN_EXTEND;
01240         else if (F->getAttributes().hasAttribute(AttributeSet::ReturnIndex,
01241                                                  Attribute::ZExt))
01242           ExtendKind = ISD::ZERO_EXTEND;
01243 
01244         if (ExtendKind != ISD::ANY_EXTEND && VT.isInteger())
01245           VT = TLI->getTypeForExtArgOrReturn(VT.getSimpleVT(), ExtendKind);
01246 
01247         unsigned NumParts = TLI->getNumRegisters(*DAG.getContext(), VT);
01248         MVT PartVT = TLI->getRegisterType(*DAG.getContext(), VT);
01249         SmallVector<SDValue, 4> Parts(NumParts);
01250         getCopyToParts(DAG, getCurSDLoc(),
01251                        SDValue(RetOp.getNode(), RetOp.getResNo() + j),
01252                        &Parts[0], NumParts, PartVT, &I, ExtendKind);
01253 
01254         // 'inreg' on function refers to return value
01255         ISD::ArgFlagsTy Flags = ISD::ArgFlagsTy();
01256         if (F->getAttributes().hasAttribute(AttributeSet::ReturnIndex,
01257                                             Attribute::InReg))
01258           Flags.setInReg();
01259 
01260         // Propagate extension type if any
01261         if (ExtendKind == ISD::SIGN_EXTEND)
01262           Flags.setSExt();
01263         else if (ExtendKind == ISD::ZERO_EXTEND)
01264           Flags.setZExt();
01265 
01266         for (unsigned i = 0; i < NumParts; ++i) {
01267           Outs.push_back(ISD::OutputArg(Flags, Parts[i].getValueType(),
01268                                         VT, /*isfixed=*/true, 0, 0));
01269           OutVals.push_back(Parts[i]);
01270         }
01271       }
01272     }
01273   }
01274 
01275   bool isVarArg = DAG.getMachineFunction().getFunction()->isVarArg();
01276   CallingConv::ID CallConv =
01277     DAG.getMachineFunction().getFunction()->getCallingConv();
01278   Chain = TM.getTargetLowering()->LowerReturn(Chain, CallConv, isVarArg,
01279                                               Outs, OutVals, getCurSDLoc(),
01280                                               DAG);
01281 
01282   // Verify that the target's LowerReturn behaved as expected.
01283   assert(Chain.getNode() && Chain.getValueType() == MVT::Other &&
01284          "LowerReturn didn't return a valid chain!");
01285 
01286   // Update the DAG with the new chain value resulting from return lowering.
01287   DAG.setRoot(Chain);
01288 }
01289 
01290 /// CopyToExportRegsIfNeeded - If the given value has virtual registers
01291 /// created for it, emit nodes to copy the value into the virtual
01292 /// registers.
01293 void SelectionDAGBuilder::CopyToExportRegsIfNeeded(const Value *V) {
01294   // Skip empty types
01295   if (V->getType()->isEmptyTy())
01296     return;
01297 
01298   DenseMap<const Value *, unsigned>::iterator VMI = FuncInfo.ValueMap.find(V);
01299   if (VMI != FuncInfo.ValueMap.end()) {
01300     assert(!V->use_empty() && "Unused value assigned virtual registers!");
01301     CopyValueToVirtualRegister(V, VMI->second);
01302   }
01303 }
01304 
01305 /// ExportFromCurrentBlock - If this condition isn't known to be exported from
01306 /// the current basic block, add it to ValueMap now so that we'll get a
01307 /// CopyTo/FromReg.
01308 void SelectionDAGBuilder::ExportFromCurrentBlock(const Value *V) {
01309   // No need to export constants.
01310   if (!isa<Instruction>(V) && !isa<Argument>(V)) return;
01311 
01312   // Already exported?
01313   if (FuncInfo.isExportedInst(V)) return;
01314 
01315   unsigned Reg = FuncInfo.InitializeRegForValue(V);
01316   CopyValueToVirtualRegister(V, Reg);
01317 }
01318 
01319 bool SelectionDAGBuilder::isExportableFromCurrentBlock(const Value *V,
01320                                                      const BasicBlock *FromBB) {
01321   // The operands of the setcc have to be in this block.  We don't know
01322   // how to export them from some other block.
01323   if (const Instruction *VI = dyn_cast<Instruction>(V)) {
01324     // Can export from current BB.
01325     if (VI->getParent() == FromBB)
01326       return true;
01327 
01328     // Is already exported, noop.
01329     return FuncInfo.isExportedInst(V);
01330   }
01331 
01332   // If this is an argument, we can export it if the BB is the entry block or
01333   // if it is already exported.
01334   if (isa<Argument>(V)) {
01335     if (FromBB == &FromBB->getParent()->getEntryBlock())
01336       return true;
01337 
01338     // Otherwise, can only export this if it is already exported.
01339     return FuncInfo.isExportedInst(V);
01340   }
01341 
01342   // Otherwise, constants can always be exported.
01343   return true;
01344 }
01345 
01346 /// Return branch probability calculated by BranchProbabilityInfo for IR blocks.
01347 uint32_t SelectionDAGBuilder::getEdgeWeight(const MachineBasicBlock *Src,
01348                                             const MachineBasicBlock *Dst) const {
01349   BranchProbabilityInfo *BPI = FuncInfo.BPI;
01350   if (!BPI)
01351     return 0;
01352   const BasicBlock *SrcBB = Src->getBasicBlock();
01353   const BasicBlock *DstBB = Dst->getBasicBlock();
01354   return BPI->getEdgeWeight(SrcBB, DstBB);
01355 }
01356 
01357 void SelectionDAGBuilder::
01358 addSuccessorWithWeight(MachineBasicBlock *Src, MachineBasicBlock *Dst,
01359                        uint32_t Weight /* = 0 */) {
01360   if (!Weight)
01361     Weight = getEdgeWeight(Src, Dst);
01362   Src->addSuccessor(Dst, Weight);
01363 }
01364 
01365 
01366 static bool InBlock(const Value *V, const BasicBlock *BB) {
01367   if (const Instruction *I = dyn_cast<Instruction>(V))
01368     return I->getParent() == BB;
01369   return true;
01370 }
01371 
01372 /// EmitBranchForMergedCondition - Helper method for FindMergedConditions.
01373 /// This function emits a branch and is used at the leaves of an OR or an
01374 /// AND operator tree.
01375 ///
01376 void
01377 SelectionDAGBuilder::EmitBranchForMergedCondition(const Value *Cond,
01378                                                   MachineBasicBlock *TBB,
01379                                                   MachineBasicBlock *FBB,
01380                                                   MachineBasicBlock *CurBB,
01381                                                   MachineBasicBlock *SwitchBB,
01382                                                   uint32_t TWeight,
01383                                                   uint32_t FWeight) {
01384   const BasicBlock *BB = CurBB->getBasicBlock();
01385 
01386   // If the leaf of the tree is a comparison, merge the condition into
01387   // the caseblock.
01388   if (const CmpInst *BOp = dyn_cast<CmpInst>(Cond)) {
01389     // The operands of the cmp have to be in this block.  We don't know
01390     // how to export them from some other block.  If this is the first block
01391     // of the sequence, no exporting is needed.
01392     if (CurBB == SwitchBB ||
01393         (isExportableFromCurrentBlock(BOp->getOperand(0), BB) &&
01394          isExportableFromCurrentBlock(BOp->getOperand(1), BB))) {
01395       ISD::CondCode Condition;
01396       if (const ICmpInst *IC = dyn_cast<ICmpInst>(Cond)) {
01397         Condition = getICmpCondCode(IC->getPredicate());
01398       } else if (const FCmpInst *FC = dyn_cast<FCmpInst>(Cond)) {
01399         Condition = getFCmpCondCode(FC->getPredicate());
01400         if (TM.Options.NoNaNsFPMath)
01401           Condition = getFCmpCodeWithoutNaN(Condition);
01402       } else {
01403         Condition = ISD::SETEQ; // silence warning.
01404         llvm_unreachable("Unknown compare instruction");
01405       }
01406 
01407       CaseBlock CB(Condition, BOp->getOperand(0), BOp->getOperand(1), nullptr,
01408                    TBB, FBB, CurBB, TWeight, FWeight);
01409       SwitchCases.push_back(CB);
01410       return;
01411     }
01412   }
01413 
01414   // Create a CaseBlock record representing this branch.
01415   CaseBlock CB(ISD::SETEQ, Cond, ConstantInt::getTrue(*DAG.getContext()),
01416                nullptr, TBB, FBB, CurBB, TWeight, FWeight);
01417   SwitchCases.push_back(CB);
01418 }
01419 
01420 /// Scale down both weights to fit into uint32_t.
01421 static void ScaleWeights(uint64_t &NewTrue, uint64_t &NewFalse) {
01422   uint64_t NewMax = (NewTrue > NewFalse) ? NewTrue : NewFalse;
01423   uint32_t Scale = (NewMax / UINT32_MAX) + 1;
01424   NewTrue = NewTrue / Scale;
01425   NewFalse = NewFalse / Scale;
01426 }
01427 
01428 /// FindMergedConditions - If Cond is an expression like
01429 void SelectionDAGBuilder::FindMergedConditions(const Value *Cond,
01430                                                MachineBasicBlock *TBB,
01431                                                MachineBasicBlock *FBB,
01432                                                MachineBasicBlock *CurBB,
01433                                                MachineBasicBlock *SwitchBB,
01434                                                unsigned Opc, uint32_t TWeight,
01435                                                uint32_t FWeight) {
01436   // If this node is not part of the or/and tree, emit it as a branch.
01437   const Instruction *BOp = dyn_cast<Instruction>(Cond);
01438   if (!BOp || !(isa<BinaryOperator>(BOp) || isa<CmpInst>(BOp)) ||
01439       (unsigned)BOp->getOpcode() != Opc || !BOp->hasOneUse() ||
01440       BOp->getParent() != CurBB->getBasicBlock() ||
01441       !InBlock(BOp->getOperand(0), CurBB->getBasicBlock()) ||
01442       !InBlock(BOp->getOperand(1), CurBB->getBasicBlock())) {
01443     EmitBranchForMergedCondition(Cond, TBB, FBB, CurBB, SwitchBB,
01444                                  TWeight, FWeight);
01445     return;
01446   }
01447 
01448   //  Create TmpBB after CurBB.
01449   MachineFunction::iterator BBI = CurBB;
01450   MachineFunction &MF = DAG.getMachineFunction();
01451   MachineBasicBlock *TmpBB = MF.CreateMachineBasicBlock(CurBB->getBasicBlock());
01452   CurBB->getParent()->insert(++BBI, TmpBB);
01453 
01454   if (Opc == Instruction::Or) {
01455     // Codegen X | Y as:
01456     // BB1:
01457     //   jmp_if_X TBB
01458     //   jmp TmpBB
01459     // TmpBB:
01460     //   jmp_if_Y TBB
01461     //   jmp FBB
01462     //
01463 
01464     // We have flexibility in setting Prob for BB1 and Prob for TmpBB.
01465     // The requirement is that
01466     //   TrueProb for BB1 + (FalseProb for BB1 * TrueProb for TmpBB)
01467     //     = TrueProb for orignal BB.
01468     // Assuming the orignal weights are A and B, one choice is to set BB1's
01469     // weights to A and A+2B, and set TmpBB's weights to A and 2B. This choice
01470     // assumes that
01471     //   TrueProb for BB1 == FalseProb for BB1 * TrueProb for TmpBB.
01472     // Another choice is to assume TrueProb for BB1 equals to TrueProb for
01473     // TmpBB, but the math is more complicated.
01474 
01475     uint64_t NewTrueWeight = TWeight;
01476     uint64_t NewFalseWeight = (uint64_t)TWeight + 2 * (uint64_t)FWeight;
01477     ScaleWeights(NewTrueWeight, NewFalseWeight);
01478     // Emit the LHS condition.
01479     FindMergedConditions(BOp->getOperand(0), TBB, TmpBB, CurBB, SwitchBB, Opc,
01480                          NewTrueWeight, NewFalseWeight);
01481 
01482     NewTrueWeight = TWeight;
01483     NewFalseWeight = 2 * (uint64_t)FWeight;
01484     ScaleWeights(NewTrueWeight, NewFalseWeight);
01485     // Emit the RHS condition into TmpBB.
01486     FindMergedConditions(BOp->getOperand(1), TBB, FBB, TmpBB, SwitchBB, Opc,
01487                          NewTrueWeight, NewFalseWeight);
01488   } else {
01489     assert(Opc == Instruction::And && "Unknown merge op!");
01490     // Codegen X & Y as:
01491     // BB1:
01492     //   jmp_if_X TmpBB
01493     //   jmp FBB
01494     // TmpBB:
01495     //   jmp_if_Y TBB
01496     //   jmp FBB
01497     //
01498     //  This requires creation of TmpBB after CurBB.
01499 
01500     // We have flexibility in setting Prob for BB1 and Prob for TmpBB.
01501     // The requirement is that
01502     //   FalseProb for BB1 + (TrueProb for BB1 * FalseProb for TmpBB)
01503     //     = FalseProb for orignal BB.
01504     // Assuming the orignal weights are A and B, one choice is to set BB1's
01505     // weights to 2A+B and B, and set TmpBB's weights to 2A and B. This choice
01506     // assumes that
01507     //   FalseProb for BB1 == TrueProb for BB1 * FalseProb for TmpBB.
01508 
01509     uint64_t NewTrueWeight = 2 * (uint64_t)TWeight + (uint64_t)FWeight;
01510     uint64_t NewFalseWeight = FWeight;
01511     ScaleWeights(NewTrueWeight, NewFalseWeight);
01512     // Emit the LHS condition.
01513     FindMergedConditions(BOp->getOperand(0), TmpBB, FBB, CurBB, SwitchBB, Opc,
01514                          NewTrueWeight, NewFalseWeight);
01515 
01516     NewTrueWeight = 2 * (uint64_t)TWeight;
01517     NewFalseWeight = FWeight;
01518     ScaleWeights(NewTrueWeight, NewFalseWeight);
01519     // Emit the RHS condition into TmpBB.
01520     FindMergedConditions(BOp->getOperand(1), TBB, FBB, TmpBB, SwitchBB, Opc,
01521                          NewTrueWeight, NewFalseWeight);
01522   }
01523 }
01524 
01525 /// If the set of cases should be emitted as a series of branches, return true.
01526 /// If we should emit this as a bunch of and/or'd together conditions, return
01527 /// false.
01528 bool
01529 SelectionDAGBuilder::ShouldEmitAsBranches(const std::vector<CaseBlock> &Cases) {
01530   if (Cases.size() != 2) return true;
01531 
01532   // If this is two comparisons of the same values or'd or and'd together, they
01533   // will get folded into a single comparison, so don't emit two blocks.
01534   if ((Cases[0].CmpLHS == Cases[1].CmpLHS &&
01535        Cases[0].CmpRHS == Cases[1].CmpRHS) ||
01536       (Cases[0].CmpRHS == Cases[1].CmpLHS &&
01537        Cases[0].CmpLHS == Cases[1].CmpRHS)) {
01538     return false;
01539   }
01540 
01541   // Handle: (X != null) | (Y != null) --> (X|Y) != 0
01542   // Handle: (X == null) & (Y == null) --> (X|Y) == 0
01543   if (Cases[0].CmpRHS == Cases[1].CmpRHS &&
01544       Cases[0].CC == Cases[1].CC &&
01545       isa<Constant>(Cases[0].CmpRHS) &&
01546       cast<Constant>(Cases[0].CmpRHS)->isNullValue()) {
01547     if (Cases[0].CC == ISD::SETEQ && Cases[0].TrueBB == Cases[1].ThisBB)
01548       return false;
01549     if (Cases[0].CC == ISD::SETNE && Cases[0].FalseBB == Cases[1].ThisBB)
01550       return false;
01551   }
01552 
01553   return true;
01554 }
01555 
01556 void SelectionDAGBuilder::visitBr(const BranchInst &I) {
01557   MachineBasicBlock *BrMBB = FuncInfo.MBB;
01558 
01559   // Update machine-CFG edges.
01560   MachineBasicBlock *Succ0MBB = FuncInfo.MBBMap[I.getSuccessor(0)];
01561 
01562   // Figure out which block is immediately after the current one.
01563   MachineBasicBlock *NextBlock = nullptr;
01564   MachineFunction::iterator BBI = BrMBB;
01565   if (++BBI != FuncInfo.MF->end())
01566     NextBlock = BBI;
01567 
01568   if (I.isUnconditional()) {
01569     // Update machine-CFG edges.
01570     BrMBB->addSuccessor(Succ0MBB);
01571 
01572     // If this is not a fall-through branch or optimizations are switched off,
01573     // emit the branch.
01574     if (Succ0MBB != NextBlock || TM.getOptLevel() == CodeGenOpt::None)
01575       DAG.setRoot(DAG.getNode(ISD::BR, getCurSDLoc(),
01576                               MVT::Other, getControlRoot(),
01577                               DAG.getBasicBlock(Succ0MBB)));
01578 
01579     return;
01580   }
01581 
01582   // If this condition is one of the special cases we handle, do special stuff
01583   // now.
01584   const Value *CondVal = I.getCondition();
01585   MachineBasicBlock *Succ1MBB = FuncInfo.MBBMap[I.getSuccessor(1)];
01586 
01587   // If this is a series of conditions that are or'd or and'd together, emit
01588   // this as a sequence of branches instead of setcc's with and/or operations.
01589   // As long as jumps are not expensive, this should improve performance.
01590   // For example, instead of something like:
01591   //     cmp A, B
01592   //     C = seteq
01593   //     cmp D, E
01594   //     F = setle
01595   //     or C, F
01596   //     jnz foo
01597   // Emit:
01598   //     cmp A, B
01599   //     je foo
01600   //     cmp D, E
01601   //     jle foo
01602   //
01603   if (const BinaryOperator *BOp = dyn_cast<BinaryOperator>(CondVal)) {
01604     if (!TM.getTargetLowering()->isJumpExpensive() &&
01605         BOp->hasOneUse() &&
01606         (BOp->getOpcode() == Instruction::And ||
01607          BOp->getOpcode() == Instruction::Or)) {
01608       FindMergedConditions(BOp, Succ0MBB, Succ1MBB, BrMBB, BrMBB,
01609                            BOp->getOpcode(), getEdgeWeight(BrMBB, Succ0MBB),
01610                            getEdgeWeight(BrMBB, Succ1MBB));
01611       // If the compares in later blocks need to use values not currently
01612       // exported from this block, export them now.  This block should always
01613       // be the first entry.
01614       assert(SwitchCases[0].ThisBB == BrMBB && "Unexpected lowering!");
01615 
01616       // Allow some cases to be rejected.
01617       if (ShouldEmitAsBranches(SwitchCases)) {
01618         for (unsigned i = 1, e = SwitchCases.size(); i != e; ++i) {
01619           ExportFromCurrentBlock(SwitchCases[i].CmpLHS);
01620           ExportFromCurrentBlock(SwitchCases[i].CmpRHS);
01621         }
01622 
01623         // Emit the branch for this block.
01624         visitSwitchCase(SwitchCases[0], BrMBB);
01625         SwitchCases.erase(SwitchCases.begin());
01626         return;
01627       }
01628 
01629       // Okay, we decided not to do this, remove any inserted MBB's and clear
01630       // SwitchCases.
01631       for (unsigned i = 1, e = SwitchCases.size(); i != e; ++i)
01632         FuncInfo.MF->erase(SwitchCases[i].ThisBB);
01633 
01634       SwitchCases.clear();
01635     }
01636   }
01637 
01638   // Create a CaseBlock record representing this branch.
01639   CaseBlock CB(ISD::SETEQ, CondVal, ConstantInt::getTrue(*DAG.getContext()),
01640                nullptr, Succ0MBB, Succ1MBB, BrMBB);
01641 
01642   // Use visitSwitchCase to actually insert the fast branch sequence for this
01643   // cond branch.
01644   visitSwitchCase(CB, BrMBB);
01645 }
01646 
01647 /// visitSwitchCase - Emits the necessary code to represent a single node in
01648 /// the binary search tree resulting from lowering a switch instruction.
01649 void SelectionDAGBuilder::visitSwitchCase(CaseBlock &CB,
01650                                           MachineBasicBlock *SwitchBB) {
01651   SDValue Cond;
01652   SDValue CondLHS = getValue(CB.CmpLHS);
01653   SDLoc dl = getCurSDLoc();
01654 
01655   // Build the setcc now.
01656   if (!CB.CmpMHS) {
01657     // Fold "(X == true)" to X and "(X == false)" to !X to
01658     // handle common cases produced by branch lowering.
01659     if (CB.CmpRHS == ConstantInt::getTrue(*DAG.getContext()) &&
01660         CB.CC == ISD::SETEQ)
01661       Cond = CondLHS;
01662     else if (CB.CmpRHS == ConstantInt::getFalse(*DAG.getContext()) &&
01663              CB.CC == ISD::SETEQ) {
01664       SDValue True = DAG.getConstant(1, CondLHS.getValueType());
01665       Cond = DAG.getNode(ISD::XOR, dl, CondLHS.getValueType(), CondLHS, True);
01666     } else
01667       Cond = DAG.getSetCC(dl, MVT::i1, CondLHS, getValue(CB.CmpRHS), CB.CC);
01668   } else {
01669     assert(CB.CC == ISD::SETLE && "Can handle only LE ranges now");
01670 
01671     const APInt& Low = cast<ConstantInt>(CB.CmpLHS)->getValue();
01672     const APInt& High  = cast<ConstantInt>(CB.CmpRHS)->getValue();
01673 
01674     SDValue CmpOp = getValue(CB.CmpMHS);
01675     EVT VT = CmpOp.getValueType();
01676 
01677     if (cast<ConstantInt>(CB.CmpLHS)->isMinValue(true)) {
01678       Cond = DAG.getSetCC(dl, MVT::i1, CmpOp, DAG.getConstant(High, VT),
01679                           ISD::SETLE);
01680     } else {
01681       SDValue SUB = DAG.getNode(ISD::SUB, dl,
01682                                 VT, CmpOp, DAG.getConstant(Low, VT));
01683       Cond = DAG.getSetCC(dl, MVT::i1, SUB,
01684                           DAG.getConstant(High-Low, VT), ISD::SETULE);
01685     }
01686   }
01687 
01688   // Update successor info
01689   addSuccessorWithWeight(SwitchBB, CB.TrueBB, CB.TrueWeight);
01690   // TrueBB and FalseBB are always different unless the incoming IR is
01691   // degenerate. This only happens when running llc on weird IR.
01692   if (CB.TrueBB != CB.FalseBB)
01693     addSuccessorWithWeight(SwitchBB, CB.FalseBB, CB.FalseWeight);
01694 
01695   // Set NextBlock to be the MBB immediately after the current one, if any.
01696   // This is used to avoid emitting unnecessary branches to the next block.
01697   MachineBasicBlock *NextBlock = nullptr;
01698   MachineFunction::iterator BBI = SwitchBB;
01699   if (++BBI != FuncInfo.MF->end())
01700     NextBlock = BBI;
01701 
01702   // If the lhs block is the next block, invert the condition so that we can
01703   // fall through to the lhs instead of the rhs block.
01704   if (CB.TrueBB == NextBlock) {
01705     std::swap(CB.TrueBB, CB.FalseBB);
01706     SDValue True = DAG.getConstant(1, Cond.getValueType());
01707     Cond = DAG.getNode(ISD::XOR, dl, Cond.getValueType(), Cond, True);
01708   }
01709 
01710   SDValue BrCond = DAG.getNode(ISD::BRCOND, dl,
01711                                MVT::Other, getControlRoot(), Cond,
01712                                DAG.getBasicBlock(CB.TrueBB));
01713 
01714   // Insert the false branch. Do this even if it's a fall through branch,
01715   // this makes it easier to do DAG optimizations which require inverting
01716   // the branch condition.
01717   BrCond = DAG.getNode(ISD::BR, dl, MVT::Other, BrCond,
01718                        DAG.getBasicBlock(CB.FalseBB));
01719 
01720   DAG.setRoot(BrCond);
01721 }
01722 
01723 /// visitJumpTable - Emit JumpTable node in the current MBB
01724 void SelectionDAGBuilder::visitJumpTable(JumpTable &JT) {
01725   // Emit the code for the jump table
01726   assert(JT.Reg != -1U && "Should lower JT Header first!");
01727   EVT PTy = TM.getTargetLowering()->getPointerTy();
01728   SDValue Index = DAG.getCopyFromReg(getControlRoot(), getCurSDLoc(),
01729                                      JT.Reg, PTy);
01730   SDValue Table = DAG.getJumpTable(JT.JTI, PTy);
01731   SDValue BrJumpTable = DAG.getNode(ISD::BR_JT, getCurSDLoc(),
01732                                     MVT::Other, Index.getValue(1),
01733                                     Table, Index);
01734   DAG.setRoot(BrJumpTable);
01735 }
01736 
01737 /// visitJumpTableHeader - This function emits necessary code to produce index
01738 /// in the JumpTable from switch case.
01739 void SelectionDAGBuilder::visitJumpTableHeader(JumpTable &JT,
01740                                                JumpTableHeader &JTH,
01741                                                MachineBasicBlock *SwitchBB) {
01742   // Subtract the lowest switch case value from the value being switched on and
01743   // conditional branch to default mbb if the result is greater than the
01744   // difference between smallest and largest cases.
01745   SDValue SwitchOp = getValue(JTH.SValue);
01746   EVT VT = SwitchOp.getValueType();
01747   SDValue Sub = DAG.getNode(ISD::SUB, getCurSDLoc(), VT, SwitchOp,
01748                             DAG.getConstant(JTH.First, VT));
01749 
01750   // The SDNode we just created, which holds the value being switched on minus
01751   // the smallest case value, needs to be copied to a virtual register so it
01752   // can be used as an index into the jump table in a subsequent basic block.
01753   // This value may be smaller or larger than the target's pointer type, and
01754   // therefore require extension or truncating.
01755   const TargetLowering *TLI = TM.getTargetLowering();
01756   SwitchOp = DAG.getZExtOrTrunc(Sub, getCurSDLoc(), TLI->getPointerTy());
01757 
01758   unsigned JumpTableReg = FuncInfo.CreateReg(TLI->getPointerTy());
01759   SDValue CopyTo = DAG.getCopyToReg(getControlRoot(), getCurSDLoc(),
01760                                     JumpTableReg, SwitchOp);
01761   JT.Reg = JumpTableReg;
01762 
01763   // Emit the range check for the jump table, and branch to the default block
01764   // for the switch statement if the value being switched on exceeds the largest
01765   // case in the switch.
01766   SDValue CMP = DAG.getSetCC(getCurSDLoc(),
01767                              TLI->getSetCCResultType(*DAG.getContext(),
01768                                                      Sub.getValueType()),
01769                              Sub,
01770                              DAG.getConstant(JTH.Last - JTH.First,VT),
01771                              ISD::SETUGT);
01772 
01773   // Set NextBlock to be the MBB immediately after the current one, if any.
01774   // This is used to avoid emitting unnecessary branches to the next block.
01775   MachineBasicBlock *NextBlock = nullptr;
01776   MachineFunction::iterator BBI = SwitchBB;
01777 
01778   if (++BBI != FuncInfo.MF->end())
01779     NextBlock = BBI;
01780 
01781   SDValue BrCond = DAG.getNode(ISD::BRCOND, getCurSDLoc(),
01782                                MVT::Other, CopyTo, CMP,
01783                                DAG.getBasicBlock(JT.Default));
01784 
01785   if (JT.MBB != NextBlock)
01786     BrCond = DAG.getNode(ISD::BR, getCurSDLoc(), MVT::Other, BrCond,
01787                          DAG.getBasicBlock(JT.MBB));
01788 
01789   DAG.setRoot(BrCond);
01790 }
01791 
01792 /// Codegen a new tail for a stack protector check ParentMBB which has had its
01793 /// tail spliced into a stack protector check success bb.
01794 ///
01795 /// For a high level explanation of how this fits into the stack protector
01796 /// generation see the comment on the declaration of class
01797 /// StackProtectorDescriptor.
01798 void SelectionDAGBuilder::visitSPDescriptorParent(StackProtectorDescriptor &SPD,
01799                                                   MachineBasicBlock *ParentBB) {
01800 
01801   // First create the loads to the guard/stack slot for the comparison.
01802   const TargetLowering *TLI = TM.getTargetLowering();
01803   EVT PtrTy = TLI->getPointerTy();
01804 
01805   MachineFrameInfo *MFI = ParentBB->getParent()->getFrameInfo();
01806   int FI = MFI->getStackProtectorIndex();
01807 
01808   const Value *IRGuard = SPD.getGuard();
01809   SDValue GuardPtr = getValue(IRGuard);
01810   SDValue StackSlotPtr = DAG.getFrameIndex(FI, PtrTy);
01811 
01812   unsigned Align =
01813     TLI->getDataLayout()->getPrefTypeAlignment(IRGuard->getType());
01814 
01815   SDValue Guard;
01816 
01817   // If useLoadStackGuardNode returns true, retrieve the guard value from
01818   // the virtual register holding the value. Otherwise, emit a volatile load
01819   // to retrieve the stack guard value.
01820   if (TLI->useLoadStackGuardNode())
01821     Guard = DAG.getCopyFromReg(DAG.getEntryNode(), getCurSDLoc(),
01822                                SPD.getGuardReg(), PtrTy);
01823   else
01824     Guard = DAG.getLoad(PtrTy, getCurSDLoc(), DAG.getEntryNode(),
01825                         GuardPtr, MachinePointerInfo(IRGuard, 0),
01826                         true, false, false, Align);
01827 
01828   SDValue StackSlot = DAG.getLoad(PtrTy, getCurSDLoc(), DAG.getEntryNode(),
01829                                   StackSlotPtr,
01830                                   MachinePointerInfo::getFixedStack(FI),
01831                                   true, false, false, Align);
01832 
01833   // Perform the comparison via a subtract/getsetcc.
01834   EVT VT = Guard.getValueType();
01835   SDValue Sub = DAG.getNode(ISD::SUB, getCurSDLoc(), VT, Guard, StackSlot);
01836 
01837   SDValue Cmp = DAG.getSetCC(getCurSDLoc(),
01838                              TLI->getSetCCResultType(*DAG.getContext(),
01839                                                      Sub.getValueType()),
01840                              Sub, DAG.getConstant(0, VT),
01841                              ISD::SETNE);
01842 
01843   // If the sub is not 0, then we know the guard/stackslot do not equal, so
01844   // branch to failure MBB.
01845   SDValue BrCond = DAG.getNode(ISD::BRCOND, getCurSDLoc(),
01846                                MVT::Other, StackSlot.getOperand(0),
01847                                Cmp, DAG.getBasicBlock(SPD.getFailureMBB()));
01848   // Otherwise branch to success MBB.
01849   SDValue Br = DAG.getNode(ISD::BR, getCurSDLoc(),
01850                            MVT::Other, BrCond,
01851                            DAG.getBasicBlock(SPD.getSuccessMBB()));
01852 
01853   DAG.setRoot(Br);
01854 }
01855 
01856 /// Codegen the failure basic block for a stack protector check.
01857 ///
01858 /// A failure stack protector machine basic block consists simply of a call to
01859 /// __stack_chk_fail().
01860 ///
01861 /// For a high level explanation of how this fits into the stack protector
01862 /// generation see the comment on the declaration of class
01863 /// StackProtectorDescriptor.
01864 void
01865 SelectionDAGBuilder::visitSPDescriptorFailure(StackProtectorDescriptor &SPD) {
01866   const TargetLowering *TLI = TM.getTargetLowering();
01867   SDValue Chain = TLI->makeLibCall(DAG, RTLIB::STACKPROTECTOR_CHECK_FAIL,
01868                                    MVT::isVoid, nullptr, 0, false,
01869                                    getCurSDLoc(), false, false).second;
01870   DAG.setRoot(Chain);
01871 }
01872 
01873 /// visitBitTestHeader - This function emits necessary code to produce value
01874 /// suitable for "bit tests"
01875 void SelectionDAGBuilder::visitBitTestHeader(BitTestBlock &B,
01876                                              MachineBasicBlock *SwitchBB) {
01877   // Subtract the minimum value
01878   SDValue SwitchOp = getValue(B.SValue);
01879   EVT VT = SwitchOp.getValueType();
01880   SDValue Sub = DAG.getNode(ISD::SUB, getCurSDLoc(), VT, SwitchOp,
01881                             DAG.getConstant(B.First, VT));
01882 
01883   // Check range
01884   const TargetLowering *TLI = TM.getTargetLowering();
01885   SDValue RangeCmp = DAG.getSetCC(getCurSDLoc(),
01886                                   TLI->getSetCCResultType(*DAG.getContext(),
01887                                                          Sub.getValueType()),
01888                                   Sub, DAG.getConstant(B.Range, VT),
01889                                   ISD::SETUGT);
01890 
01891   // Determine the type of the test operands.
01892   bool UsePtrType = false;
01893   if (!TLI->isTypeLegal(VT))
01894     UsePtrType = true;
01895   else {
01896     for (unsigned i = 0, e = B.Cases.size(); i != e; ++i)
01897       if (!isUIntN(VT.getSizeInBits(), B.Cases[i].Mask)) {
01898         // Switch table case range are encoded into series of masks.
01899         // Just use pointer type, it's guaranteed to fit.
01900         UsePtrType = true;
01901         break;
01902       }
01903   }
01904   if (UsePtrType) {
01905     VT = TLI->getPointerTy();
01906     Sub = DAG.getZExtOrTrunc(Sub, getCurSDLoc(), VT);
01907   }
01908 
01909   B.RegVT = VT.getSimpleVT();
01910   B.Reg = FuncInfo.CreateReg(B.RegVT);
01911   SDValue CopyTo = DAG.getCopyToReg(getControlRoot(), getCurSDLoc(),
01912                                     B.Reg, Sub);
01913 
01914   // Set NextBlock to be the MBB immediately after the current one, if any.
01915   // This is used to avoid emitting unnecessary branches to the next block.
01916   MachineBasicBlock *NextBlock = nullptr;
01917   MachineFunction::iterator BBI = SwitchBB;
01918   if (++BBI != FuncInfo.MF->end())
01919     NextBlock = BBI;
01920 
01921   MachineBasicBlock* MBB = B.Cases[0].ThisBB;
01922 
01923   addSuccessorWithWeight(SwitchBB, B.Default);
01924   addSuccessorWithWeight(SwitchBB, MBB);
01925 
01926   SDValue BrRange = DAG.getNode(ISD::BRCOND, getCurSDLoc(),
01927                                 MVT::Other, CopyTo, RangeCmp,
01928                                 DAG.getBasicBlock(B.Default));
01929 
01930   if (MBB != NextBlock)
01931     BrRange = DAG.getNode(ISD::BR, getCurSDLoc(), MVT::Other, CopyTo,
01932                           DAG.getBasicBlock(MBB));
01933 
01934   DAG.setRoot(BrRange);
01935 }
01936 
01937 /// visitBitTestCase - this function produces one "bit test"
01938 void SelectionDAGBuilder::visitBitTestCase(BitTestBlock &BB,
01939                                            MachineBasicBlock* NextMBB,
01940                                            uint32_t BranchWeightToNext,
01941                                            unsigned Reg,
01942                                            BitTestCase &B,
01943                                            MachineBasicBlock *SwitchBB) {
01944   MVT VT = BB.RegVT;
01945   SDValue ShiftOp = DAG.getCopyFromReg(getControlRoot(), getCurSDLoc(),
01946                                        Reg, VT);
01947   SDValue Cmp;
01948   unsigned PopCount = CountPopulation_64(B.Mask);
01949   const TargetLowering *TLI = TM.getTargetLowering();
01950   if (PopCount == 1) {
01951     // Testing for a single bit; just compare the shift count with what it
01952     // would need to be to shift a 1 bit in that position.
01953     Cmp = DAG.getSetCC(getCurSDLoc(),
01954                        TLI->getSetCCResultType(*DAG.getContext(), VT),
01955                        ShiftOp,
01956                        DAG.getConstant(countTrailingZeros(B.Mask), VT),
01957                        ISD::SETEQ);
01958   } else if (PopCount == BB.Range) {
01959     // There is only one zero bit in the range, test for it directly.
01960     Cmp = DAG.getSetCC(getCurSDLoc(),
01961                        TLI->getSetCCResultType(*DAG.getContext(), VT),
01962                        ShiftOp,
01963                        DAG.getConstant(CountTrailingOnes_64(B.Mask), VT),
01964                        ISD::SETNE);
01965   } else {
01966     // Make desired shift
01967     SDValue SwitchVal = DAG.getNode(ISD::SHL, getCurSDLoc(), VT,
01968                                     DAG.getConstant(1, VT), ShiftOp);
01969 
01970     // Emit bit tests and jumps
01971     SDValue AndOp = DAG.getNode(ISD::AND, getCurSDLoc(),
01972                                 VT, SwitchVal, DAG.getConstant(B.Mask, VT));
01973     Cmp = DAG.getSetCC(getCurSDLoc(),
01974                        TLI->getSetCCResultType(*DAG.getContext(), VT),
01975                        AndOp, DAG.getConstant(0, VT),
01976                        ISD::SETNE);
01977   }
01978 
01979   // The branch weight from SwitchBB to B.TargetBB is B.ExtraWeight.
01980   addSuccessorWithWeight(SwitchBB, B.TargetBB, B.ExtraWeight);
01981   // The branch weight from SwitchBB to NextMBB is BranchWeightToNext.
01982   addSuccessorWithWeight(SwitchBB, NextMBB, BranchWeightToNext);
01983 
01984   SDValue BrAnd = DAG.getNode(ISD::BRCOND, getCurSDLoc(),
01985                               MVT::Other, getControlRoot(),
01986                               Cmp, DAG.getBasicBlock(B.TargetBB));
01987 
01988   // Set NextBlock to be the MBB immediately after the current one, if any.
01989   // This is used to avoid emitting unnecessary branches to the next block.
01990   MachineBasicBlock *NextBlock = nullptr;
01991   MachineFunction::iterator BBI = SwitchBB;
01992   if (++BBI != FuncInfo.MF->end())
01993     NextBlock = BBI;
01994 
01995   if (NextMBB != NextBlock)
01996     BrAnd = DAG.getNode(ISD::BR, getCurSDLoc(), MVT::Other, BrAnd,
01997                         DAG.getBasicBlock(NextMBB));
01998 
01999   DAG.setRoot(BrAnd);
02000 }
02001 
02002 void SelectionDAGBuilder::visitInvoke(const InvokeInst &I) {
02003   MachineBasicBlock *InvokeMBB = FuncInfo.MBB;
02004 
02005   // Retrieve successors.
02006   MachineBasicBlock *Return = FuncInfo.MBBMap[I.getSuccessor(0)];
02007   MachineBasicBlock *LandingPad = FuncInfo.MBBMap[I.getSuccessor(1)];
02008 
02009   const Value *Callee(I.getCalledValue());
02010   const Function *Fn = dyn_cast<Function>(Callee);
02011   if (isa<InlineAsm>(Callee))
02012     visitInlineAsm(&I);
02013   else if (Fn && Fn->isIntrinsic()) {
02014     assert(Fn->getIntrinsicID() == Intrinsic::donothing);
02015     // Ignore invokes to @llvm.donothing: jump directly to the next BB.
02016   } else
02017     LowerCallTo(&I, getValue(Callee), false, LandingPad);
02018 
02019   // If the value of the invoke is used outside of its defining block, make it
02020   // available as a virtual register.
02021   CopyToExportRegsIfNeeded(&I);
02022 
02023   // Update successor info
02024   addSuccessorWithWeight(InvokeMBB, Return);
02025   addSuccessorWithWeight(InvokeMBB, LandingPad);
02026 
02027   // Drop into normal successor.
02028   DAG.setRoot(DAG.getNode(ISD::BR, getCurSDLoc(),
02029                           MVT::Other, getControlRoot(),
02030                           DAG.getBasicBlock(Return)));
02031 }
02032 
02033 void SelectionDAGBuilder::visitResume(const ResumeInst &RI) {
02034   llvm_unreachable("SelectionDAGBuilder shouldn't visit resume instructions!");
02035 }
02036 
02037 void SelectionDAGBuilder::visitLandingPad(const LandingPadInst &LP) {
02038   assert(FuncInfo.MBB->isLandingPad() &&
02039          "Call to landingpad not in landing pad!");
02040 
02041   MachineBasicBlock *MBB = FuncInfo.MBB;
02042   MachineModuleInfo &MMI = DAG.getMachineFunction().getMMI();
02043   AddLandingPadInfo(LP, MMI, MBB);
02044 
02045   // If there aren't registers to copy the values into (e.g., during SjLj
02046   // exceptions), then don't bother to create these DAG nodes.
02047   const TargetLowering *TLI = TM.getTargetLowering();
02048   if (TLI->getExceptionPointerRegister() == 0 &&
02049       TLI->getExceptionSelectorRegister() == 0)
02050     return;
02051 
02052   SmallVector<EVT, 2> ValueVTs;
02053   ComputeValueVTs(*TLI, LP.getType(), ValueVTs);
02054   assert(ValueVTs.size() == 2 && "Only two-valued landingpads are supported");
02055 
02056   // Get the two live-in registers as SDValues. The physregs have already been
02057   // copied into virtual registers.
02058   SDValue Ops[2];
02059   Ops[0] = DAG.getZExtOrTrunc(
02060     DAG.getCopyFromReg(DAG.getEntryNode(), getCurSDLoc(),
02061                        FuncInfo.ExceptionPointerVirtReg, TLI->getPointerTy()),
02062     getCurSDLoc(), ValueVTs[0]);
02063   Ops[1] = DAG.getZExtOrTrunc(
02064     DAG.getCopyFromReg(DAG.getEntryNode(), getCurSDLoc(),
02065                        FuncInfo.ExceptionSelectorVirtReg, TLI->getPointerTy()),
02066     getCurSDLoc(), ValueVTs[1]);
02067 
02068   // Merge into one.
02069   SDValue Res = DAG.getNode(ISD::MERGE_VALUES, getCurSDLoc(),
02070                             DAG.getVTList(ValueVTs), Ops);
02071   setValue(&LP, Res);
02072 }
02073 
02074 /// handleSmallSwitchCaseRange - Emit a series of specific tests (suitable for
02075 /// small case ranges).
02076 bool SelectionDAGBuilder::handleSmallSwitchRange(CaseRec& CR,
02077                                                  CaseRecVector& WorkList,
02078                                                  const Value* SV,
02079                                                  MachineBasicBlock *Default,
02080                                                  MachineBasicBlock *SwitchBB) {
02081   // Size is the number of Cases represented by this range.
02082   size_t Size = CR.Range.second - CR.Range.first;
02083   if (Size > 3)
02084     return false;
02085 
02086   // Get the MachineFunction which holds the current MBB.  This is used when
02087   // inserting any additional MBBs necessary to represent the switch.
02088   MachineFunction *CurMF = FuncInfo.MF;
02089 
02090   // Figure out which block is immediately after the current one.
02091   MachineBasicBlock *NextBlock = nullptr;
02092   MachineFunction::iterator BBI = CR.CaseBB;
02093 
02094   if (++BBI != FuncInfo.MF->end())
02095     NextBlock = BBI;
02096 
02097   BranchProbabilityInfo *BPI = FuncInfo.BPI;
02098   // If any two of the cases has the same destination, and if one value
02099   // is the same as the other, but has one bit unset that the other has set,
02100   // use bit manipulation to do two compares at once.  For example:
02101   // "if (X == 6 || X == 4)" -> "if ((X|2) == 6)"
02102   // TODO: This could be extended to merge any 2 cases in switches with 3 cases.
02103   // TODO: Handle cases where CR.CaseBB != SwitchBB.
02104   if (Size == 2 && CR.CaseBB == SwitchBB) {
02105     Case &Small = *CR.Range.first;
02106     Case &Big = *(CR.Range.second-1);
02107 
02108     if (Small.Low == Small.High && Big.Low == Big.High && Small.BB == Big.BB) {
02109       const APInt& SmallValue = cast<ConstantInt>(Small.Low)->getValue();
02110       const APInt& BigValue = cast<ConstantInt>(Big.Low)->getValue();
02111 
02112       // Check that there is only one bit different.
02113       if (BigValue.countPopulation() == SmallValue.countPopulation() + 1 &&
02114           (SmallValue | BigValue) == BigValue) {
02115         // Isolate the common bit.
02116         APInt CommonBit = BigValue & ~SmallValue;
02117         assert((SmallValue | CommonBit) == BigValue &&
02118                CommonBit.countPopulation() == 1 && "Not a common bit?");
02119 
02120         SDValue CondLHS = getValue(SV);
02121         EVT VT = CondLHS.getValueType();
02122         SDLoc DL = getCurSDLoc();
02123 
02124         SDValue Or = DAG.getNode(ISD::OR, DL, VT, CondLHS,
02125                                  DAG.getConstant(CommonBit, VT));
02126         SDValue Cond = DAG.getSetCC(DL, MVT::i1,
02127                                     Or, DAG.getConstant(BigValue, VT),
02128                                     ISD::SETEQ);
02129 
02130         // Update successor info.
02131         // Both Small and Big will jump to Small.BB, so we sum up the weights.
02132         addSuccessorWithWeight(SwitchBB, Small.BB,
02133                                Small.ExtraWeight + Big.ExtraWeight);
02134         addSuccessorWithWeight(SwitchBB, Default,
02135           // The default destination is the first successor in IR.
02136           BPI ? BPI->getEdgeWeight(SwitchBB->getBasicBlock(), (unsigned)0) : 0);
02137 
02138         // Insert the true branch.
02139         SDValue BrCond = DAG.getNode(ISD::BRCOND, DL, MVT::Other,
02140                                      getControlRoot(), Cond,
02141                                      DAG.getBasicBlock(Small.BB));
02142 
02143         // Insert the false branch.
02144         BrCond = DAG.getNode(ISD::BR, DL, MVT::Other, BrCond,
02145                              DAG.getBasicBlock(Default));
02146 
02147         DAG.setRoot(BrCond);
02148         return true;
02149       }
02150     }
02151   }
02152 
02153   // Order cases by weight so the most likely case will be checked first.
02154   uint32_t UnhandledWeights = 0;
02155   if (BPI) {
02156     for (CaseItr I = CR.Range.first, IE = CR.Range.second; I != IE; ++I) {
02157       uint32_t IWeight = I->ExtraWeight;
02158       UnhandledWeights += IWeight;
02159       for (CaseItr J = CR.Range.first; J < I; ++J) {
02160         uint32_t JWeight = J->ExtraWeight;
02161         if (IWeight > JWeight)
02162           std::swap(*I, *J);
02163       }
02164     }
02165   }
02166   // Rearrange the case blocks so that the last one falls through if possible.
02167   Case &BackCase = *(CR.Range.second-1);
02168   if (Size > 1 &&
02169       NextBlock && Default != NextBlock && BackCase.BB != NextBlock) {
02170     // The last case block won't fall through into 'NextBlock' if we emit the
02171     // branches in this order.  See if rearranging a case value would help.
02172     // We start at the bottom as it's the case with the least weight.
02173     for (Case *I = &*(CR.Range.second-2), *E = &*CR.Range.first-1; I != E; --I)
02174       if (I->BB == NextBlock) {
02175         std::swap(*I, BackCase);
02176         break;
02177       }
02178   }
02179 
02180   // Create a CaseBlock record representing a conditional branch to
02181   // the Case's target mbb if the value being switched on SV is equal
02182   // to C.
02183   MachineBasicBlock *CurBlock = CR.CaseBB;
02184   for (CaseItr I = CR.Range.first, E = CR.Range.second; I != E; ++I) {
02185     MachineBasicBlock *FallThrough;
02186     if (I != E-1) {
02187       FallThrough = CurMF->CreateMachineBasicBlock(CurBlock->getBasicBlock());
02188       CurMF->insert(BBI, FallThrough);
02189 
02190       // Put SV in a virtual register to make it available from the new blocks.
02191       ExportFromCurrentBlock(SV);
02192     } else {
02193       // If the last case doesn't match, go to the default block.
02194       FallThrough = Default;
02195     }
02196 
02197     const Value *RHS, *LHS, *MHS;
02198     ISD::CondCode CC;
02199     if (I->High == I->Low) {
02200       // This is just small small case range :) containing exactly 1 case
02201       CC = ISD::SETEQ;
02202       LHS = SV; RHS = I->High; MHS = nullptr;
02203     } else {
02204       CC = ISD::SETLE;
02205       LHS = I->Low; MHS = SV; RHS = I->High;
02206     }
02207 
02208     // The false weight should be sum of all un-handled cases.
02209     UnhandledWeights -= I->ExtraWeight;
02210     CaseBlock CB(CC, LHS, RHS, MHS, /* truebb */ I->BB, /* falsebb */ FallThrough,
02211                  /* me */ CurBlock,
02212                  /* trueweight */ I->ExtraWeight,
02213                  /* falseweight */ UnhandledWeights);
02214 
02215     // If emitting the first comparison, just call visitSwitchCase to emit the
02216     // code into the current block.  Otherwise, push the CaseBlock onto the
02217     // vector to be later processed by SDISel, and insert the node's MBB
02218     // before the next MBB.
02219     if (CurBlock == SwitchBB)
02220       visitSwitchCase(CB, SwitchBB);
02221     else
02222       SwitchCases.push_back(CB);
02223 
02224     CurBlock = FallThrough;
02225   }
02226 
02227   return true;
02228 }
02229 
02230 static inline bool areJTsAllowed(const TargetLowering &TLI) {
02231   return TLI.supportJumpTables() &&
02232           (TLI.isOperationLegalOrCustom(ISD::BR_JT, MVT::Other) ||
02233            TLI.isOperationLegalOrCustom(ISD::BRIND, MVT::Other));
02234 }
02235 
02236 static APInt ComputeRange(const APInt &First, const APInt &Last) {
02237   uint32_t BitWidth = std::max(Last.getBitWidth(), First.getBitWidth()) + 1;
02238   APInt LastExt = Last.sext(BitWidth), FirstExt = First.sext(BitWidth);
02239   return (LastExt - FirstExt + 1ULL);
02240 }
02241 
02242 /// handleJTSwitchCase - Emit jumptable for current switch case range
02243 bool SelectionDAGBuilder::handleJTSwitchCase(CaseRec &CR,
02244                                              CaseRecVector &WorkList,
02245                                              const Value *SV,
02246                                              MachineBasicBlock *Default,
02247                                              MachineBasicBlock *SwitchBB) {
02248   Case& FrontCase = *CR.Range.first;
02249   Case& BackCase  = *(CR.Range.second-1);
02250 
02251   const APInt &First = cast<ConstantInt>(FrontCase.Low)->getValue();
02252   const APInt &Last  = cast<ConstantInt>(BackCase.High)->getValue();
02253 
02254   APInt TSize(First.getBitWidth(), 0);
02255   for (CaseItr I = CR.Range.first, E = CR.Range.second; I != E; ++I)
02256     TSize += I->size();
02257 
02258   const TargetLowering *TLI = TM.getTargetLowering();
02259   if (!areJTsAllowed(*TLI) || TSize.ult(TLI->getMinimumJumpTableEntries()))
02260     return false;
02261 
02262   APInt Range = ComputeRange(First, Last);
02263   // The density is TSize / Range. Require at least 40%.
02264   // It should not be possible for IntTSize to saturate for sane code, but make
02265   // sure we handle Range saturation correctly.
02266   uint64_t IntRange = Range.getLimitedValue(UINT64_MAX/10);
02267   uint64_t IntTSize = TSize.getLimitedValue(UINT64_MAX/10);
02268   if (IntTSize * 10 < IntRange * 4)
02269     return false;
02270 
02271   DEBUG(dbgs() << "Lowering jump table\n"
02272                << "First entry: " << First << ". Last entry: " << Last << '\n'
02273                << "Range: " << Range << ". Size: " << TSize << ".\n\n");
02274 
02275   // Get the MachineFunction which holds the current MBB.  This is used when
02276   // inserting any additional MBBs necessary to represent the switch.
02277   MachineFunction *CurMF = FuncInfo.MF;
02278 
02279   // Figure out which block is immediately after the current one.
02280   MachineFunction::iterator BBI = CR.CaseBB;
02281   ++BBI;
02282 
02283   const BasicBlock *LLVMBB = CR.CaseBB->getBasicBlock();
02284 
02285   // Create a new basic block to hold the code for loading the address
02286   // of the jump table, and jumping to it.  Update successor information;
02287   // we will either branch to the default case for the switch, or the jump
02288   // table.
02289   MachineBasicBlock *JumpTableBB = CurMF->CreateMachineBasicBlock(LLVMBB);
02290   CurMF->insert(BBI, JumpTableBB);
02291 
02292   addSuccessorWithWeight(CR.CaseBB, Default);
02293   addSuccessorWithWeight(CR.CaseBB, JumpTableBB);
02294 
02295   // Build a vector of destination BBs, corresponding to each target
02296   // of the jump table. If the value of the jump table slot corresponds to
02297   // a case statement, push the case's BB onto the vector, otherwise, push
02298   // the default BB.
02299   std::vector<MachineBasicBlock*> DestBBs;
02300   APInt TEI = First;
02301   for (CaseItr I = CR.Range.first, E = CR.Range.second; I != E; ++TEI) {
02302     const APInt &Low = cast<ConstantInt>(I->Low)->getValue();
02303     const APInt &High = cast<ConstantInt>(I->High)->getValue();
02304 
02305     if (Low.sle(TEI) && TEI.sle(High)) {
02306       DestBBs.push_back(I->BB);
02307       if (TEI==High)
02308         ++I;
02309     } else {
02310       DestBBs.push_back(Default);
02311     }
02312   }
02313 
02314   // Calculate weight for each unique destination in CR.
02315   DenseMap<MachineBasicBlock*, uint32_t> DestWeights;
02316   if (FuncInfo.BPI)
02317     for (CaseItr I = CR.Range.first, E = CR.Range.second; I != E; ++I) {
02318       DenseMap<MachineBasicBlock*, uint32_t>::iterator Itr =
02319           DestWeights.find(I->BB);
02320       if (Itr != DestWeights.end())
02321         Itr->second += I->ExtraWeight;
02322       else
02323         DestWeights[I->BB] = I->ExtraWeight;
02324     }
02325 
02326   // Update successor info. Add one edge to each unique successor.
02327   BitVector SuccsHandled(CR.CaseBB->getParent()->getNumBlockIDs());
02328   for (std::vector<MachineBasicBlock*>::iterator I = DestBBs.begin(),
02329          E = DestBBs.end(); I != E; ++I) {
02330     if (!SuccsHandled[(*I)->getNumber()]) {
02331       SuccsHandled[(*I)->getNumber()] = true;
02332       DenseMap<MachineBasicBlock*, uint32_t>::iterator Itr =
02333           DestWeights.find(*I);
02334       addSuccessorWithWeight(JumpTableBB, *I,
02335                              Itr != DestWeights.end() ? Itr->second : 0);
02336     }
02337   }
02338 
02339   // Create a jump table index for this jump table.
02340   unsigned JTEncoding = TLI->getJumpTableEncoding();
02341   unsigned JTI = CurMF->getOrCreateJumpTableInfo(JTEncoding)
02342                        ->createJumpTableIndex(DestBBs);
02343 
02344   // Set the jump table information so that we can codegen it as a second
02345   // MachineBasicBlock
02346   JumpTable JT(-1U, JTI, JumpTableBB, Default);
02347   JumpTableHeader JTH(First, Last, SV, CR.CaseBB, (CR.CaseBB == SwitchBB));
02348   if (CR.CaseBB == SwitchBB)
02349     visitJumpTableHeader(JT, JTH, SwitchBB);
02350 
02351   JTCases.push_back(JumpTableBlock(JTH, JT));
02352   return true;
02353 }
02354 
02355 /// handleBTSplitSwitchCase - emit comparison and split binary search tree into
02356 /// 2 subtrees.
02357 bool SelectionDAGBuilder::handleBTSplitSwitchCase(CaseRec& CR,
02358                                                   CaseRecVector& WorkList,
02359                                                   const Value* SV,
02360                                                   MachineBasicBlock* Default,
02361                                                   MachineBasicBlock* SwitchBB) {
02362   // Get the MachineFunction which holds the current MBB.  This is used when
02363   // inserting any additional MBBs necessary to represent the switch.
02364   MachineFunction *CurMF = FuncInfo.MF;
02365 
02366   // Figure out which block is immediately after the current one.
02367   MachineFunction::iterator BBI = CR.CaseBB;
02368   ++BBI;
02369 
02370   Case& FrontCase = *CR.Range.first;
02371   Case& BackCase  = *(CR.Range.second-1);
02372   const BasicBlock *LLVMBB = CR.CaseBB->getBasicBlock();
02373 
02374   // Size is the number of Cases represented by this range.
02375   unsigned Size = CR.Range.second - CR.Range.first;
02376 
02377   const APInt &First = cast<ConstantInt>(FrontCase.Low)->getValue();
02378   const APInt &Last  = cast<ConstantInt>(BackCase.High)->getValue();
02379   double FMetric = 0;
02380   CaseItr Pivot = CR.Range.first + Size/2;
02381 
02382   // Select optimal pivot, maximizing sum density of LHS and RHS. This will
02383   // (heuristically) allow us to emit JumpTable's later.
02384   APInt TSize(First.getBitWidth(), 0);
02385   for (CaseItr I = CR.Range.first, E = CR.Range.second;
02386        I!=E; ++I)
02387     TSize += I->size();
02388 
02389   APInt LSize = FrontCase.size();
02390   APInt RSize = TSize-LSize;
02391   DEBUG(dbgs() << "Selecting best pivot: \n"
02392                << "First: " << First << ", Last: " << Last <<'\n'
02393                << "LSize: " << LSize << ", RSize: " << RSize << '\n');
02394   for (CaseItr I = CR.Range.first, J=I+1, E = CR.Range.second;
02395        J!=E; ++I, ++J) {
02396     const APInt &LEnd = cast<ConstantInt>(I->High)->getValue();
02397     const APInt &RBegin = cast<ConstantInt>(J->Low)->getValue();
02398     APInt Range = ComputeRange(LEnd, RBegin);
02399     assert((Range - 2ULL).isNonNegative() &&
02400            "Invalid case distance");
02401     // Use volatile double here to avoid excess precision issues on some hosts,
02402     // e.g. that use 80-bit X87 registers.
02403     volatile double LDensity =
02404        (double)LSize.roundToDouble() /
02405                            (LEnd - First + 1ULL).roundToDouble();
02406     volatile double RDensity =
02407       (double)RSize.roundToDouble() /
02408                            (Last - RBegin + 1ULL).roundToDouble();
02409     volatile double Metric = Range.logBase2()*(LDensity+RDensity);
02410     // Should always split in some non-trivial place
02411     DEBUG(dbgs() <<"=>Step\n"
02412                  << "LEnd: " << LEnd << ", RBegin: " << RBegin << '\n'
02413                  << "LDensity: " << LDensity
02414                  << ", RDensity: " << RDensity << '\n'
02415                  << "Metric: " << Metric << '\n');
02416     if (FMetric < Metric) {
02417       Pivot = J;
02418       FMetric = Metric;
02419       DEBUG(dbgs() << "Current metric set to: " << FMetric << '\n');
02420     }
02421 
02422     LSize += J->size();
02423     RSize -= J->size();
02424   }
02425 
02426   const TargetLowering *TLI = TM.getTargetLowering();
02427   if (areJTsAllowed(*TLI)) {
02428     // If our case is dense we *really* should handle it earlier!
02429     assert((FMetric > 0) && "Should handle dense range earlier!");
02430   } else {
02431     Pivot = CR.Range.first + Size/2;
02432   }
02433 
02434   CaseRange LHSR(CR.Range.first, Pivot);
02435   CaseRange RHSR(Pivot, CR.Range.second);
02436   const Constant *C = Pivot->Low;
02437   MachineBasicBlock *FalseBB = nullptr, *TrueBB = nullptr;
02438 
02439   // We know that we branch to the LHS if the Value being switched on is
02440   // less than the Pivot value, C.  We use this to optimize our binary
02441   // tree a bit, by recognizing that if SV is greater than or equal to the
02442   // LHS's Case Value, and that Case Value is exactly one less than the
02443   // Pivot's Value, then we can branch directly to the LHS's Target,
02444   // rather than creating a leaf node for it.
02445   if ((LHSR.second - LHSR.first) == 1 &&
02446       LHSR.first->High == CR.GE &&
02447       cast<ConstantInt>(C)->getValue() ==
02448       (cast<ConstantInt>(CR.GE)->getValue() + 1LL)) {
02449     TrueBB = LHSR.first->BB;
02450   } else {
02451     TrueBB = CurMF->CreateMachineBasicBlock(LLVMBB);
02452     CurMF->insert(BBI, TrueBB);
02453     WorkList.push_back(CaseRec(TrueBB, C, CR.GE, LHSR));
02454 
02455     // Put SV in a virtual register to make it available from the new blocks.
02456     ExportFromCurrentBlock(SV);
02457   }
02458 
02459   // Similar to the optimization above, if the Value being switched on is
02460   // known to be less than the Constant CR.LT, and the current Case Value
02461   // is CR.LT - 1, then we can branch directly to the target block for
02462   // the current Case Value, rather than emitting a RHS leaf node for it.
02463   if ((RHSR.second - RHSR.first) == 1 && CR.LT &&
02464       cast<ConstantInt>(RHSR.first->Low)->getValue() ==
02465       (cast<ConstantInt>(CR.LT)->getValue() - 1LL)) {
02466     FalseBB = RHSR.first->BB;
02467   } else {
02468     FalseBB = CurMF->CreateMachineBasicBlock(LLVMBB);
02469     CurMF->insert(BBI, FalseBB);
02470     WorkList.push_back(CaseRec(FalseBB,CR.LT,C,RHSR));
02471 
02472     // Put SV in a virtual register to make it available from the new blocks.
02473     ExportFromCurrentBlock(SV);
02474   }
02475 
02476   // Create a CaseBlock record representing a conditional branch to
02477   // the LHS node if the value being switched on SV is less than C.
02478   // Otherwise, branch to LHS.
02479   CaseBlock CB(ISD::SETLT, SV, C, nullptr, TrueBB, FalseBB, CR.CaseBB);
02480 
02481   if (CR.CaseBB == SwitchBB)
02482     visitSwitchCase(CB, SwitchBB);
02483   else
02484     SwitchCases.push_back(CB);
02485 
02486   return true;
02487 }
02488 
02489 /// handleBitTestsSwitchCase - if current case range has few destination and
02490 /// range span less, than machine word bitwidth, encode case range into series
02491 /// of masks and emit bit tests with these masks.
02492 bool SelectionDAGBuilder::handleBitTestsSwitchCase(CaseRec& CR,
02493                                                    CaseRecVector& WorkList,
02494                                                    const Value* SV,
02495                                                    MachineBasicBlock* Default,
02496                                                    MachineBasicBlock* SwitchBB) {
02497   const TargetLowering *TLI = TM.getTargetLowering();
02498   EVT PTy = TLI->getPointerTy();
02499   unsigned IntPtrBits = PTy.getSizeInBits();
02500 
02501   Case& FrontCase = *CR.Range.first;
02502   Case& BackCase  = *(CR.Range.second-1);
02503 
02504   // Get the MachineFunction which holds the current MBB.  This is used when
02505   // inserting any additional MBBs necessary to represent the switch.
02506   MachineFunction *CurMF = FuncInfo.MF;
02507 
02508   // If target does not have legal shift left, do not emit bit tests at all.
02509   if (!TLI->isOperationLegal(ISD::SHL, PTy))
02510     return false;
02511 
02512   size_t numCmps = 0;
02513   for (CaseItr I = CR.Range.first, E = CR.Range.second;
02514        I!=E; ++I) {
02515     // Single case counts one, case range - two.
02516     numCmps += (I->Low == I->High ? 1 : 2);
02517   }
02518 
02519   // Count unique destinations
02520   SmallSet<MachineBasicBlock*, 4> Dests;
02521   for (CaseItr I = CR.Range.first, E = CR.Range.second; I!=E; ++I) {
02522     Dests.insert(I->BB);
02523     if (Dests.size() > 3)
02524       // Don't bother the code below, if there are too much unique destinations
02525       return false;
02526   }
02527   DEBUG(dbgs() << "Total number of unique destinations: "
02528         << Dests.size() << '\n'
02529         << "Total number of comparisons: " << numCmps << '\n');
02530 
02531   // Compute span of values.
02532   const APInt& minValue = cast<ConstantInt>(FrontCase.Low)->getValue();
02533   const APInt& maxValue = cast<ConstantInt>(BackCase.High)->getValue();
02534   APInt cmpRange = maxValue - minValue;
02535 
02536   DEBUG(dbgs() << "Compare range: " << cmpRange << '\n'
02537                << "Low bound: " << minValue << '\n'
02538                << "High bound: " << maxValue << '\n');
02539 
02540   if (cmpRange.uge(IntPtrBits) ||
02541       (!(Dests.size() == 1 && numCmps >= 3) &&
02542        !(Dests.size() == 2 && numCmps >= 5) &&
02543        !(Dests.size() >= 3 && numCmps >= 6)))
02544     return false;
02545 
02546   DEBUG(dbgs() << "Emitting bit tests\n");
02547   APInt lowBound = APInt::getNullValue(cmpRange.getBitWidth());
02548 
02549   // Optimize the case where all the case values fit in a
02550   // word without having to subtract minValue. In this case,
02551   // we can optimize away the subtraction.
02552   if (minValue.isNonNegative() && maxValue.slt(IntPtrBits)) {
02553     cmpRange = maxValue;
02554   } else {
02555     lowBound = minValue;
02556   }
02557 
02558   CaseBitsVector CasesBits;
02559   unsigned i, count = 0;
02560 
02561   for (CaseItr I = CR.Range.first, E = CR.Range.second; I!=E; ++I) {
02562     MachineBasicBlock* Dest = I->BB;
02563     for (i = 0; i < count; ++i)
02564       if (Dest == CasesBits[i].BB)
02565         break;
02566 
02567     if (i == count) {
02568       assert((count < 3) && "Too much destinations to test!");
02569       CasesBits.push_back(CaseBits(0, Dest, 0, 0/*Weight*/));
02570       count++;
02571     }
02572 
02573     const APInt& lowValue = cast<ConstantInt>(I->Low)->getValue();
02574     const APInt& highValue = cast<ConstantInt>(I->High)->getValue();
02575 
02576     uint64_t lo = (lowValue - lowBound).getZExtValue();
02577     uint64_t hi = (highValue - lowBound).getZExtValue();
02578     CasesBits[i].ExtraWeight += I->ExtraWeight;
02579 
02580     for (uint64_t j = lo; j <= hi; j++) {
02581       CasesBits[i].Mask |=  1ULL << j;
02582       CasesBits[i].Bits++;
02583     }
02584 
02585   }
02586   std::sort(CasesBits.begin(), CasesBits.end(), CaseBitsCmp());
02587 
02588   BitTestInfo BTC;
02589 
02590   // Figure out which block is immediately after the current one.
02591   MachineFunction::iterator BBI = CR.CaseBB;
02592   ++BBI;
02593 
02594   const BasicBlock *LLVMBB = CR.CaseBB->getBasicBlock();
02595 
02596   DEBUG(dbgs() << "Cases:\n");
02597   for (unsigned i = 0, e = CasesBits.size(); i!=e; ++i) {
02598     DEBUG(dbgs() << "Mask: " << CasesBits[i].Mask
02599                  << ", Bits: " << CasesBits[i].Bits
02600                  << ", BB: " << CasesBits[i].BB << '\n');
02601 
02602     MachineBasicBlock *CaseBB = CurMF->CreateMachineBasicBlock(LLVMBB);
02603     CurMF->insert(BBI, CaseBB);
02604     BTC.push_back(BitTestCase(CasesBits[i].Mask,
02605                               CaseBB,
02606                               CasesBits[i].BB, CasesBits[i].ExtraWeight));
02607 
02608     // Put SV in a virtual register to make it available from the new blocks.
02609     ExportFromCurrentBlock(SV);
02610   }
02611 
02612   BitTestBlock BTB(lowBound, cmpRange, SV,
02613                    -1U, MVT::Other, (CR.CaseBB == SwitchBB),
02614                    CR.CaseBB, Default, BTC);
02615 
02616   if (CR.CaseBB == SwitchBB)
02617     visitBitTestHeader(BTB, SwitchBB);
02618 
02619   BitTestCases.push_back(BTB);
02620 
02621   return true;
02622 }
02623 
02624 /// Clusterify - Transform simple list of Cases into list of CaseRange's
02625 size_t SelectionDAGBuilder::Clusterify(CaseVector& Cases,
02626                                        const SwitchInst& SI) {
02627   size_t numCmps = 0;
02628 
02629   BranchProbabilityInfo *BPI = FuncInfo.BPI;
02630   // Start with "simple" cases
02631   for (SwitchInst::ConstCaseIt i = SI.case_begin(), e = SI.case_end();
02632        i != e; ++i) {
02633     const BasicBlock *SuccBB = i.getCaseSuccessor();
02634     MachineBasicBlock *SMBB = FuncInfo.MBBMap[SuccBB];
02635 
02636     uint32_t ExtraWeight =
02637       BPI ? BPI->getEdgeWeight(SI.getParent(), i.getSuccessorIndex()) : 0;
02638 
02639     Cases.push_back(Case(i.getCaseValue(), i.getCaseValue(),
02640                          SMBB, ExtraWeight));
02641   }
02642   std::sort(Cases.begin(), Cases.end(), CaseCmp());
02643 
02644   // Merge case into clusters
02645   if (Cases.size() >= 2)
02646     // Must recompute end() each iteration because it may be
02647     // invalidated by erase if we hold on to it
02648     for (CaseItr I = Cases.begin(), J = std::next(Cases.begin());
02649          J != Cases.end(); ) {
02650       const APInt& nextValue = cast<ConstantInt>(J->Low)->getValue();
02651       const APInt& currentValue = cast<ConstantInt>(I->High)->getValue();
02652       MachineBasicBlock* nextBB = J->BB;
02653       MachineBasicBlock* currentBB = I->BB;
02654 
02655       // If the two neighboring cases go to the same destination, merge them
02656       // into a single case.
02657       if ((nextValue - currentValue == 1) && (currentBB == nextBB)) {
02658         I->High = J->High;
02659         I->ExtraWeight += J->ExtraWeight;
02660         J = Cases.erase(J);
02661       } else {
02662         I = J++;
02663       }
02664     }
02665 
02666   for (CaseItr I=Cases.begin(), E=Cases.end(); I!=E; ++I, ++numCmps) {
02667     if (I->Low != I->High)
02668       // A range counts double, since it requires two compares.
02669       ++numCmps;
02670   }
02671 
02672   return numCmps;
02673 }
02674 
02675 void SelectionDAGBuilder::UpdateSplitBlock(MachineBasicBlock *First,
02676                                            MachineBasicBlock *Last) {
02677   // Update JTCases.
02678   for (unsigned i = 0, e = JTCases.size(); i != e; ++i)
02679     if (JTCases[i].first.HeaderBB == First)
02680       JTCases[i].first.HeaderBB = Last;
02681 
02682   // Update BitTestCases.
02683   for (unsigned i = 0, e = BitTestCases.size(); i != e; ++i)
02684     if (BitTestCases[i].Parent == First)
02685       BitTestCases[i].Parent = Last;
02686 }
02687 
02688 void SelectionDAGBuilder::visitSwitch(const SwitchInst &SI) {
02689   MachineBasicBlock *SwitchMBB = FuncInfo.MBB;
02690 
02691   // Figure out which block is immediately after the current one.
02692   MachineBasicBlock *NextBlock = nullptr;
02693   MachineBasicBlock *Default = FuncInfo.MBBMap[SI.getDefaultDest()];
02694 
02695   // If there is only the default destination, branch to it if it is not the
02696   // next basic block.  Otherwise, just fall through.
02697   if (!SI.getNumCases()) {
02698     // Update machine-CFG edges.
02699 
02700     // If this is not a fall-through branch, emit the branch.
02701     SwitchMBB->addSuccessor(Default);
02702     if (Default != NextBlock)
02703       DAG.setRoot(DAG.getNode(ISD::BR, getCurSDLoc(),
02704                               MVT::Other, getControlRoot(),
02705                               DAG.getBasicBlock(Default)));
02706 
02707     return;
02708   }
02709 
02710   // If there are any non-default case statements, create a vector of Cases
02711   // representing each one, and sort the vector so that we can efficiently
02712   // create a binary search tree from them.
02713   CaseVector Cases;
02714   size_t numCmps = Clusterify(Cases, SI);
02715   DEBUG(dbgs() << "Clusterify finished. Total clusters: " << Cases.size()
02716                << ". Total compares: " << numCmps << '\n');
02717   (void)numCmps;
02718 
02719   // Get the Value to be switched on and default basic blocks, which will be
02720   // inserted into CaseBlock records, representing basic blocks in the binary
02721   // search tree.
02722   const Value *SV = SI.getCondition();
02723 
02724   // Push the initial CaseRec onto the worklist
02725   CaseRecVector WorkList;
02726   WorkList.push_back(CaseRec(SwitchMBB,nullptr,nullptr,
02727                              CaseRange(Cases.begin(),Cases.end())));
02728 
02729   while (!WorkList.empty()) {
02730     // Grab a record representing a case range to process off the worklist
02731     CaseRec CR = WorkList.back();
02732     WorkList.pop_back();
02733 
02734     if (handleBitTestsSwitchCase(CR, WorkList, SV, Default, SwitchMBB))
02735       continue;
02736 
02737     // If the range has few cases (two or less) emit a series of specific
02738     // tests.
02739     if (handleSmallSwitchRange(CR, WorkList, SV, Default, SwitchMBB))
02740       continue;
02741 
02742     // If the switch has more than N blocks, and is at least 40% dense, and the
02743     // target supports indirect branches, then emit a jump table rather than
02744     // lowering the switch to a binary tree of conditional branches.
02745     // N defaults to 4 and is controlled via TLS.getMinimumJumpTableEntries().
02746     if (handleJTSwitchCase(CR, WorkList, SV, Default, SwitchMBB))
02747       continue;
02748 
02749     // Emit binary tree. We need to pick a pivot, and push left and right ranges
02750     // onto the worklist. Leafs are handled via handleSmallSwitchRange() call.
02751     handleBTSplitSwitchCase(CR, WorkList, SV, Default, SwitchMBB);
02752   }
02753 }
02754 
02755 void SelectionDAGBuilder::visitIndirectBr(const IndirectBrInst &I) {
02756   MachineBasicBlock *IndirectBrMBB = FuncInfo.MBB;
02757 
02758   // Update machine-CFG edges with unique successors.
02759   SmallSet<BasicBlock*, 32> Done;
02760   for (unsigned i = 0, e = I.getNumSuccessors(); i != e; ++i) {
02761     BasicBlock *BB = I.getSuccessor(i);
02762     bool Inserted = Done.insert(BB);
02763     if (!Inserted)
02764         continue;
02765 
02766     MachineBasicBlock *Succ = FuncInfo.MBBMap[BB];
02767     addSuccessorWithWeight(IndirectBrMBB, Succ);
02768   }
02769 
02770   DAG.setRoot(DAG.getNode(ISD::BRIND, getCurSDLoc(),
02771                           MVT::Other, getControlRoot(),
02772                           getValue(I.getAddress())));
02773 }
02774 
02775 void SelectionDAGBuilder::visitUnreachable(const UnreachableInst &I) {
02776   if (DAG.getTarget().Options.TrapUnreachable)
02777     DAG.setRoot(DAG.getNode(ISD::TRAP, getCurSDLoc(), MVT::Other, DAG.getRoot()));
02778 }
02779 
02780 void SelectionDAGBuilder::visitFSub(const User &I) {
02781   // -0.0 - X --> fneg
02782   Type *Ty = I.getType();
02783   if (isa<Constant>(I.getOperand(0)) &&
02784       I.getOperand(0) == ConstantFP::getZeroValueForNegation(Ty)) {
02785     SDValue Op2 = getValue(I.getOperand(1));
02786     setValue(&I, DAG.getNode(ISD::FNEG, getCurSDLoc(),
02787                              Op2.getValueType(), Op2));
02788     return;
02789   }
02790 
02791   visitBinary(I, ISD::FSUB);
02792 }
02793 
02794 void SelectionDAGBuilder::visitBinary(const User &I, unsigned OpCode) {
02795   SDValue Op1 = getValue(I.getOperand(0));
02796   SDValue Op2 = getValue(I.getOperand(1));
02797 
02798   bool nuw = false;
02799   bool nsw = false;
02800   bool exact = false;
02801   if (const OverflowingBinaryOperator *OFBinOp =
02802           dyn_cast<const OverflowingBinaryOperator>(&I)) {
02803     nuw = OFBinOp->hasNoUnsignedWrap();
02804     nsw = OFBinOp->hasNoSignedWrap();
02805   }
02806   if (const PossiblyExactOperator *ExactOp =
02807           dyn_cast<const PossiblyExactOperator>(&I))
02808     exact = ExactOp->isExact();
02809 
02810   SDValue BinNodeValue = DAG.getNode(OpCode, getCurSDLoc(), Op1.getValueType(),
02811                                      Op1, Op2, nuw, nsw, exact);
02812   setValue(&I, BinNodeValue);
02813 }
02814 
02815 void SelectionDAGBuilder::visitShift(const User &I, unsigned Opcode) {
02816   SDValue Op1 = getValue(I.getOperand(0));
02817   SDValue Op2 = getValue(I.getOperand(1));
02818 
02819   EVT ShiftTy = TM.getTargetLowering()->getShiftAmountTy(Op2.getValueType());
02820 
02821   // Coerce the shift amount to the right type if we can.
02822   if (!I.getType()->isVectorTy() && Op2.getValueType() != ShiftTy) {
02823     unsigned ShiftSize = ShiftTy.getSizeInBits();
02824     unsigned Op2Size = Op2.getValueType().getSizeInBits();
02825     SDLoc DL = getCurSDLoc();
02826 
02827     // If the operand is smaller than the shift count type, promote it.
02828     if (ShiftSize > Op2Size)
02829       Op2 = DAG.getNode(ISD::ZERO_EXTEND, DL, ShiftTy, Op2);
02830 
02831     // If the operand is larger than the shift count type but the shift
02832     // count type has enough bits to represent any shift value, truncate
02833     // it now. This is a common case and it exposes the truncate to
02834     // optimization early.
02835     else if (ShiftSize >= Log2_32_Ceil(Op2.getValueType().getSizeInBits()))
02836       Op2 = DAG.getNode(ISD::TRUNCATE, DL, ShiftTy, Op2);
02837     // Otherwise we'll need to temporarily settle for some other convenient
02838     // type.  Type legalization will make adjustments once the shiftee is split.
02839     else
02840       Op2 = DAG.getZExtOrTrunc(Op2, DL, MVT::i32);
02841   }
02842 
02843   bool nuw = false;
02844   bool nsw = false;
02845   bool exact = false;
02846 
02847   if (Opcode == ISD::SRL || Opcode == ISD::SRA || Opcode == ISD::SHL) {
02848 
02849     if (const OverflowingBinaryOperator *OFBinOp =
02850             dyn_cast<const OverflowingBinaryOperator>(&I)) {
02851       nuw = OFBinOp->hasNoUnsignedWrap();
02852       nsw = OFBinOp->hasNoSignedWrap();
02853     }
02854     if (const PossiblyExactOperator *ExactOp =
02855             dyn_cast<const PossiblyExactOperator>(&I))
02856       exact = ExactOp->isExact();
02857   }
02858 
02859   SDValue Res = DAG.getNode(Opcode, getCurSDLoc(), Op1.getValueType(), Op1, Op2,
02860                             nuw, nsw, exact);
02861   setValue(&I, Res);
02862 }
02863 
02864 void SelectionDAGBuilder::visitSDiv(const User &I) {
02865   SDValue Op1 = getValue(I.getOperand(0));
02866   SDValue Op2 = getValue(I.getOperand(1));
02867 
02868   // Turn exact SDivs into multiplications.
02869   // FIXME: This should be in DAGCombiner, but it doesn't have access to the
02870   // exact bit.
02871   if (isa<BinaryOperator>(&I) && cast<BinaryOperator>(&I)->isExact() &&
02872       !isa<ConstantSDNode>(Op1) &&
02873       isa<ConstantSDNode>(Op2) && !cast<ConstantSDNode>(Op2)->isNullValue())
02874     setValue(&I, TM.getTargetLowering()->BuildExactSDIV(Op1, Op2,
02875                                                         getCurSDLoc(), DAG));
02876   else
02877     setValue(&I, DAG.getNode(ISD::SDIV, getCurSDLoc(), Op1.getValueType(),
02878                              Op1, Op2));
02879 }
02880 
02881 void SelectionDAGBuilder::visitICmp(const User &I) {
02882   ICmpInst::Predicate predicate = ICmpInst::BAD_ICMP_PREDICATE;
02883   if (const ICmpInst *IC = dyn_cast<ICmpInst>(&I))
02884     predicate = IC->getPredicate();
02885   else if (const ConstantExpr *IC = dyn_cast<ConstantExpr>(&I))
02886     predicate = ICmpInst::Predicate(IC->getPredicate());
02887   SDValue Op1 = getValue(I.getOperand(0));
02888   SDValue Op2 = getValue(I.getOperand(1));
02889   ISD::CondCode Opcode = getICmpCondCode(predicate);
02890 
02891   EVT DestVT = TM.getTargetLowering()->getValueType(I.getType());
02892   setValue(&I, DAG.getSetCC(getCurSDLoc(), DestVT, Op1, Op2, Opcode));
02893 }
02894 
02895 void SelectionDAGBuilder::visitFCmp(const User &I) {
02896   FCmpInst::Predicate predicate = FCmpInst::BAD_FCMP_PREDICATE;
02897   if (const FCmpInst *FC = dyn_cast<FCmpInst>(&I))
02898     predicate = FC->getPredicate();
02899   else if (const ConstantExpr *FC = dyn_cast<ConstantExpr>(&I))
02900     predicate = FCmpInst::Predicate(FC->getPredicate());
02901   SDValue Op1 = getValue(I.getOperand(0));
02902   SDValue Op2 = getValue(I.getOperand(1));
02903   ISD::CondCode Condition = getFCmpCondCode(predicate);
02904   if (TM.Options.NoNaNsFPMath)
02905     Condition = getFCmpCodeWithoutNaN(Condition);
02906   EVT DestVT = TM.getTargetLowering()->getValueType(I.getType());
02907   setValue(&I, DAG.getSetCC(getCurSDLoc(), DestVT, Op1, Op2, Condition));
02908 }
02909 
02910 void SelectionDAGBuilder::visitSelect(const User &I) {
02911   SmallVector<EVT, 4> ValueVTs;
02912   ComputeValueVTs(*TM.getTargetLowering(), I.getType(), ValueVTs);
02913   unsigned NumValues = ValueVTs.size();
02914   if (NumValues == 0) return;
02915 
02916   SmallVector<SDValue, 4> Values(NumValues);
02917   SDValue Cond     = getValue(I.getOperand(0));
02918   SDValue TrueVal  = getValue(I.getOperand(1));
02919   SDValue FalseVal = getValue(I.getOperand(2));
02920   ISD::NodeType OpCode = Cond.getValueType().isVector() ?
02921     ISD::VSELECT : ISD::SELECT;
02922 
02923   for (unsigned i = 0; i != NumValues; ++i)
02924     Values[i] = DAG.getNode(OpCode, getCurSDLoc(),
02925                             TrueVal.getNode()->getValueType(TrueVal.getResNo()+i),
02926                             Cond,
02927                             SDValue(TrueVal.getNode(),
02928                                     TrueVal.getResNo() + i),
02929                             SDValue(FalseVal.getNode(),
02930                                     FalseVal.getResNo() + i));
02931 
02932   setValue(&I, DAG.getNode(ISD::MERGE_VALUES, getCurSDLoc(),
02933                            DAG.getVTList(ValueVTs), Values));
02934 }
02935 
02936 void SelectionDAGBuilder::visitTrunc(const User &I) {
02937   // TruncInst cannot be a no-op cast because sizeof(src) > sizeof(dest).
02938   SDValue N = getValue(I.getOperand(0));
02939   EVT DestVT = TM.getTargetLowering()->getValueType(I.getType());
02940   setValue(&I, DAG.getNode(ISD::TRUNCATE, getCurSDLoc(), DestVT, N));
02941 }
02942 
02943 void SelectionDAGBuilder::visitZExt(const User &I) {
02944   // ZExt cannot be a no-op cast because sizeof(src) < sizeof(dest).
02945   // ZExt also can't be a cast to bool for same reason. So, nothing much to do
02946   SDValue N = getValue(I.getOperand(0));
02947   EVT DestVT = TM.getTargetLowering()->getValueType(I.getType());
02948   setValue(&I, DAG.getNode(ISD::ZERO_EXTEND, getCurSDLoc(), DestVT, N));
02949 }
02950 
02951 void SelectionDAGBuilder::visitSExt(const User &I) {
02952   // SExt cannot be a no-op cast because sizeof(src) < sizeof(dest).
02953   // SExt also can't be a cast to bool for same reason. So, nothing much to do
02954   SDValue N = getValue(I.getOperand(0));
02955   EVT DestVT = TM.getTargetLowering()->getValueType(I.getType());
02956   setValue(&I, DAG.getNode(ISD::SIGN_EXTEND, getCurSDLoc(), DestVT, N));
02957 }
02958 
02959 void SelectionDAGBuilder::visitFPTrunc(const User &I) {
02960   // FPTrunc is never a no-op cast, no need to check
02961   SDValue N = getValue(I.getOperand(0));
02962   const TargetLowering *TLI = TM.getTargetLowering();
02963   EVT DestVT = TLI->getValueType(I.getType());
02964   setValue(&I, DAG.getNode(ISD::FP_ROUND, getCurSDLoc(),
02965                            DestVT, N,
02966                            DAG.getTargetConstant(0, TLI->getPointerTy())));
02967 }
02968 
02969 void SelectionDAGBuilder::visitFPExt(const User &I) {
02970   // FPExt is never a no-op cast, no need to check
02971   SDValue N = getValue(I.getOperand(0));
02972   EVT DestVT = TM.getTargetLowering()->getValueType(I.getType());
02973   setValue(&I, DAG.getNode(ISD::FP_EXTEND, getCurSDLoc(), DestVT, N));
02974 }
02975 
02976 void SelectionDAGBuilder::visitFPToUI(const User &I) {
02977   // FPToUI is never a no-op cast, no need to check
02978   SDValue N = getValue(I.getOperand(0));
02979   EVT DestVT = TM.getTargetLowering()->getValueType(I.getType());
02980   setValue(&I, DAG.getNode(ISD::FP_TO_UINT, getCurSDLoc(), DestVT, N));
02981 }
02982 
02983 void SelectionDAGBuilder::visitFPToSI(const User &I) {
02984   // FPToSI is never a no-op cast, no need to check
02985   SDValue N = getValue(I.getOperand(0));
02986   EVT DestVT = TM.getTargetLowering()->getValueType(I.getType());
02987   setValue(&I, DAG.getNode(ISD::FP_TO_SINT, getCurSDLoc(), DestVT, N));
02988 }
02989 
02990 void SelectionDAGBuilder::visitUIToFP(const User &I) {
02991   // UIToFP is never a no-op cast, no need to check
02992   SDValue N = getValue(I.getOperand(0));
02993   EVT DestVT = TM.getTargetLowering()->getValueType(I.getType());
02994   setValue(&I, DAG.getNode(ISD::UINT_TO_FP, getCurSDLoc(), DestVT, N));
02995 }
02996 
02997 void SelectionDAGBuilder::visitSIToFP(const User &I) {
02998   // SIToFP is never a no-op cast, no need to check
02999   SDValue N = getValue(I.getOperand(0));
03000   EVT DestVT = TM.getTargetLowering()->getValueType(I.getType());
03001   setValue(&I, DAG.getNode(ISD::SINT_TO_FP, getCurSDLoc(), DestVT, N));
03002 }
03003 
03004 void SelectionDAGBuilder::visitPtrToInt(const User &I) {
03005   // What to do depends on the size of the integer and the size of the pointer.
03006   // We can either truncate, zero extend, or no-op, accordingly.
03007   SDValue N = getValue(I.getOperand(0));
03008   EVT DestVT = TM.getTargetLowering()->getValueType(I.getType());
03009   setValue(&I, DAG.getZExtOrTrunc(N, getCurSDLoc(), DestVT));
03010 }
03011 
03012 void SelectionDAGBuilder::visitIntToPtr(const User &I) {
03013   // What to do depends on the size of the integer and the size of the pointer.
03014   // We can either truncate, zero extend, or no-op, accordingly.
03015   SDValue N = getValue(I.getOperand(0));
03016   EVT DestVT = TM.getTargetLowering()->getValueType(I.getType());
03017   setValue(&I, DAG.getZExtOrTrunc(N, getCurSDLoc(), DestVT));
03018 }
03019 
03020 void SelectionDAGBuilder::visitBitCast(const User &I) {
03021   SDValue N = getValue(I.getOperand(0));
03022   EVT DestVT = TM.getTargetLowering()->getValueType(I.getType());
03023 
03024   // BitCast assures us that source and destination are the same size so this is
03025   // either a BITCAST or a no-op.
03026   if (DestVT != N.getValueType())
03027     setValue(&I, DAG.getNode(ISD::BITCAST, getCurSDLoc(),
03028                              DestVT, N)); // convert types.
03029   // Check if the original LLVM IR Operand was a ConstantInt, because getValue()
03030   // might fold any kind of constant expression to an integer constant and that
03031   // is not what we are looking for. Only regcognize a bitcast of a genuine
03032   // constant integer as an opaque constant.
03033   else if(ConstantInt *C = dyn_cast<ConstantInt>(I.getOperand(0)))
03034     setValue(&I, DAG.getConstant(C->getValue(), DestVT, /*isTarget=*/false,
03035                                  /*isOpaque*/true));
03036   else
03037     setValue(&I, N);            // noop cast.
03038 }
03039 
03040 void SelectionDAGBuilder::visitAddrSpaceCast(const User &I) {
03041   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
03042   const Value *SV = I.getOperand(0);
03043   SDValue N = getValue(SV);
03044   EVT DestVT = TM.getTargetLowering()->getValueType(I.getType());
03045 
03046   unsigned SrcAS = SV->getType()->getPointerAddressSpace();
03047   unsigned DestAS = I.getType()->getPointerAddressSpace();
03048 
03049   if (!TLI.isNoopAddrSpaceCast(SrcAS, DestAS))
03050     N = DAG.getAddrSpaceCast(getCurSDLoc(), DestVT, N, SrcAS, DestAS);
03051 
03052   setValue(&I, N);
03053 }
03054 
03055 void SelectionDAGBuilder::visitInsertElement(const User &I) {
03056   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
03057   SDValue InVec = getValue(I.getOperand(0));
03058   SDValue InVal = getValue(I.getOperand(1));
03059   SDValue InIdx = DAG.getSExtOrTrunc(getValue(I.getOperand(2)),
03060                                      getCurSDLoc(), TLI.getVectorIdxTy());
03061   setValue(&I, DAG.getNode(ISD::INSERT_VECTOR_ELT, getCurSDLoc(),
03062                            TM.getTargetLowering()->getValueType(I.getType()),
03063                            InVec, InVal, InIdx));
03064 }
03065 
03066 void SelectionDAGBuilder::visitExtractElement(const User &I) {
03067   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
03068   SDValue InVec = getValue(I.getOperand(0));
03069   SDValue InIdx = DAG.getSExtOrTrunc(getValue(I.getOperand(1)),
03070                                      getCurSDLoc(), TLI.getVectorIdxTy());
03071   setValue(&I, DAG.getNode(ISD::EXTRACT_VECTOR_ELT, getCurSDLoc(),
03072                            TM.getTargetLowering()->getValueType(I.getType()),
03073                            InVec, InIdx));
03074 }
03075 
03076 // Utility for visitShuffleVector - Return true if every element in Mask,
03077 // beginning from position Pos and ending in Pos+Size, falls within the
03078 // specified sequential range [L, L+Pos). or is undef.
03079 static bool isSequentialInRange(const SmallVectorImpl<int> &Mask,
03080                                 unsigned Pos, unsigned Size, int Low) {
03081   for (unsigned i = Pos, e = Pos+Size; i != e; ++i, ++Low)
03082     if (Mask[i] >= 0 && Mask[i] != Low)
03083       return false;
03084   return true;
03085 }
03086 
03087 void SelectionDAGBuilder::visitShuffleVector(const User &I) {
03088   SDValue Src1 = getValue(I.getOperand(0));
03089   SDValue Src2 = getValue(I.getOperand(1));
03090 
03091   SmallVector<int, 8> Mask;
03092   ShuffleVectorInst::getShuffleMask(cast<Constant>(I.getOperand(2)), Mask);
03093   unsigned MaskNumElts = Mask.size();
03094 
03095   const TargetLowering *TLI = TM.getTargetLowering();
03096   EVT VT = TLI->getValueType(I.getType());
03097   EVT SrcVT = Src1.getValueType();
03098   unsigned SrcNumElts = SrcVT.getVectorNumElements();
03099 
03100   if (SrcNumElts == MaskNumElts) {
03101     setValue(&I, DAG.getVectorShuffle(VT, getCurSDLoc(), Src1, Src2,
03102                                       &Mask[0]));
03103     return;
03104   }
03105 
03106   // Normalize the shuffle vector since mask and vector length don't match.
03107   if (SrcNumElts < MaskNumElts && MaskNumElts % SrcNumElts == 0) {
03108     // Mask is longer than the source vectors and is a multiple of the source
03109     // vectors.  We can use concatenate vector to make the mask and vectors
03110     // lengths match.
03111     if (SrcNumElts*2 == MaskNumElts) {
03112       // First check for Src1 in low and Src2 in high
03113       if (isSequentialInRange(Mask, 0, SrcNumElts, 0) &&
03114           isSequentialInRange(Mask, SrcNumElts, SrcNumElts, SrcNumElts)) {
03115         // The shuffle is concatenating two vectors together.
03116         setValue(&I, DAG.getNode(ISD::CONCAT_VECTORS, getCurSDLoc(),
03117                                  VT, Src1, Src2));
03118         return;
03119       }
03120       // Then check for Src2 in low and Src1 in high
03121       if (isSequentialInRange(Mask, 0, SrcNumElts, SrcNumElts) &&
03122           isSequentialInRange(Mask, SrcNumElts, SrcNumElts, 0)) {
03123         // The shuffle is concatenating two vectors together.
03124         setValue(&I, DAG.getNode(ISD::CONCAT_VECTORS, getCurSDLoc(),
03125                                  VT, Src2, Src1));
03126         return;
03127       }
03128     }
03129 
03130     // Pad both vectors with undefs to make them the same length as the mask.
03131     unsigned NumConcat = MaskNumElts / SrcNumElts;
03132     bool Src1U = Src1.getOpcode() == ISD::UNDEF;
03133     bool Src2U = Src2.getOpcode() == ISD::UNDEF;
03134     SDValue UndefVal = DAG.getUNDEF(SrcVT);
03135 
03136     SmallVector<SDValue, 8> MOps1(NumConcat, UndefVal);
03137     SmallVector<SDValue, 8> MOps2(NumConcat, UndefVal);
03138     MOps1[0] = Src1;
03139     MOps2[0] = Src2;
03140 
03141     Src1 = Src1U ? DAG.getUNDEF(VT) : DAG.getNode(ISD::CONCAT_VECTORS,
03142                                                   getCurSDLoc(), VT, MOps1);
03143     Src2 = Src2U ? DAG.getUNDEF(VT) : DAG.getNode(ISD::CONCAT_VECTORS,
03144                                                   getCurSDLoc(), VT, MOps2);
03145 
03146     // Readjust mask for new input vector length.
03147     SmallVector<int, 8> MappedOps;
03148     for (unsigned i = 0; i != MaskNumElts; ++i) {
03149       int Idx = Mask[i];
03150       if (Idx >= (int)SrcNumElts)
03151         Idx -= SrcNumElts - MaskNumElts;
03152       MappedOps.push_back(Idx);
03153     }
03154 
03155     setValue(&I, DAG.getVectorShuffle(VT, getCurSDLoc(), Src1, Src2,
03156                                       &MappedOps[0]));
03157     return;
03158   }
03159 
03160   if (SrcNumElts > MaskNumElts) {
03161     // Analyze the access pattern of the vector to see if we can extract
03162     // two subvectors and do the shuffle. The analysis is done by calculating
03163     // the range of elements the mask access on both vectors.
03164     int MinRange[2] = { static_cast<int>(SrcNumElts),
03165                         static_cast<int>(SrcNumElts)};
03166     int MaxRange[2] = {-1, -1};
03167 
03168     for (unsigned i = 0; i != MaskNumElts; ++i) {
03169       int Idx = Mask[i];
03170       unsigned Input = 0;
03171       if (Idx < 0)
03172         continue;
03173 
03174       if (Idx >= (int)SrcNumElts) {
03175         Input = 1;
03176         Idx -= SrcNumElts;
03177       }
03178       if (Idx > MaxRange[Input])
03179         MaxRange[Input] = Idx;
03180       if (Idx < MinRange[Input])
03181         MinRange[Input] = Idx;
03182     }
03183 
03184     // Check if the access is smaller than the vector size and can we find
03185     // a reasonable extract index.
03186     int RangeUse[2] = { -1, -1 };  // 0 = Unused, 1 = Extract, -1 = Can not
03187                                    // Extract.
03188     int StartIdx[2];  // StartIdx to extract from
03189     for (unsigned Input = 0; Input < 2; ++Input) {
03190       if (MinRange[Input] >= (int)SrcNumElts && MaxRange[Input] < 0) {
03191         RangeUse[Input] = 0; // Unused
03192         StartIdx[Input] = 0;
03193         continue;
03194       }
03195 
03196       // Find a good start index that is a multiple of the mask length. Then
03197       // see if the rest of the elements are in range.
03198       StartIdx[Input] = (MinRange[Input]/MaskNumElts)*MaskNumElts;
03199       if (MaxRange[Input] - StartIdx[Input] < (int)MaskNumElts &&
03200           StartIdx[Input] + MaskNumElts <= SrcNumElts)
03201         RangeUse[Input] = 1; // Extract from a multiple of the mask length.
03202     }
03203 
03204     if (RangeUse[0] == 0 && RangeUse[1] == 0) {
03205       setValue(&I, DAG.getUNDEF(VT)); // Vectors are not used.
03206       return;
03207     }
03208     if (RangeUse[0] >= 0 && RangeUse[1] >= 0) {
03209       // Extract appropriate subvector and generate a vector shuffle
03210       for (unsigned Input = 0; Input < 2; ++Input) {
03211         SDValue &Src = Input == 0 ? Src1 : Src2;
03212         if (RangeUse[Input] == 0)
03213           Src = DAG.getUNDEF(VT);
03214         else
03215           Src = DAG.getNode(ISD::EXTRACT_SUBVECTOR, getCurSDLoc(), VT,
03216                             Src, DAG.getConstant(StartIdx[Input],
03217                                                  TLI->getVectorIdxTy()));
03218       }
03219 
03220       // Calculate new mask.
03221       SmallVector<int, 8> MappedOps;
03222       for (unsigned i = 0; i != MaskNumElts; ++i) {
03223         int Idx = Mask[i];
03224         if (Idx >= 0) {
03225           if (Idx < (int)SrcNumElts)
03226             Idx -= StartIdx[0];
03227           else
03228             Idx -= SrcNumElts + StartIdx[1] - MaskNumElts;
03229         }
03230         MappedOps.push_back(Idx);
03231       }
03232 
03233       setValue(&I, DAG.getVectorShuffle(VT, getCurSDLoc(), Src1, Src2,
03234                                         &MappedOps[0]));
03235       return;
03236     }
03237   }
03238 
03239   // We can't use either concat vectors or extract subvectors so fall back to
03240   // replacing the shuffle with extract and build vector.
03241   // to insert and build vector.
03242   EVT EltVT = VT.getVectorElementType();
03243   EVT IdxVT = TLI->getVectorIdxTy();
03244   SmallVector<SDValue,8> Ops;
03245   for (unsigned i = 0; i != MaskNumElts; ++i) {
03246     int Idx = Mask[i];
03247     SDValue Res;
03248 
03249     if (Idx < 0) {
03250       Res = DAG.getUNDEF(EltVT);
03251     } else {
03252       SDValue &Src = Idx < (int)SrcNumElts ? Src1 : Src2;
03253       if (Idx >= (int)SrcNumElts) Idx -= SrcNumElts;
03254 
03255       Res = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, getCurSDLoc(),
03256                         EltVT, Src, DAG.getConstant(Idx, IdxVT));
03257     }
03258 
03259     Ops.push_back(Res);
03260   }
03261 
03262   setValue(&I, DAG.getNode(ISD::BUILD_VECTOR, getCurSDLoc(), VT, Ops));
03263 }
03264 
03265 void SelectionDAGBuilder::visitInsertValue(const InsertValueInst &I) {
03266   const Value *Op0 = I.getOperand(0);
03267   const Value *Op1 = I.getOperand(1);
03268   Type *AggTy = I.getType();
03269   Type *ValTy = Op1->getType();
03270   bool IntoUndef = isa<UndefValue>(Op0);
03271   bool FromUndef = isa<UndefValue>(Op1);
03272 
03273   unsigned LinearIndex = ComputeLinearIndex(AggTy, I.getIndices());
03274 
03275   const TargetLowering *TLI = TM.getTargetLowering();
03276   SmallVector<EVT, 4> AggValueVTs;
03277   ComputeValueVTs(*TLI, AggTy, AggValueVTs);
03278   SmallVector<EVT, 4> ValValueVTs;
03279   ComputeValueVTs(*TLI, ValTy, ValValueVTs);
03280 
03281   unsigned NumAggValues = AggValueVTs.size();
03282   unsigned NumValValues = ValValueVTs.size();
03283   SmallVector<SDValue, 4> Values(NumAggValues);
03284 
03285   SDValue Agg = getValue(Op0);
03286   unsigned i = 0;
03287   // Copy the beginning value(s) from the original aggregate.
03288   for (; i != LinearIndex; ++i)
03289     Values[i] = IntoUndef ? DAG.getUNDEF(AggValueVTs[i]) :
03290                 SDValue(Agg.getNode(), Agg.getResNo() + i);
03291   // Copy values from the inserted value(s).
03292   if (NumValValues) {
03293     SDValue Val = getValue(Op1);
03294     for (; i != LinearIndex + NumValValues; ++i)
03295       Values[i] = FromUndef ? DAG.getUNDEF(AggValueVTs[i]) :
03296                   SDValue(Val.getNode(), Val.getResNo() + i - LinearIndex);
03297   }
03298   // Copy remaining value(s) from the original aggregate.
03299   for (; i != NumAggValues; ++i)
03300     Values[i] = IntoUndef ? DAG.getUNDEF(AggValueVTs[i]) :
03301                 SDValue(Agg.getNode(), Agg.getResNo() + i);
03302 
03303   setValue(&I, DAG.getNode(ISD::MERGE_VALUES, getCurSDLoc(),
03304                            DAG.getVTList(AggValueVTs), Values));
03305 }
03306 
03307 void SelectionDAGBuilder::visitExtractValue(const ExtractValueInst &I) {
03308   const Value *Op0 = I.getOperand(0);
03309   Type *AggTy = Op0->getType();
03310   Type *ValTy = I.getType();
03311   bool OutOfUndef = isa<UndefValue>(Op0);
03312 
03313   unsigned LinearIndex = ComputeLinearIndex(AggTy, I.getIndices());
03314 
03315   const TargetLowering *TLI = TM.getTargetLowering();
03316   SmallVector<EVT, 4> ValValueVTs;
03317   ComputeValueVTs(*TLI, ValTy, ValValueVTs);
03318 
03319   unsigned NumValValues = ValValueVTs.size();
03320 
03321   // Ignore a extractvalue that produces an empty object
03322   if (!NumValValues) {
03323     setValue(&I, DAG.getUNDEF(MVT(MVT::Other)));
03324     return;
03325   }
03326 
03327   SmallVector<SDValue, 4> Values(NumValValues);
03328 
03329   SDValue Agg = getValue(Op0);
03330   // Copy out the selected value(s).
03331   for (unsigned i = LinearIndex; i != LinearIndex + NumValValues; ++i)
03332     Values[i - LinearIndex] =
03333       OutOfUndef ?
03334         DAG.getUNDEF(Agg.getNode()->getValueType(Agg.getResNo() + i)) :
03335         SDValue(Agg.getNode(), Agg.getResNo() + i);
03336 
03337   setValue(&I, DAG.getNode(ISD::MERGE_VALUES, getCurSDLoc(),
03338                            DAG.getVTList(ValValueVTs), Values));
03339 }
03340 
03341 void SelectionDAGBuilder::visitGetElementPtr(const User &I) {
03342   Value *Op0 = I.getOperand(0);
03343   // Note that the pointer operand may be a vector of pointers. Take the scalar
03344   // element which holds a pointer.
03345   Type *Ty = Op0->getType()->getScalarType();
03346   unsigned AS = Ty->getPointerAddressSpace();
03347   SDValue N = getValue(Op0);
03348 
03349   for (GetElementPtrInst::const_op_iterator OI = I.op_begin()+1, E = I.op_end();
03350        OI != E; ++OI) {
03351     const Value *Idx = *OI;
03352     if (StructType *StTy = dyn_cast<StructType>(Ty)) {
03353       unsigned Field = cast<Constant>(Idx)->getUniqueInteger().getZExtValue();
03354       if (Field) {
03355         // N = N + Offset
03356         uint64_t Offset = DL->getStructLayout(StTy)->getElementOffset(Field);
03357         N = DAG.getNode(ISD::ADD, getCurSDLoc(), N.getValueType(), N,
03358                         DAG.getConstant(Offset, N.getValueType()));
03359       }
03360 
03361       Ty = StTy->getElementType(Field);
03362     } else {
03363       Ty = cast<SequentialType>(Ty)->getElementType();
03364 
03365       // If this is a constant subscript, handle it quickly.
03366       const TargetLowering *TLI = TM.getTargetLowering();
03367       if (const ConstantInt *CI = dyn_cast<ConstantInt>(Idx)) {
03368         if (CI->isZero()) continue;
03369         uint64_t Offs =
03370             DL->getTypeAllocSize(Ty)*cast<ConstantInt>(CI)->getSExtValue();
03371         SDValue OffsVal;
03372         EVT PTy = TLI->getPointerTy(AS);
03373         unsigned PtrBits = PTy.getSizeInBits();
03374         if (PtrBits < 64)
03375           OffsVal = DAG.getNode(ISD::TRUNCATE, getCurSDLoc(), PTy,
03376                                 DAG.getConstant(Offs, MVT::i64));
03377         else
03378           OffsVal = DAG.getConstant(Offs, PTy);
03379 
03380         N = DAG.getNode(ISD::ADD, getCurSDLoc(), N.getValueType(), N,
03381                         OffsVal);
03382         continue;
03383       }
03384 
03385       // N = N + Idx * ElementSize;
03386       APInt ElementSize = APInt(TLI->getPointerSizeInBits(AS),
03387                                 DL->getTypeAllocSize(Ty));
03388       SDValue IdxN = getValue(Idx);
03389 
03390       // If the index is smaller or larger than intptr_t, truncate or extend
03391       // it.
03392       IdxN = DAG.getSExtOrTrunc(IdxN, getCurSDLoc(), N.getValueType());
03393 
03394       // If this is a multiply by a power of two, turn it into a shl
03395       // immediately.  This is a very common case.
03396       if (ElementSize != 1) {
03397         if (ElementSize.isPowerOf2()) {
03398           unsigned Amt = ElementSize.logBase2();
03399           IdxN = DAG.getNode(ISD::SHL, getCurSDLoc(),
03400                              N.getValueType(), IdxN,
03401                              DAG.getConstant(Amt, IdxN.getValueType()));
03402         } else {
03403           SDValue Scale = DAG.getConstant(ElementSize, IdxN.getValueType());
03404           IdxN = DAG.getNode(ISD::MUL, getCurSDLoc(),
03405                              N.getValueType(), IdxN, Scale);
03406         }
03407       }
03408 
03409       N = DAG.getNode(ISD::ADD, getCurSDLoc(),
03410                       N.getValueType(), N, IdxN);
03411     }
03412   }
03413 
03414   setValue(&I, N);
03415 }
03416 
03417 void SelectionDAGBuilder::visitAlloca(const AllocaInst &I) {
03418   // If this is a fixed sized alloca in the entry block of the function,
03419   // allocate it statically on the stack.
03420   if (FuncInfo.StaticAllocaMap.count(&I))
03421     return;   // getValue will auto-populate this.
03422 
03423   Type *Ty = I.getAllocatedType();
03424   const TargetLowering *TLI = TM.getTargetLowering();
03425   uint64_t TySize = TLI->getDataLayout()->getTypeAllocSize(Ty);
03426   unsigned Align =
03427     std::max((unsigned)TLI->getDataLayout()->getPrefTypeAlignment(Ty),
03428              I.getAlignment());
03429 
03430   SDValue AllocSize = getValue(I.getArraySize());
03431 
03432   EVT IntPtr = TLI->getPointerTy();
03433   if (AllocSize.getValueType() != IntPtr)
03434     AllocSize = DAG.getZExtOrTrunc(AllocSize, getCurSDLoc(), IntPtr);
03435 
03436   AllocSize = DAG.getNode(ISD::MUL, getCurSDLoc(), IntPtr,
03437                           AllocSize,
03438                           DAG.getConstant(TySize, IntPtr));
03439 
03440   // Handle alignment.  If the requested alignment is less than or equal to
03441   // the stack alignment, ignore it.  If the size is greater than or equal to
03442   // the stack alignment, we note this in the DYNAMIC_STACKALLOC node.
03443   unsigned StackAlign = TM.getFrameLowering()->getStackAlignment();
03444   if (Align <= StackAlign)
03445     Align = 0;
03446 
03447   // Round the size of the allocation up to the stack alignment size
03448   // by add SA-1 to the size.
03449   AllocSize = DAG.getNode(ISD::ADD, getCurSDLoc(),
03450                           AllocSize.getValueType(), AllocSize,
03451                           DAG.getIntPtrConstant(StackAlign-1));
03452 
03453   // Mask out the low bits for alignment purposes.
03454   AllocSize = DAG.getNode(ISD::AND, getCurSDLoc(),
03455                           AllocSize.getValueType(), AllocSize,
03456                           DAG.getIntPtrConstant(~(uint64_t)(StackAlign-1)));
03457 
03458   SDValue Ops[] = { getRoot(), AllocSize, DAG.getIntPtrConstant(Align) };
03459   SDVTList VTs = DAG.getVTList(AllocSize.getValueType(), MVT::Other);
03460   SDValue DSA = DAG.getNode(ISD::DYNAMIC_STACKALLOC, getCurSDLoc(), VTs, Ops);
03461   setValue(&I, DSA);
03462   DAG.setRoot(DSA.getValue(1));
03463 
03464   assert(FuncInfo.MF->getFrameInfo()->hasVarSizedObjects());
03465 }
03466 
03467 void SelectionDAGBuilder::visitLoad(const LoadInst &I) {
03468   if (I.isAtomic())
03469     return visitAtomicLoad(I);
03470 
03471   const Value *SV = I.getOperand(0);
03472   SDValue Ptr = getValue(SV);
03473 
03474   Type *Ty = I.getType();
03475 
03476   bool isVolatile = I.isVolatile();
03477   bool isNonTemporal = I.getMetadata("nontemporal") != nullptr;
03478   bool isInvariant = I.getMetadata("invariant.load") != nullptr;
03479   unsigned Alignment = I.getAlignment();
03480 
03481   AAMDNodes AAInfo;
03482   I.getAAMetadata(AAInfo);
03483   const MDNode *Ranges = I.getMetadata(LLVMContext::MD_range);
03484 
03485   SmallVector<EVT, 4> ValueVTs;
03486   SmallVector<uint64_t, 4> Offsets;
03487   ComputeValueVTs(*TM.getTargetLowering(), Ty, ValueVTs, &Offsets);
03488   unsigned NumValues = ValueVTs.size();
03489   if (NumValues == 0)
03490     return;
03491 
03492   SDValue Root;
03493   bool ConstantMemory = false;
03494   if (isVolatile || NumValues > MaxParallelChains)
03495     // Serialize volatile loads with other side effects.
03496     Root = getRoot();
03497   else if (AA->pointsToConstantMemory(
03498              AliasAnalysis::Location(SV, AA->getTypeStoreSize(Ty), AAInfo))) {
03499     // Do not serialize (non-volatile) loads of constant memory with anything.
03500     Root = DAG.getEntryNode();
03501     ConstantMemory = true;
03502   } else {
03503     // Do not serialize non-volatile loads against each other.
03504     Root = DAG.getRoot();
03505   }
03506 
03507   const TargetLowering *TLI = TM.getTargetLowering();
03508   if (isVolatile)
03509     Root = TLI->prepareVolatileOrAtomicLoad(Root, getCurSDLoc(), DAG);
03510 
03511   SmallVector<SDValue, 4> Values(NumValues);
03512   SmallVector<SDValue, 4> Chains(std::min(unsigned(MaxParallelChains),
03513                                           NumValues));
03514   EVT PtrVT = Ptr.getValueType();
03515   unsigned ChainI = 0;
03516   for (unsigned i = 0; i != NumValues; ++i, ++ChainI) {
03517     // Serializing loads here may result in excessive register pressure, and
03518     // TokenFactor places arbitrary choke points on the scheduler. SD scheduling
03519     // could recover a bit by hoisting nodes upward in the chain by recognizing
03520     // they are side-effect free or do not alias. The optimizer should really
03521     // avoid this case by converting large object/array copies to llvm.memcpy
03522     // (MaxParallelChains should always remain as failsafe).
03523     if (ChainI == MaxParallelChains) {
03524       assert(PendingLoads.empty() && "PendingLoads must be serialized first");
03525       SDValue Chain = DAG.getNode(ISD::TokenFactor, getCurSDLoc(), MVT::Other,
03526                                   makeArrayRef(Chains.data(), ChainI));
03527       Root = Chain;
03528       ChainI = 0;
03529     }
03530     SDValue A = DAG.getNode(ISD::ADD, getCurSDLoc(),
03531                             PtrVT, Ptr,
03532                             DAG.getConstant(Offsets[i], PtrVT));
03533     SDValue L = DAG.getLoad(ValueVTs[i], getCurSDLoc(), Root,
03534                             A, MachinePointerInfo(SV, Offsets[i]), isVolatile,
03535                             isNonTemporal, isInvariant, Alignment, AAInfo,
03536                             Ranges);
03537 
03538     Values[i] = L;
03539     Chains[ChainI] = L.getValue(1);
03540   }
03541 
03542   if (!ConstantMemory) {
03543     SDValue Chain = DAG.getNode(ISD::TokenFactor, getCurSDLoc(), MVT::Other,
03544                                 makeArrayRef(Chains.data(), ChainI));
03545     if (isVolatile)
03546       DAG.setRoot(Chain);
03547     else
03548       PendingLoads.push_back(Chain);
03549   }
03550 
03551   setValue(&I, DAG.getNode(ISD::MERGE_VALUES, getCurSDLoc(),
03552                            DAG.getVTList(ValueVTs), Values));
03553 }
03554 
03555 void SelectionDAGBuilder::visitStore(const StoreInst &I) {
03556   if (I.isAtomic())
03557     return visitAtomicStore(I);
03558 
03559   const Value *SrcV = I.getOperand(0);
03560   const Value *PtrV = I.getOperand(1);
03561 
03562   SmallVector<EVT, 4> ValueVTs;
03563   SmallVector<uint64_t, 4> Offsets;
03564   ComputeValueVTs(*TM.getTargetLowering(), SrcV->getType(), ValueVTs, &Offsets);
03565   unsigned NumValues = ValueVTs.size();
03566   if (NumValues == 0)
03567     return;
03568 
03569   // Get the lowered operands. Note that we do this after
03570   // checking if NumResults is zero, because with zero results
03571   // the operands won't have values in the map.
03572   SDValue Src = getValue(SrcV);
03573   SDValue Ptr = getValue(PtrV);
03574 
03575   SDValue Root = getRoot();
03576   SmallVector<SDValue, 4> Chains(std::min(unsigned(MaxParallelChains),
03577                                           NumValues));
03578   EVT PtrVT = Ptr.getValueType();
03579   bool isVolatile = I.isVolatile();
03580   bool isNonTemporal = I.getMetadata("nontemporal") != nullptr;
03581   unsigned Alignment = I.getAlignment();
03582 
03583   AAMDNodes AAInfo;
03584   I.getAAMetadata(AAInfo);
03585 
03586   unsigned ChainI = 0;
03587   for (unsigned i = 0; i != NumValues; ++i, ++ChainI) {
03588     // See visitLoad comments.
03589     if (ChainI == MaxParallelChains) {
03590       SDValue Chain = DAG.getNode(ISD::TokenFactor, getCurSDLoc(), MVT::Other,
03591                                   makeArrayRef(Chains.data(), ChainI));
03592       Root = Chain;
03593       ChainI = 0;
03594     }
03595     SDValue Add = DAG.getNode(ISD::ADD, getCurSDLoc(), PtrVT, Ptr,
03596                               DAG.getConstant(Offsets[i], PtrVT));
03597     SDValue St = DAG.getStore(Root, getCurSDLoc(),
03598                               SDValue(Src.getNode(), Src.getResNo() + i),
03599                               Add, MachinePointerInfo(PtrV, Offsets[i]),
03600                               isVolatile, isNonTemporal, Alignment, AAInfo);
03601     Chains[ChainI] = St;
03602   }
03603 
03604   SDValue StoreNode = DAG.getNode(ISD::TokenFactor, getCurSDLoc(), MVT::Other,
03605                                   makeArrayRef(Chains.data(), ChainI));
03606   DAG.setRoot(StoreNode);
03607 }
03608 
03609 static SDValue InsertFenceForAtomic(SDValue Chain, AtomicOrdering Order,
03610                                     SynchronizationScope Scope,
03611                                     bool Before, SDLoc dl,
03612                                     SelectionDAG &DAG,
03613                                     const TargetLowering &TLI) {
03614   // Fence, if necessary
03615   if (Before) {
03616     if (Order == AcquireRelease || Order == SequentiallyConsistent)
03617       Order = Release;
03618     else if (Order == Acquire || Order == Monotonic || Order == Unordered)
03619       return Chain;
03620   } else {
03621     if (Order == AcquireRelease)
03622       Order = Acquire;
03623     else if (Order == Release || Order == Monotonic || Order == Unordered)
03624       return Chain;
03625   }
03626   SDValue Ops[3];
03627   Ops[0] = Chain;
03628   Ops[1] = DAG.getConstant(Order, TLI.getPointerTy());
03629   Ops[2] = DAG.getConstant(Scope, TLI.getPointerTy());
03630   return DAG.getNode(ISD::ATOMIC_FENCE, dl, MVT::Other, Ops);
03631 }
03632 
03633 void SelectionDAGBuilder::visitAtomicCmpXchg(const AtomicCmpXchgInst &I) {
03634   SDLoc dl = getCurSDLoc();
03635   AtomicOrdering SuccessOrder = I.getSuccessOrdering();
03636   AtomicOrdering FailureOrder = I.getFailureOrdering();
03637   SynchronizationScope Scope = I.getSynchScope();
03638 
03639   SDValue InChain = getRoot();
03640 
03641   const TargetLowering *TLI = TM.getTargetLowering();
03642   if (TLI->getInsertFencesForAtomic())
03643     InChain = InsertFenceForAtomic(InChain, SuccessOrder, Scope, true, dl,
03644                                    DAG, *TLI);
03645 
03646   MVT MemVT = getValue(I.getCompareOperand()).getSimpleValueType();
03647   SDVTList VTs = DAG.getVTList(MemVT, MVT::i1, MVT::Other);
03648   SDValue L = DAG.getAtomicCmpSwap(
03649       ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS, dl, MemVT, VTs, InChain,
03650       getValue(I.getPointerOperand()), getValue(I.getCompareOperand()),
03651       getValue(I.getNewValOperand()), MachinePointerInfo(I.getPointerOperand()),
03652       0 /* Alignment */,
03653       TLI->getInsertFencesForAtomic() ? Monotonic : SuccessOrder,
03654       TLI->getInsertFencesForAtomic() ? Monotonic : FailureOrder, Scope);
03655 
03656   SDValue OutChain = L.getValue(2);
03657 
03658   if (TLI->getInsertFencesForAtomic())
03659     OutChain = InsertFenceForAtomic(OutChain, SuccessOrder, Scope, false, dl,
03660                                     DAG, *TLI);
03661 
03662   setValue(&I, L);
03663   DAG.setRoot(OutChain);
03664 }
03665 
03666 void SelectionDAGBuilder::visitAtomicRMW(const AtomicRMWInst &I) {
03667   SDLoc dl = getCurSDLoc();
03668   ISD::NodeType NT;
03669   switch (I.getOperation()) {
03670   default: llvm_unreachable("Unknown atomicrmw operation");
03671   case AtomicRMWInst::Xchg: NT = ISD::ATOMIC_SWAP; break;
03672   case AtomicRMWInst::Add:  NT = ISD::ATOMIC_LOAD_ADD; break;
03673   case AtomicRMWInst::Sub:  NT = ISD::ATOMIC_LOAD_SUB; break;
03674   case AtomicRMWInst::And:  NT = ISD::ATOMIC_LOAD_AND; break;
03675   case AtomicRMWInst::Nand: NT = ISD::ATOMIC_LOAD_NAND; break;
03676   case AtomicRMWInst::Or:   NT = ISD::ATOMIC_LOAD_OR; break;
03677   case AtomicRMWInst::Xor:  NT = ISD::ATOMIC_LOAD_XOR; break;
03678   case AtomicRMWInst::Max:  NT = ISD::ATOMIC_LOAD_MAX; break;
03679   case AtomicRMWInst::Min:  NT = ISD::ATOMIC_LOAD_MIN; break;
03680   case AtomicRMWInst::UMax: NT = ISD::ATOMIC_LOAD_UMAX; break;
03681   case AtomicRMWInst::UMin: NT = ISD::ATOMIC_LOAD_UMIN; break;
03682   }
03683   AtomicOrdering Order = I.getOrdering();
03684   SynchronizationScope Scope = I.getSynchScope();
03685 
03686   SDValue InChain = getRoot();
03687 
03688   const TargetLowering *TLI = TM.getTargetLowering();
03689   if (TLI->getInsertFencesForAtomic())
03690     InChain = InsertFenceForAtomic(InChain, Order, Scope, true, dl,
03691                                    DAG, *TLI);
03692 
03693   SDValue L =
03694     DAG.getAtomic(NT, dl,
03695                   getValue(I.getValOperand()).getSimpleValueType(),
03696                   InChain,
03697                   getValue(I.getPointerOperand()),
03698                   getValue(I.getValOperand()),
03699                   I.getPointerOperand(), 0 /* Alignment */,
03700                   TLI->getInsertFencesForAtomic() ? Monotonic : Order,
03701                   Scope);
03702 
03703   SDValue OutChain = L.getValue(1);
03704 
03705   if (TLI->getInsertFencesForAtomic())
03706     OutChain = InsertFenceForAtomic(OutChain, Order, Scope, false, dl,
03707                                     DAG, *TLI);
03708 
03709   setValue(&I, L);
03710   DAG.setRoot(OutChain);
03711 }
03712 
03713 void SelectionDAGBuilder::visitFence(const FenceInst &I) {
03714   SDLoc dl = getCurSDLoc();
03715   const TargetLowering *TLI = TM.getTargetLowering();
03716   SDValue Ops[3];
03717   Ops[0] = getRoot();
03718   Ops[1] = DAG.getConstant(I.getOrdering(), TLI->getPointerTy());
03719   Ops[2] = DAG.getConstant(I.getSynchScope(), TLI->getPointerTy());
03720   DAG.setRoot(DAG.getNode(ISD::ATOMIC_FENCE, dl, MVT::Other, Ops));
03721 }
03722 
03723 void SelectionDAGBuilder::visitAtomicLoad(const LoadInst &I) {
03724   SDLoc dl = getCurSDLoc();
03725   AtomicOrdering Order = I.getOrdering();
03726   SynchronizationScope Scope = I.getSynchScope();
03727 
03728   SDValue InChain = getRoot();
03729 
03730   const TargetLowering *TLI = TM.getTargetLowering();
03731   EVT VT = TLI->getValueType(I.getType());
03732 
03733   if (I.getAlignment() < VT.getSizeInBits() / 8)
03734     report_fatal_error("Cannot generate unaligned atomic load");
03735 
03736   MachineMemOperand *MMO =
03737       DAG.getMachineFunction().
03738       getMachineMemOperand(MachinePointerInfo(I.getPointerOperand()),
03739                            MachineMemOperand::MOVolatile |
03740                            MachineMemOperand::MOLoad,
03741                            VT.getStoreSize(),
03742                            I.getAlignment() ? I.getAlignment() :
03743                                               DAG.getEVTAlignment(VT));
03744 
03745   InChain = TLI->prepareVolatileOrAtomicLoad(InChain, dl, DAG);
03746   SDValue L =
03747       DAG.getAtomic(ISD::ATOMIC_LOAD, dl, VT, VT, InChain,
03748                     getValue(I.getPointerOperand()), MMO,
03749                     TLI->getInsertFencesForAtomic() ? Monotonic : Order,
03750                     Scope);
03751 
03752   SDValue OutChain = L.getValue(1);
03753 
03754   if (TLI->getInsertFencesForAtomic())
03755     OutChain = InsertFenceForAtomic(OutChain, Order, Scope, false, dl,
03756                                     DAG, *TLI);
03757 
03758   setValue(&I, L);
03759   DAG.setRoot(OutChain);
03760 }
03761 
03762 void SelectionDAGBuilder::visitAtomicStore(const StoreInst &I) {
03763   SDLoc dl = getCurSDLoc();
03764 
03765   AtomicOrdering Order = I.getOrdering();
03766   SynchronizationScope Scope = I.getSynchScope();
03767 
03768   SDValue InChain = getRoot();
03769 
03770   const TargetLowering *TLI = TM.getTargetLowering();
03771   EVT VT = TLI->getValueType(I.getValueOperand()->getType());
03772 
03773   if (I.getAlignment() < VT.getSizeInBits() / 8)
03774     report_fatal_error("Cannot generate unaligned atomic store");
03775 
03776   if (TLI->getInsertFencesForAtomic())
03777     InChain = InsertFenceForAtomic(InChain, Order, Scope, true, dl,
03778                                    DAG, *TLI);
03779 
03780   SDValue OutChain =
03781     DAG.getAtomic(ISD::ATOMIC_STORE, dl, VT,
03782                   InChain,
03783                   getValue(I.getPointerOperand()),
03784                   getValue(I.getValueOperand()),
03785                   I.getPointerOperand(), I.getAlignment(),
03786                   TLI->getInsertFencesForAtomic() ? Monotonic : Order,
03787                   Scope);
03788 
03789   if (TLI->getInsertFencesForAtomic())
03790     OutChain = InsertFenceForAtomic(OutChain, Order, Scope, false, dl,
03791                                     DAG, *TLI);
03792 
03793   DAG.setRoot(OutChain);
03794 }
03795 
03796 /// visitTargetIntrinsic - Lower a call of a target intrinsic to an INTRINSIC
03797 /// node.
03798 void SelectionDAGBuilder::visitTargetIntrinsic(const CallInst &I,
03799                                                unsigned Intrinsic) {
03800   bool HasChain = !I.doesNotAccessMemory();
03801   bool OnlyLoad = HasChain && I.onlyReadsMemory();
03802 
03803   // Build the operand list.
03804   SmallVector<SDValue, 8> Ops;
03805   if (HasChain) {  // If this intrinsic has side-effects, chainify it.
03806     if (OnlyLoad) {
03807       // We don't need to serialize loads against other loads.
03808       Ops.push_back(DAG.getRoot());
03809     } else {
03810       Ops.push_back(getRoot());
03811     }
03812   }
03813 
03814   // Info is set by getTgtMemInstrinsic
03815   TargetLowering::IntrinsicInfo Info;
03816   const TargetLowering *TLI = TM.getTargetLowering();
03817   bool IsTgtIntrinsic = TLI->getTgtMemIntrinsic(Info, I, Intrinsic);
03818 
03819   // Add the intrinsic ID as an integer operand if it's not a target intrinsic.
03820   if (!IsTgtIntrinsic || Info.opc == ISD::INTRINSIC_VOID ||
03821       Info.opc == ISD::INTRINSIC_W_CHAIN)
03822     Ops.push_back(DAG.getTargetConstant(Intrinsic, TLI->getPointerTy()));
03823 
03824   // Add all operands of the call to the operand list.
03825   for (unsigned i = 0, e = I.getNumArgOperands(); i != e; ++i) {
03826     SDValue Op = getValue(I.getArgOperand(i));
03827     Ops.push_back(Op);
03828   }
03829 
03830   SmallVector<EVT, 4> ValueVTs;
03831   ComputeValueVTs(*TLI, I.getType(), ValueVTs);
03832 
03833   if (HasChain)
03834     ValueVTs.push_back(MVT::Other);
03835 
03836   SDVTList VTs = DAG.getVTList(ValueVTs);
03837 
03838   // Create the node.
03839   SDValue Result;
03840   if (IsTgtIntrinsic) {
03841     // This is target intrinsic that touches memory
03842     Result = DAG.getMemIntrinsicNode(Info.opc, getCurSDLoc(),
03843                                      VTs, Ops, Info.memVT,
03844                                    MachinePointerInfo(Info.ptrVal, Info.offset),
03845                                      Info.align, Info.vol,
03846                                      Info.readMem, Info.writeMem);
03847   } else if (!HasChain) {
03848     Result = DAG.getNode(ISD::INTRINSIC_WO_CHAIN, getCurSDLoc(), VTs, Ops);
03849   } else if (!I.getType()->isVoidTy()) {
03850     Result = DAG.getNode(ISD::INTRINSIC_W_CHAIN, getCurSDLoc(), VTs, Ops);
03851   } else {
03852     Result = DAG.getNode(ISD::INTRINSIC_VOID, getCurSDLoc(), VTs, Ops);
03853   }
03854 
03855   if (HasChain) {
03856     SDValue Chain = Result.getValue(Result.getNode()->getNumValues()-1);
03857     if (OnlyLoad)
03858       PendingLoads.push_back(Chain);
03859     else
03860       DAG.setRoot(Chain);
03861   }
03862 
03863   if (!I.getType()->isVoidTy()) {
03864     if (VectorType *PTy = dyn_cast<VectorType>(I.getType())) {
03865       EVT VT = TLI->getValueType(PTy);
03866       Result = DAG.getNode(ISD::BITCAST, getCurSDLoc(), VT, Result);
03867     }
03868 
03869     setValue(&I, Result);
03870   }
03871 }
03872 
03873 /// GetSignificand - Get the significand and build it into a floating-point
03874 /// number with exponent of 1:
03875 ///
03876 ///   Op = (Op & 0x007fffff) | 0x3f800000;
03877 ///
03878 /// where Op is the hexadecimal representation of floating point value.
03879 static SDValue
03880 GetSignificand(SelectionDAG &DAG, SDValue Op, SDLoc dl) {
03881   SDValue t1 = DAG.getNode(ISD::AND, dl, MVT::i32, Op,
03882                            DAG.getConstant(0x007fffff, MVT::i32));
03883   SDValue t2 = DAG.getNode(ISD::OR, dl, MVT::i32, t1,
03884                            DAG.getConstant(0x3f800000, MVT::i32));
03885   return DAG.getNode(ISD::BITCAST, dl, MVT::f32, t2);
03886 }
03887 
03888 /// GetExponent - Get the exponent:
03889 ///
03890 ///   (float)(int)(((Op & 0x7f800000) >> 23) - 127);
03891 ///
03892 /// where Op is the hexadecimal representation of floating point value.
03893 static SDValue
03894 GetExponent(SelectionDAG &DAG, SDValue Op, const TargetLowering &TLI,
03895             SDLoc dl) {
03896   SDValue t0 = DAG.getNode(ISD::AND, dl, MVT::i32, Op,
03897                            DAG.getConstant(0x7f800000, MVT::i32));
03898   SDValue t1 = DAG.getNode(ISD::SRL, dl, MVT::i32, t0,
03899                            DAG.getConstant(23, TLI.getPointerTy()));
03900   SDValue t2 = DAG.getNode(ISD::SUB, dl, MVT::i32, t1,
03901                            DAG.getConstant(127, MVT::i32));
03902   return DAG.getNode(ISD::SINT_TO_FP, dl, MVT::f32, t2);
03903 }
03904 
03905 /// getF32Constant - Get 32-bit floating point constant.
03906 static SDValue
03907 getF32Constant(SelectionDAG &DAG, unsigned Flt) {
03908   return DAG.getConstantFP(APFloat(APFloat::IEEEsingle, APInt(32, Flt)),
03909                            MVT::f32);
03910 }
03911 
03912 /// expandExp - Lower an exp intrinsic. Handles the special sequences for
03913 /// limited-precision mode.
03914 static SDValue expandExp(SDLoc dl, SDValue Op, SelectionDAG &DAG,
03915                          const TargetLowering &TLI) {
03916   if (Op.getValueType() == MVT::f32 &&
03917       LimitFloatPrecision > 0 && LimitFloatPrecision <= 18) {
03918 
03919     // Put the exponent in the right bit position for later addition to the
03920     // final result:
03921     //
03922     //   #define LOG2OFe 1.4426950f
03923     //   IntegerPartOfX = ((int32_t)(X * LOG2OFe));
03924     SDValue t0 = DAG.getNode(ISD::FMUL, dl, MVT::f32, Op,
03925                              getF32Constant(DAG, 0x3fb8aa3b));
03926     SDValue IntegerPartOfX = DAG.getNode(ISD::FP_TO_SINT, dl, MVT::i32, t0);
03927 
03928     //   FractionalPartOfX = (X * LOG2OFe) - (float)IntegerPartOfX;
03929     SDValue t1 = DAG.getNode(ISD::SINT_TO_FP, dl, MVT::f32, IntegerPartOfX);
03930     SDValue X = DAG.getNode(ISD::FSUB, dl, MVT::f32, t0, t1);
03931 
03932     //   IntegerPartOfX <<= 23;
03933     IntegerPartOfX = DAG.getNode(ISD::SHL, dl, MVT::i32, IntegerPartOfX,
03934                                  DAG.getConstant(23, TLI.getPointerTy()));
03935 
03936     SDValue TwoToFracPartOfX;
03937     if (LimitFloatPrecision <= 6) {
03938       // For floating-point precision of 6:
03939       //
03940       //   TwoToFractionalPartOfX =
03941       //     0.997535578f +
03942       //       (0.735607626f + 0.252464424f * x) * x;
03943       //
03944       // error 0.0144103317, which is 6 bits
03945       SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X,
03946                                getF32Constant(DAG, 0x3e814304));
03947       SDValue t3 = DAG.getNode(ISD::FADD, dl, MVT::f32, t2,
03948                                getF32Constant(DAG, 0x3f3c50c8));
03949       SDValue t4 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t3, X);
03950       TwoToFracPartOfX = DAG.getNode(ISD::FADD, dl, MVT::f32, t4,
03951                                      getF32Constant(DAG, 0x3f7f5e7e));
03952     } else if (LimitFloatPrecision <= 12) {
03953       // For floating-point precision of 12:
03954       //
03955       //   TwoToFractionalPartOfX =
03956       //     0.999892986f +
03957       //       (0.696457318f +
03958       //         (0.224338339f + 0.792043434e-1f * x) * x) * x;
03959       //
03960       // 0.000107046256 error, which is 13 to 14 bits
03961       SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X,
03962                                getF32Constant(DAG, 0x3da235e3));
03963       SDValue t3 = DAG.getNode(ISD::FADD, dl, MVT::f32, t2,
03964                                getF32Constant(DAG, 0x3e65b8f3));
03965       SDValue t4 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t3, X);
03966       SDValue t5 = DAG.getNode(ISD::FADD, dl, MVT::f32, t4,
03967                                getF32Constant(DAG, 0x3f324b07));
03968       SDValue t6 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t5, X);
03969       TwoToFracPartOfX = DAG.getNode(ISD::FADD, dl, MVT::f32, t6,
03970                                      getF32Constant(DAG, 0x3f7ff8fd));
03971     } else { // LimitFloatPrecision <= 18
03972       // For floating-point precision of 18:
03973       //
03974       //   TwoToFractionalPartOfX =
03975       //     0.999999982f +
03976       //       (0.693148872f +
03977       //         (0.240227044f +
03978       //           (0.554906021e-1f +
03979       //             (0.961591928e-2f +
03980       //               (0.136028312e-2f + 0.157059148e-3f *x)*x)*x)*x)*x)*x;
03981       //
03982       // error 2.47208000*10^(-7), which is better than 18 bits
03983       SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X,
03984                                getF32Constant(DAG, 0x3924b03e));
03985       SDValue t3 = DAG.getNode(ISD::FADD, dl, MVT::f32, t2,
03986                                getF32Constant(DAG, 0x3ab24b87));
03987       SDValue t4 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t3, X);
03988       SDValue t5 = DAG.getNode(ISD::FADD, dl, MVT::f32, t4,
03989                                getF32Constant(DAG, 0x3c1d8c17));
03990       SDValue t6 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t5, X);
03991       SDValue t7 = DAG.getNode(ISD::FADD, dl, MVT::f32, t6,
03992                                getF32Constant(DAG, 0x3d634a1d));
03993       SDValue t8 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t7, X);
03994       SDValue t9 = DAG.getNode(ISD::FADD, dl, MVT::f32, t8,
03995                                getF32Constant(DAG, 0x3e75fe14));
03996       SDValue t10 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t9, X);
03997       SDValue t11 = DAG.getNode(ISD::FADD, dl, MVT::f32, t10,
03998                                 getF32Constant(DAG, 0x3f317234));
03999       SDValue t12 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t11, X);
04000       TwoToFracPartOfX = DAG.getNode(ISD::FADD, dl, MVT::f32, t12,
04001                                      getF32Constant(DAG, 0x3f800000));
04002     }
04003 
04004     // Add the exponent into the result in integer domain.
04005     SDValue t13 = DAG.getNode(ISD::BITCAST, dl, MVT::i32, TwoToFracPartOfX);
04006     return DAG.getNode(ISD::BITCAST, dl, MVT::f32,
04007                        DAG.getNode(ISD::ADD, dl, MVT::i32,
04008                                    t13, IntegerPartOfX));
04009   }
04010 
04011   // No special expansion.
04012   return DAG.getNode(ISD::FEXP, dl, Op.getValueType(), Op);
04013 }
04014 
04015 /// expandLog - Lower a log intrinsic. Handles the special sequences for
04016 /// limited-precision mode.
04017 static SDValue expandLog(SDLoc dl, SDValue Op, SelectionDAG &DAG,
04018                          const TargetLowering &TLI) {
04019   if (Op.getValueType() == MVT::f32 &&
04020       LimitFloatPrecision > 0 && LimitFloatPrecision <= 18) {
04021     SDValue Op1 = DAG.getNode(ISD::BITCAST, dl, MVT::i32, Op);
04022 
04023     // Scale the exponent by log(2) [0.69314718f].
04024     SDValue Exp = GetExponent(DAG, Op1, TLI, dl);
04025     SDValue LogOfExponent = DAG.getNode(ISD::FMUL, dl, MVT::f32, Exp,
04026                                         getF32Constant(DAG, 0x3f317218));
04027 
04028     // Get the significand and build it into a floating-point number with
04029     // exponent of 1.
04030     SDValue X = GetSignificand(DAG, Op1, dl);
04031 
04032     SDValue LogOfMantissa;
04033     if (LimitFloatPrecision <= 6) {
04034       // For floating-point precision of 6:
04035       //
04036       //   LogofMantissa =
04037       //     -1.1609546f +
04038       //       (1.4034025f - 0.23903021f * x) * x;
04039       //
04040       // error 0.0034276066, which is better than 8 bits
04041       SDValue t0 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X,
04042                                getF32Constant(DAG, 0xbe74c456));
04043       SDValue t1 = DAG.getNode(ISD::FADD, dl, MVT::f32, t0,
04044                                getF32Constant(DAG, 0x3fb3a2b1));
04045       SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t1, X);
04046       LogOfMantissa = DAG.getNode(ISD::FSUB, dl, MVT::f32, t2,
04047                                   getF32Constant(DAG, 0x3f949a29));
04048     } else if (LimitFloatPrecision <= 12) {
04049       // For floating-point precision of 12:
04050       //
04051       //   LogOfMantissa =
04052       //     -1.7417939f +
04053       //       (2.8212026f +
04054       //         (-1.4699568f +
04055       //           (0.44717955f - 0.56570851e-1f * x) * x) * x) * x;
04056       //
04057       // error 0.000061011436, which is 14 bits
04058       SDValue t0 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X,
04059                                getF32Constant(DAG, 0xbd67b6d6));
04060       SDValue t1 = DAG.getNode(ISD::FADD, dl, MVT::f32, t0,
04061                                getF32Constant(DAG, 0x3ee4f4b8));
04062       SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t1, X);
04063       SDValue t3 = DAG.getNode(ISD::FSUB, dl, MVT::f32, t2,
04064                                getF32Constant(DAG, 0x3fbc278b));
04065       SDValue t4 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t3, X);
04066       SDValue t5 = DAG.getNode(ISD::FADD, dl, MVT::f32, t4,
04067                                getF32Constant(DAG, 0x40348e95));
04068       SDValue t6 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t5, X);
04069       LogOfMantissa = DAG.getNode(ISD::FSUB, dl, MVT::f32, t6,
04070                                   getF32Constant(DAG, 0x3fdef31a));
04071     } else { // LimitFloatPrecision <= 18
04072       // For floating-point precision of 18:
04073       //
04074       //   LogOfMantissa =
04075       //     -2.1072184f +
04076       //       (4.2372794f +
04077       //         (-3.7029485f +
04078       //           (2.2781945f +
04079       //             (-0.87823314f +
04080       //               (0.19073739f - 0.17809712e-1f * x) * x) * x) * x) * x)*x;
04081       //
04082       // error 0.0000023660568, which is better than 18 bits
04083       SDValue t0 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X,
04084                                getF32Constant(DAG, 0xbc91e5ac));
04085       SDValue t1 = DAG.getNode(ISD::FADD, dl, MVT::f32, t0,
04086                                getF32Constant(DAG, 0x3e4350aa));
04087       SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t1, X);
04088       SDValue t3 = DAG.getNode(ISD::FSUB, dl, MVT::f32, t2,
04089                                getF32Constant(DAG, 0x3f60d3e3));
04090       SDValue t4 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t3, X);
04091       SDValue t5 = DAG.getNode(ISD::FADD, dl, MVT::f32, t4,
04092                                getF32Constant(DAG, 0x4011cdf0));
04093       SDValue t6 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t5, X);
04094       SDValue t7 = DAG.getNode(ISD::FSUB, dl, MVT::f32, t6,
04095                                getF32Constant(DAG, 0x406cfd1c));
04096       SDValue t8 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t7, X);
04097       SDValue t9 = DAG.getNode(ISD::FADD, dl, MVT::f32, t8,
04098                                getF32Constant(DAG, 0x408797cb));
04099       SDValue t10 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t9, X);
04100       LogOfMantissa = DAG.getNode(ISD::FSUB, dl, MVT::f32, t10,
04101                                   getF32Constant(DAG, 0x4006dcab));
04102     }
04103 
04104     return DAG.getNode(ISD::FADD, dl, MVT::f32, LogOfExponent, LogOfMantissa);
04105   }
04106 
04107   // No special expansion.
04108   return DAG.getNode(ISD::FLOG, dl, Op.getValueType(), Op);
04109 }
04110 
04111 /// expandLog2 - Lower a log2 intrinsic. Handles the special sequences for
04112 /// limited-precision mode.
04113 static SDValue expandLog2(SDLoc dl, SDValue Op, SelectionDAG &DAG,
04114                           const TargetLowering &TLI) {
04115   if (Op.getValueType() == MVT::f32 &&
04116       LimitFloatPrecision > 0 && LimitFloatPrecision <= 18) {
04117     SDValue Op1 = DAG.getNode(ISD::BITCAST, dl, MVT::i32, Op);
04118 
04119     // Get the exponent.
04120     SDValue LogOfExponent = GetExponent(DAG, Op1, TLI, dl);
04121 
04122     // Get the significand and build it into a floating-point number with
04123     // exponent of 1.
04124     SDValue X = GetSignificand(DAG, Op1, dl);
04125 
04126     // Different possible minimax approximations of significand in
04127     // floating-point for various degrees of accuracy over [1,2].
04128     SDValue Log2ofMantissa;
04129     if (LimitFloatPrecision <= 6) {
04130       // For floating-point precision of 6:
04131       //
04132       //   Log2ofMantissa = -1.6749035f + (2.0246817f - .34484768f * x) * x;
04133       //
04134       // error 0.0049451742, which is more than 7 bits
04135       SDValue t0 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X,
04136                                getF32Constant(DAG, 0xbeb08fe0));
04137       SDValue t1 = DAG.getNode(ISD::FADD, dl, MVT::f32, t0,
04138                                getF32Constant(DAG, 0x40019463));
04139       SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t1, X);
04140       Log2ofMantissa = DAG.getNode(ISD::FSUB, dl, MVT::f32, t2,
04141                                    getF32Constant(DAG, 0x3fd6633d));
04142     } else if (LimitFloatPrecision <= 12) {
04143       // For floating-point precision of 12:
04144       //
04145       //   Log2ofMantissa =
04146       //     -2.51285454f +
04147       //       (4.07009056f +
04148       //         (-2.12067489f +
04149       //           (.645142248f - 0.816157886e-1f * x) * x) * x) * x;
04150       //
04151       // error 0.0000876136000, which is better than 13 bits
04152       SDValue t0 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X,
04153                                getF32Constant(DAG, 0xbda7262e));
04154       SDValue t1 = DAG.getNode(ISD::FADD, dl, MVT::f32, t0,
04155                                getF32Constant(DAG, 0x3f25280b));
04156       SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t1, X);
04157       SDValue t3 = DAG.getNode(ISD::FSUB, dl, MVT::f32, t2,
04158                                getF32Constant(DAG, 0x4007b923));
04159       SDValue t4 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t3, X);
04160       SDValue t5 = DAG.getNode(ISD::FADD, dl, MVT::f32, t4,
04161                                getF32Constant(DAG, 0x40823e2f));
04162       SDValue t6 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t5, X);
04163       Log2ofMantissa = DAG.getNode(ISD::FSUB, dl, MVT::f32, t6,
04164                                    getF32Constant(DAG, 0x4020d29c));
04165     } else { // LimitFloatPrecision <= 18
04166       // For floating-point precision of 18:
04167       //
04168       //   Log2ofMantissa =
04169       //     -3.0400495f +
04170       //       (6.1129976f +
04171       //         (-5.3420409f +
04172       //           (3.2865683f +
04173       //             (-1.2669343f +
04174       //               (0.27515199f -
04175       //                 0.25691327e-1f * x) * x) * x) * x) * x) * x;
04176       //
04177       // error 0.0000018516, which is better than 18 bits
04178       SDValue t0 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X,
04179                                getF32Constant(DAG, 0xbcd2769e));
04180       SDValue t1 = DAG.getNode(ISD::FADD, dl, MVT::f32, t0,
04181                                getF32Constant(DAG, 0x3e8ce0b9));
04182       SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t1, X);
04183       SDValue t3 = DAG.getNode(ISD::FSUB, dl, MVT::f32, t2,
04184                                getF32Constant(DAG, 0x3fa22ae7));
04185       SDValue t4 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t3, X);
04186       SDValue t5 = DAG.getNode(ISD::FADD, dl, MVT::f32, t4,
04187                                getF32Constant(DAG, 0x40525723));
04188       SDValue t6 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t5, X);
04189       SDValue t7 = DAG.getNode(ISD::FSUB, dl, MVT::f32, t6,
04190                                getF32Constant(DAG, 0x40aaf200));
04191       SDValue t8 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t7, X);
04192       SDValue t9 = DAG.getNode(ISD::FADD, dl, MVT::f32, t8,
04193                                getF32Constant(DAG, 0x40c39dad));
04194       SDValue t10 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t9, X);
04195       Log2ofMantissa = DAG.getNode(ISD::FSUB, dl, MVT::f32, t10,
04196                                    getF32Constant(DAG, 0x4042902c));
04197     }
04198 
04199     return DAG.getNode(ISD::FADD, dl, MVT::f32, LogOfExponent, Log2ofMantissa);
04200   }
04201 
04202   // No special expansion.
04203   return DAG.getNode(ISD::FLOG2, dl, Op.getValueType(), Op);
04204 }
04205 
04206 /// expandLog10 - Lower a log10 intrinsic. Handles the special sequences for
04207 /// limited-precision mode.
04208 static SDValue expandLog10(SDLoc dl, SDValue Op, SelectionDAG &DAG,
04209                            const TargetLowering &TLI) {
04210   if (Op.getValueType() == MVT::f32 &&
04211       LimitFloatPrecision > 0 && LimitFloatPrecision <= 18) {
04212     SDValue Op1 = DAG.getNode(ISD::BITCAST, dl, MVT::i32, Op);
04213 
04214     // Scale the exponent by log10(2) [0.30102999f].
04215     SDValue Exp = GetExponent(DAG, Op1, TLI, dl);
04216     SDValue LogOfExponent = DAG.getNode(ISD::FMUL, dl, MVT::f32, Exp,
04217                                         getF32Constant(DAG, 0x3e9a209a));
04218 
04219     // Get the significand and build it into a floating-point number with
04220     // exponent of 1.
04221     SDValue X = GetSignificand(DAG, Op1, dl);
04222 
04223     SDValue Log10ofMantissa;
04224     if (LimitFloatPrecision <= 6) {
04225       // For floating-point precision of 6:
04226       //
04227       //   Log10ofMantissa =
04228       //     -0.50419619f +
04229       //       (0.60948995f - 0.10380950f * x) * x;
04230       //
04231       // error 0.0014886165, which is 6 bits
04232       SDValue t0 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X,
04233                                getF32Constant(DAG, 0xbdd49a13));
04234       SDValue t1 = DAG.getNode(ISD::FADD, dl, MVT::f32, t0,
04235                                getF32Constant(DAG, 0x3f1c0789));
04236       SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t1, X);
04237       Log10ofMantissa = DAG.getNode(ISD::FSUB, dl, MVT::f32, t2,
04238                                     getF32Constant(DAG, 0x3f011300));
04239     } else if (LimitFloatPrecision <= 12) {
04240       // For floating-point precision of 12:
04241       //
04242       //   Log10ofMantissa =
04243       //     -0.64831180f +
04244       //       (0.91751397f +
04245       //         (-0.31664806f + 0.47637168e-1f * x) * x) * x;
04246       //
04247       // error 0.00019228036, which is better than 12 bits
04248       SDValue t0 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X,
04249                                getF32Constant(DAG, 0x3d431f31));
04250       SDValue t1 = DAG.getNode(ISD::FSUB, dl, MVT::f32, t0,
04251                                getF32Constant(DAG, 0x3ea21fb2));
04252       SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t1, X);
04253       SDValue t3 = DAG.getNode(ISD::FADD, dl, MVT::f32, t2,
04254                                getF32Constant(DAG, 0x3f6ae232));
04255       SDValue t4 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t3, X);
04256       Log10ofMantissa = DAG.getNode(ISD::FSUB, dl, MVT::f32, t4,
04257                                     getF32Constant(DAG, 0x3f25f7c3));
04258     } else { // LimitFloatPrecision <= 18
04259       // For floating-point precision of 18:
04260       //
04261       //   Log10ofMantissa =
04262       //     -0.84299375f +
04263       //       (1.5327582f +
04264       //         (-1.0688956f +
04265       //           (0.49102474f +
04266       //             (-0.12539807f + 0.13508273e-1f * x) * x) * x) * x) * x;
04267       //
04268       // error 0.0000037995730, which is better than 18 bits
04269       SDValue t0 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X,
04270                                getF32Constant(DAG, 0x3c5d51ce));
04271       SDValue t1 = DAG.getNode(ISD::FSUB, dl, MVT::f32, t0,
04272                                getF32Constant(DAG, 0x3e00685a));
04273       SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t1, X);
04274       SDValue t3 = DAG.getNode(ISD::FADD, dl, MVT::f32, t2,
04275                                getF32Constant(DAG, 0x3efb6798));
04276       SDValue t4 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t3, X);
04277       SDValue t5 = DAG.getNode(ISD::FSUB, dl, MVT::f32, t4,
04278                                getF32Constant(DAG, 0x3f88d192));
04279       SDValue t6 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t5, X);
04280       SDValue t7 = DAG.getNode(ISD::FADD, dl, MVT::f32, t6,
04281                                getF32Constant(DAG, 0x3fc4316c));
04282       SDValue t8 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t7, X);
04283       Log10ofMantissa = DAG.getNode(ISD::FSUB, dl, MVT::f32, t8,
04284                                     getF32Constant(DAG, 0x3f57ce70));
04285     }
04286 
04287     return DAG.getNode(ISD::FADD, dl, MVT::f32, LogOfExponent, Log10ofMantissa);
04288   }
04289 
04290   // No special expansion.
04291   return DAG.getNode(ISD::FLOG10, dl, Op.getValueType(), Op);
04292 }
04293 
04294 /// expandExp2 - Lower an exp2 intrinsic. Handles the special sequences for
04295 /// limited-precision mode.
04296 static SDValue expandExp2(SDLoc dl, SDValue Op, SelectionDAG &DAG,
04297                           const TargetLowering &TLI) {
04298   if (Op.getValueType() == MVT::f32 &&
04299       LimitFloatPrecision > 0 && LimitFloatPrecision <= 18) {
04300     SDValue IntegerPartOfX = DAG.getNode(ISD::FP_TO_SINT, dl, MVT::i32, Op);
04301 
04302     //   FractionalPartOfX = x - (float)IntegerPartOfX;
04303     SDValue t1 = DAG.getNode(ISD::SINT_TO_FP, dl, MVT::f32, IntegerPartOfX);
04304     SDValue X = DAG.getNode(ISD::FSUB, dl, MVT::f32, Op, t1);
04305 
04306     //   IntegerPartOfX <<= 23;
04307     IntegerPartOfX = DAG.getNode(ISD::SHL, dl, MVT::i32, IntegerPartOfX,
04308                                  DAG.getConstant(23, TLI.getPointerTy()));
04309 
04310     SDValue TwoToFractionalPartOfX;
04311     if (LimitFloatPrecision <= 6) {
04312       // For floating-point precision of 6:
04313       //
04314       //   TwoToFractionalPartOfX =
04315       //     0.997535578f +
04316       //       (0.735607626f + 0.252464424f * x) * x;
04317       //
04318       // error 0.0144103317, which is 6 bits
04319       SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X,
04320                                getF32Constant(DAG, 0x3e814304));
04321       SDValue t3 = DAG.getNode(ISD::FADD, dl, MVT::f32, t2,
04322                                getF32Constant(DAG, 0x3f3c50c8));
04323       SDValue t4 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t3, X);
04324       TwoToFractionalPartOfX = DAG.getNode(ISD::FADD, dl, MVT::f32, t4,
04325                                            getF32Constant(DAG, 0x3f7f5e7e));
04326     } else if (LimitFloatPrecision <= 12) {
04327       // For floating-point precision of 12:
04328       //
04329       //   TwoToFractionalPartOfX =
04330       //     0.999892986f +
04331       //       (0.696457318f +
04332       //         (0.224338339f + 0.792043434e-1f * x) * x) * x;
04333       //
04334       // error 0.000107046256, which is 13 to 14 bits
04335       SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X,
04336                                getF32Constant(DAG, 0x3da235e3));
04337       SDValue t3 = DAG.getNode(ISD::FADD, dl, MVT::f32, t2,
04338                                getF32Constant(DAG, 0x3e65b8f3));
04339       SDValue t4 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t3, X);
04340       SDValue t5 = DAG.getNode(ISD::FADD, dl, MVT::f32, t4,
04341                                getF32Constant(DAG, 0x3f324b07));
04342       SDValue t6 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t5, X);
04343       TwoToFractionalPartOfX = DAG.getNode(ISD::FADD, dl, MVT::f32, t6,
04344                                            getF32Constant(DAG, 0x3f7ff8fd));
04345     } else { // LimitFloatPrecision <= 18
04346       // For floating-point precision of 18:
04347       //
04348       //   TwoToFractionalPartOfX =
04349       //     0.999999982f +
04350       //       (0.693148872f +
04351       //         (0.240227044f +
04352       //           (0.554906021e-1f +
04353       //             (0.961591928e-2f +
04354       //               (0.136028312e-2f + 0.157059148e-3f *x)*x)*x)*x)*x)*x;
04355       // error 2.47208000*10^(-7), which is better than 18 bits
04356       SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X,
04357                                getF32Constant(DAG, 0x3924b03e));
04358       SDValue t3 = DAG.getNode(ISD::FADD, dl, MVT::f32, t2,
04359                                getF32Constant(DAG, 0x3ab24b87));
04360       SDValue t4 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t3, X);
04361       SDValue t5 = DAG.getNode(ISD::FADD, dl, MVT::f32, t4,
04362                                getF32Constant(DAG, 0x3c1d8c17));
04363       SDValue t6 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t5, X);
04364       SDValue t7 = DAG.getNode(ISD::FADD, dl, MVT::f32, t6,
04365                                getF32Constant(DAG, 0x3d634a1d));
04366       SDValue t8 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t7, X);
04367       SDValue t9 = DAG.getNode(ISD::FADD, dl, MVT::f32, t8,
04368                                getF32Constant(DAG, 0x3e75fe14));
04369       SDValue t10 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t9, X);
04370       SDValue t11 = DAG.getNode(ISD::FADD, dl, MVT::f32, t10,
04371                                 getF32Constant(DAG, 0x3f317234));
04372       SDValue t12 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t11, X);
04373       TwoToFractionalPartOfX = DAG.getNode(ISD::FADD, dl, MVT::f32, t12,
04374                                            getF32Constant(DAG, 0x3f800000));
04375     }
04376 
04377     // Add the exponent into the result in integer domain.
04378     SDValue t13 = DAG.getNode(ISD::BITCAST, dl, MVT::i32,
04379                               TwoToFractionalPartOfX);
04380     return DAG.getNode(ISD::BITCAST, dl, MVT::f32,
04381                        DAG.getNode(ISD::ADD, dl, MVT::i32,
04382                                    t13, IntegerPartOfX));
04383   }
04384 
04385   // No special expansion.
04386   return DAG.getNode(ISD::FEXP2, dl, Op.getValueType(), Op);
04387 }
04388 
04389 /// visitPow - Lower a pow intrinsic. Handles the special sequences for
04390 /// limited-precision mode with x == 10.0f.
04391 static SDValue expandPow(SDLoc dl, SDValue LHS, SDValue RHS,
04392                          SelectionDAG &DAG, const TargetLowering &TLI) {
04393   bool IsExp10 = false;
04394   if (LHS.getValueType() == MVT::f32 && RHS.getValueType() == MVT::f32 &&
04395       LimitFloatPrecision > 0 && LimitFloatPrecision <= 18) {
04396     if (ConstantFPSDNode *LHSC = dyn_cast<ConstantFPSDNode>(LHS)) {
04397       APFloat Ten(10.0f);
04398       IsExp10 = LHSC->isExactlyValue(Ten);
04399     }
04400   }
04401 
04402   if (IsExp10) {
04403     // Put the exponent in the right bit position for later addition to the
04404     // final result:
04405     //
04406     //   #define LOG2OF10 3.3219281f
04407     //   IntegerPartOfX = (int32_t)(x * LOG2OF10);
04408     SDValue t0 = DAG.getNode(ISD::FMUL, dl, MVT::f32, RHS,
04409                              getF32Constant(DAG, 0x40549a78));
04410     SDValue IntegerPartOfX = DAG.getNode(ISD::FP_TO_SINT, dl, MVT::i32, t0);
04411 
04412     //   FractionalPartOfX = x - (float)IntegerPartOfX;
04413     SDValue t1 = DAG.getNode(ISD::SINT_TO_FP, dl, MVT::f32, IntegerPartOfX);
04414     SDValue X = DAG.getNode(ISD::FSUB, dl, MVT::f32, t0, t1);
04415 
04416     //   IntegerPartOfX <<= 23;
04417     IntegerPartOfX = DAG.getNode(ISD::SHL, dl, MVT::i32, IntegerPartOfX,
04418                                  DAG.getConstant(23, TLI.getPointerTy()));
04419 
04420     SDValue TwoToFractionalPartOfX;
04421     if (LimitFloatPrecision <= 6) {
04422       // For floating-point precision of 6:
04423       //
04424       //   twoToFractionalPartOfX =
04425       //     0.997535578f +
04426       //       (0.735607626f + 0.252464424f * x) * x;
04427       //
04428       // error 0.0144103317, which is 6 bits
04429       SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X,
04430                                getF32Constant(DAG, 0x3e814304));
04431       SDValue t3 = DAG.getNode(ISD::FADD, dl, MVT::f32, t2,
04432                                getF32Constant(DAG, 0x3f3c50c8));
04433       SDValue t4 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t3, X);
04434       TwoToFractionalPartOfX = DAG.getNode(ISD::FADD, dl, MVT::f32, t4,
04435                                            getF32Constant(DAG, 0x3f7f5e7e));
04436     } else if (LimitFloatPrecision <= 12) {
04437       // For floating-point precision of 12:
04438       //
04439       //   TwoToFractionalPartOfX =
04440       //     0.999892986f +
04441       //       (0.696457318f +
04442       //         (0.224338339f + 0.792043434e-1f * x) * x) * x;
04443       //
04444       // error 0.000107046256, which is 13 to 14 bits
04445       SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X,
04446                                getF32Constant(DAG, 0x3da235e3));
04447       SDValue t3 = DAG.getNode(ISD::FADD, dl, MVT::f32, t2,
04448                                getF32Constant(DAG, 0x3e65b8f3));
04449       SDValue t4 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t3, X);
04450       SDValue t5 = DAG.getNode(ISD::FADD, dl, MVT::f32, t4,
04451                                getF32Constant(DAG, 0x3f324b07));
04452       SDValue t6 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t5, X);
04453       TwoToFractionalPartOfX = DAG.getNode(ISD::FADD, dl, MVT::f32, t6,
04454                                            getF32Constant(DAG, 0x3f7ff8fd));
04455     } else { // LimitFloatPrecision <= 18
04456       // For floating-point precision of 18:
04457       //
04458       //   TwoToFractionalPartOfX =
04459       //     0.999999982f +
04460       //       (0.693148872f +
04461       //         (0.240227044f +
04462       //           (0.554906021e-1f +
04463       //             (0.961591928e-2f +
04464       //               (0.136028312e-2f + 0.157059148e-3f *x)*x)*x)*x)*x)*x;
04465       // error 2.47208000*10^(-7), which is better than 18 bits
04466       SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X,
04467                                getF32Constant(DAG, 0x3924b03e));
04468       SDValue t3 = DAG.getNode(ISD::FADD, dl, MVT::f32, t2,
04469                                getF32Constant(DAG, 0x3ab24b87));
04470       SDValue t4 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t3, X);
04471       SDValue t5 = DAG.getNode(ISD::FADD, dl, MVT::f32, t4,
04472                                getF32Constant(DAG, 0x3c1d8c17));
04473       SDValue t6 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t5, X);
04474       SDValue t7 = DAG.getNode(ISD::FADD, dl, MVT::f32, t6,
04475                                getF32Constant(DAG, 0x3d634a1d));
04476       SDValue t8 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t7, X);
04477       SDValue t9 = DAG.getNode(ISD::FADD, dl, MVT::f32, t8,
04478                                getF32Constant(DAG, 0x3e75fe14));
04479       SDValue t10 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t9, X);
04480       SDValue t11 = DAG.getNode(ISD::FADD, dl, MVT::f32, t10,
04481                                 getF32Constant(DAG, 0x3f317234));
04482       SDValue t12 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t11, X);
04483       TwoToFractionalPartOfX = DAG.getNode(ISD::FADD, dl, MVT::f32, t12,
04484                                            getF32Constant(DAG, 0x3f800000));
04485     }
04486 
04487     SDValue t13 = DAG.getNode(ISD::BITCAST, dl,MVT::i32,TwoToFractionalPartOfX);
04488     return DAG.getNode(ISD::BITCAST, dl, MVT::f32,
04489                        DAG.getNode(ISD::ADD, dl, MVT::i32,
04490                                    t13, IntegerPartOfX));
04491   }
04492 
04493   // No special expansion.
04494   return DAG.getNode(ISD::FPOW, dl, LHS.getValueType(), LHS, RHS);
04495 }
04496 
04497 
04498 /// ExpandPowI - Expand a llvm.powi intrinsic.
04499 static SDValue ExpandPowI(SDLoc DL, SDValue LHS, SDValue RHS,
04500                           SelectionDAG &DAG) {
04501   // If RHS is a constant, we can expand this out to a multiplication tree,
04502   // otherwise we end up lowering to a call to __powidf2 (for example).  When
04503   // optimizing for size, we only want to do this if the expansion would produce
04504   // a small number of multiplies, otherwise we do the full expansion.
04505   if (ConstantSDNode *RHSC = dyn_cast<ConstantSDNode>(RHS)) {
04506     // Get the exponent as a positive value.
04507     unsigned Val = RHSC->getSExtValue();
04508     if ((int)Val < 0) Val = -Val;
04509 
04510     // powi(x, 0) -> 1.0
04511     if (Val == 0)
04512       return DAG.getConstantFP(1.0, LHS.getValueType());
04513 
04514     const Function *F = DAG.getMachineFunction().getFunction();
04515     if (!F->getAttributes().hasAttribute(AttributeSet::FunctionIndex,
04516                                          Attribute::OptimizeForSize) ||
04517         // If optimizing for size, don't insert too many multiplies.  This
04518         // inserts up to 5 multiplies.
04519         CountPopulation_32(Val)+Log2_32(Val) < 7) {
04520       // We use the simple binary decomposition method to generate the multiply
04521       // sequence.  There are more optimal ways to do this (for example,
04522       // powi(x,15) generates one more multiply than it should), but this has
04523       // the benefit of being both really simple and much better than a libcall.
04524       SDValue Res;  // Logically starts equal to 1.0
04525       SDValue CurSquare = LHS;
04526       while (Val) {
04527         if (Val & 1) {
04528           if (Res.getNode())
04529             Res = DAG.getNode(ISD::FMUL, DL,Res.getValueType(), Res, CurSquare);
04530           else
04531             Res = CurSquare;  // 1.0*CurSquare.
04532         }
04533 
04534         CurSquare = DAG.getNode(ISD::FMUL, DL, CurSquare.getValueType(),
04535                                 CurSquare, CurSquare);
04536         Val >>= 1;
04537       }
04538 
04539       // If the original was negative, invert the result, producing 1/(x*x*x).
04540       if (RHSC->getSExtValue() < 0)
04541         Res = DAG.getNode(ISD::FDIV, DL, LHS.getValueType(),
04542                           DAG.getConstantFP(1.0, LHS.getValueType()), Res);
04543       return Res;
04544     }
04545   }
04546 
04547   // Otherwise, expand to a libcall.
04548   return DAG.getNode(ISD::FPOWI, DL, LHS.getValueType(), LHS, RHS);
04549 }
04550 
04551 // getTruncatedArgReg - Find underlying register used for an truncated
04552 // argument.
04553 static unsigned getTruncatedArgReg(const SDValue &N) {
04554   if (N.getOpcode() != ISD::TRUNCATE)
04555     return 0;
04556 
04557   const SDValue &Ext = N.getOperand(0);
04558   if (Ext.getOpcode() == ISD::AssertZext ||
04559       Ext.getOpcode() == ISD::AssertSext) {
04560     const SDValue &CFR = Ext.getOperand(0);
04561     if (CFR.getOpcode() == ISD::CopyFromReg)
04562       return cast<RegisterSDNode>(CFR.getOperand(1))->getReg();
04563     if (CFR.getOpcode() == ISD::TRUNCATE)
04564       return getTruncatedArgReg(CFR);
04565   }
04566   return 0;
04567 }
04568 
04569 /// EmitFuncArgumentDbgValue - If the DbgValueInst is a dbg_value of a function
04570 /// argument, create the corresponding DBG_VALUE machine instruction for it now.
04571 /// At the end of instruction selection, they will be inserted to the entry BB.
04572 bool
04573 SelectionDAGBuilder::EmitFuncArgumentDbgValue(const Value *V, MDNode *Variable,
04574                                               int64_t Offset, bool IsIndirect,
04575                                               const SDValue &N) {
04576   const Argument *Arg = dyn_cast<Argument>(V);
04577   if (!Arg)
04578     return false;
04579 
04580   MachineFunction &MF = DAG.getMachineFunction();
04581   const TargetInstrInfo *TII = DAG.getTarget().getInstrInfo();
04582 
04583   // Ignore inlined function arguments here.
04584   DIVariable DV(Variable);
04585   if (DV.isInlinedFnArgument(MF.getFunction()))
04586     return false;
04587 
04588   Optional<MachineOperand> Op;
04589   // Some arguments' frame index is recorded during argument lowering.
04590   if (int FI = FuncInfo.getArgumentFrameIndex(Arg))
04591     Op = MachineOperand::CreateFI(FI);
04592 
04593   if (!Op && N.getNode()) {
04594     unsigned Reg;
04595     if (N.getOpcode() == ISD::CopyFromReg)
04596       Reg = cast<RegisterSDNode>(N.getOperand(1))->getReg();
04597     else
04598       Reg = getTruncatedArgReg(N);
04599     if (Reg && TargetRegisterInfo::isVirtualRegister(Reg)) {
04600       MachineRegisterInfo &RegInfo = MF.getRegInfo();
04601       unsigned PR = RegInfo.getLiveInPhysReg(Reg);
04602       if (PR)
04603         Reg = PR;
04604     }
04605     if (Reg)
04606       Op = MachineOperand::CreateReg(Reg, false);
04607   }
04608 
04609   if (!Op) {
04610     // Check if ValueMap has reg number.
04611     DenseMap<const Value *, unsigned>::iterator VMI = FuncInfo.ValueMap.find(V);
04612     if (VMI != FuncInfo.ValueMap.end())
04613       Op = MachineOperand::CreateReg(VMI->second, false);
04614   }
04615 
04616   if (!Op && N.getNode())
04617     // Check if frame index is available.
04618     if (LoadSDNode *LNode = dyn_cast<LoadSDNode>(N.getNode()))
04619       if (FrameIndexSDNode *FINode =
04620           dyn_cast<FrameIndexSDNode>(LNode->getBasePtr().getNode()))
04621         Op = MachineOperand::CreateFI(FINode->getIndex());
04622 
04623   if (!Op)
04624     return false;
04625 
04626   if (Op->isReg())
04627     FuncInfo.ArgDbgValues.push_back(BuildMI(MF, getCurDebugLoc(),
04628                                             TII->get(TargetOpcode::DBG_VALUE),
04629                                             IsIndirect,
04630                                             Op->getReg(), Offset, Variable));
04631   else
04632     FuncInfo.ArgDbgValues.push_back(
04633       BuildMI(MF, getCurDebugLoc(), TII->get(TargetOpcode::DBG_VALUE))
04634           .addOperand(*Op).addImm(Offset).addMetadata(Variable));
04635 
04636   return true;
04637 }
04638 
04639 // VisualStudio defines setjmp as _setjmp
04640 #if defined(_MSC_VER) && defined(setjmp) && \
04641                          !defined(setjmp_undefined_for_msvc)
04642 #  pragma push_macro("setjmp")
04643 #  undef setjmp
04644 #  define setjmp_undefined_for_msvc
04645 #endif
04646 
04647 /// visitIntrinsicCall - Lower the call to the specified intrinsic function.  If
04648 /// we want to emit this as a call to a named external function, return the name
04649 /// otherwise lower it and return null.
04650 const char *
04651 SelectionDAGBuilder::visitIntrinsicCall(const CallInst &I, unsigned Intrinsic) {
04652   const TargetLowering *TLI = TM.getTargetLowering();
04653   SDLoc sdl = getCurSDLoc();
04654   DebugLoc dl = getCurDebugLoc();
04655   SDValue Res;
04656 
04657   switch (Intrinsic) {
04658   default:
04659     // By default, turn this into a target intrinsic node.
04660     visitTargetIntrinsic(I, Intrinsic);
04661     return nullptr;
04662   case Intrinsic::vastart:  visitVAStart(I); return nullptr;
04663   case Intrinsic::vaend:    visitVAEnd(I); return nullptr;
04664   case Intrinsic::vacopy:   visitVACopy(I); return nullptr;
04665   case Intrinsic::returnaddress:
04666     setValue(&I, DAG.getNode(ISD::RETURNADDR, sdl, TLI->getPointerTy(),
04667                              getValue(I.getArgOperand(0))));
04668     return nullptr;
04669   case Intrinsic::frameaddress:
04670     setValue(&I, DAG.getNode(ISD::FRAMEADDR, sdl, TLI->getPointerTy(),
04671                              getValue(I.getArgOperand(0))));
04672     return nullptr;
04673   case Intrinsic::read_register: {
04674     Value *Reg = I.getArgOperand(0);
04675     SDValue RegName = DAG.getMDNode(cast<MDNode>(Reg));
04676     EVT VT = TM.getTargetLowering()->getValueType(I.getType());
04677     setValue(&I, DAG.getNode(ISD::READ_REGISTER, sdl, VT, RegName));
04678     return nullptr;
04679   }
04680   case Intrinsic::write_register: {
04681     Value *Reg = I.getArgOperand(0);
04682     Value *RegValue = I.getArgOperand(1);
04683     SDValue Chain = getValue(RegValue).getOperand(0);
04684     SDValue RegName = DAG.getMDNode(cast<MDNode>(Reg));
04685     DAG.setRoot(DAG.getNode(ISD::WRITE_REGISTER, sdl, MVT::Other, Chain,
04686                             RegName, getValue(RegValue)));
04687     return nullptr;
04688   }
04689   case Intrinsic::setjmp:
04690     return &"_setjmp"[!TLI->usesUnderscoreSetJmp()];
04691   case Intrinsic::longjmp:
04692     return &"_longjmp"[!TLI->usesUnderscoreLongJmp()];
04693   case Intrinsic::memcpy: {
04694     // Assert for address < 256 since we support only user defined address
04695     // spaces.
04696     assert(cast<PointerType>(I.getArgOperand(0)->getType())->getAddressSpace()
04697            < 256 &&
04698            cast<PointerType>(I.getArgOperand(1)->getType())->getAddressSpace()
04699            < 256 &&
04700            "Unknown address space");
04701     SDValue Op1 = getValue(I.getArgOperand(0));
04702     SDValue Op2 = getValue(I.getArgOperand(1));
04703     SDValue Op3 = getValue(I.getArgOperand(2));
04704     unsigned Align = cast<ConstantInt>(I.getArgOperand(3))->getZExtValue();
04705     if (!Align)
04706       Align = 1; // @llvm.memcpy defines 0 and 1 to both mean no alignment.
04707     bool isVol = cast<ConstantInt>(I.getArgOperand(4))->getZExtValue();
04708     DAG.setRoot(DAG.getMemcpy(getRoot(), sdl, Op1, Op2, Op3, Align, isVol, false,
04709                               MachinePointerInfo(I.getArgOperand(0)),
04710                               MachinePointerInfo(I.getArgOperand(1))));
04711     return nullptr;
04712   }
04713   case Intrinsic::memset: {
04714     // Assert for address < 256 since we support only user defined address
04715     // spaces.
04716     assert(cast<PointerType>(I.getArgOperand(0)->getType())->getAddressSpace()
04717            < 256 &&
04718            "Unknown address space");
04719     SDValue Op1 = getValue(I.getArgOperand(0));
04720     SDValue Op2 = getValue(I.getArgOperand(1));
04721     SDValue Op3 = getValue(I.getArgOperand(2));
04722     unsigned Align = cast<ConstantInt>(I.getArgOperand(3))->getZExtValue();
04723     if (!Align)
04724       Align = 1; // @llvm.memset defines 0 and 1 to both mean no alignment.
04725     bool isVol = cast<ConstantInt>(I.getArgOperand(4))->getZExtValue();
04726     DAG.setRoot(DAG.getMemset(getRoot(), sdl, Op1, Op2, Op3, Align, isVol,
04727                               MachinePointerInfo(I.getArgOperand(0))));
04728     return nullptr;
04729   }
04730   case Intrinsic::memmove: {
04731     // Assert for address < 256 since we support only user defined address
04732     // spaces.
04733     assert(cast<PointerType>(I.getArgOperand(0)->getType())->getAddressSpace()
04734            < 256 &&
04735            cast<PointerType>(I.getArgOperand(1)->getType())->getAddressSpace()
04736            < 256 &&
04737            "Unknown address space");
04738     SDValue Op1 = getValue(I.getArgOperand(0));
04739     SDValue Op2 = getValue(I.getArgOperand(1));
04740     SDValue Op3 = getValue(I.getArgOperand(2));
04741     unsigned Align = cast<ConstantInt>(I.getArgOperand(3))->getZExtValue();
04742     if (!Align)
04743       Align = 1; // @llvm.memmove defines 0 and 1 to both mean no alignment.
04744     bool isVol = cast<ConstantInt>(I.getArgOperand(4))->getZExtValue();
04745     DAG.setRoot(DAG.getMemmove(getRoot(), sdl, Op1, Op2, Op3, Align, isVol,
04746                                MachinePointerInfo(I.getArgOperand(0)),
04747                                MachinePointerInfo(I.getArgOperand(1))));
04748     return nullptr;
04749   }
04750   case Intrinsic::dbg_declare: {
04751     const DbgDeclareInst &DI = cast<DbgDeclareInst>(I);
04752     MDNode *Variable = DI.getVariable();
04753     const Value *Address = DI.getAddress();
04754     DIVariable DIVar(Variable);
04755     assert((!DIVar || DIVar.isVariable()) &&
04756       "Variable in DbgDeclareInst should be either null or a DIVariable.");
04757     if (!Address || !DIVar) {
04758       DEBUG(dbgs() << "Dropping debug info for " << DI << "\n");
04759       return nullptr;
04760     }
04761 
04762     // Check if address has undef value.
04763     if (isa<UndefValue>(Address) ||
04764         (Address->use_empty() && !isa<Argument>(Address))) {
04765       DEBUG(dbgs() << "Dropping debug info for " << DI << "\n");
04766       return nullptr;
04767     }
04768 
04769     SDValue &N = NodeMap[Address];
04770     if (!N.getNode() && isa<Argument>(Address))
04771       // Check unused arguments map.
04772       N = UnusedArgNodeMap[Address];
04773     SDDbgValue *SDV;
04774     if (N.getNode()) {
04775       if (const BitCastInst *BCI = dyn_cast<BitCastInst>(Address))
04776         Address = BCI->getOperand(0);
04777       // Parameters are handled specially.
04778       bool isParameter =
04779         (DIVariable(Variable).getTag() == dwarf::DW_TAG_arg_variable ||
04780          isa<Argument>(Address));
04781 
04782       const AllocaInst *AI = dyn_cast<AllocaInst>(Address);
04783 
04784       if (isParameter && !AI) {
04785         FrameIndexSDNode *FINode = dyn_cast<FrameIndexSDNode>(N.getNode());
04786         if (FINode)
04787           // Byval parameter.  We have a frame index at this point.
04788           SDV = DAG.getFrameIndexDbgValue(Variable, FINode->getIndex(),
04789             0, dl, SDNodeOrder);
04790         else {
04791           // Address is an argument, so try to emit its dbg value using
04792           // virtual register info from the FuncInfo.ValueMap.
04793           EmitFuncArgumentDbgValue(Address, Variable, 0, false, N);
04794           return nullptr;
04795         }
04796       } else if (AI)
04797         SDV = DAG.getDbgValue(Variable, N.getNode(), N.getResNo(),
04798                               true, 0, dl, SDNodeOrder);
04799       else {
04800         // Can't do anything with other non-AI cases yet.
04801         DEBUG(dbgs() << "Dropping debug info for " << DI << "\n");
04802         DEBUG(dbgs() << "non-AllocaInst issue for Address: \n\t");
04803         DEBUG(Address->dump());
04804         return nullptr;
04805       }
04806       DAG.AddDbgValue(SDV, N.getNode(), isParameter);
04807     } else {
04808       // If Address is an argument then try to emit its dbg value using
04809       // virtual register info from the FuncInfo.ValueMap.
04810       if (!EmitFuncArgumentDbgValue(Address, Variable, 0, false, N)) {
04811         // If variable is pinned by a alloca in dominating bb then
04812         // use StaticAllocaMap.
04813         if (const AllocaInst *AI = dyn_cast<AllocaInst>(Address)) {
04814           if (AI->getParent() != DI.getParent()) {
04815             DenseMap<const AllocaInst*, int>::iterator SI =
04816               FuncInfo.StaticAllocaMap.find(AI);
04817             if (SI != FuncInfo.StaticAllocaMap.end()) {
04818               SDV = DAG.getFrameIndexDbgValue(Variable, SI->second,
04819                                               0, dl, SDNodeOrder);
04820               DAG.AddDbgValue(SDV, nullptr, false);
04821               return nullptr;
04822             }
04823           }
04824         }
04825         DEBUG(dbgs() << "Dropping debug info for " << DI << "\n");
04826       }
04827     }
04828     return nullptr;
04829   }
04830   case Intrinsic::dbg_value: {
04831     const DbgValueInst &DI = cast<DbgValueInst>(I);
04832     DIVariable DIVar(DI.getVariable());
04833     assert((!DIVar || DIVar.isVariable()) &&
04834       "Variable in DbgValueInst should be either null or a DIVariable.");
04835     if (!DIVar)
04836       return nullptr;
04837 
04838     MDNode *Variable = DI.getVariable();
04839     uint64_t Offset = DI.getOffset();
04840     const Value *V = DI.getValue();
04841     if (!V)
04842       return nullptr;
04843 
04844     SDDbgValue *SDV;
04845     if (isa<ConstantInt>(V) || isa<ConstantFP>(V) || isa<UndefValue>(V)) {
04846       SDV = DAG.getConstantDbgValue(Variable, V, Offset, dl, SDNodeOrder);
04847       DAG.AddDbgValue(SDV, nullptr, false);
04848     } else {
04849       // Do not use getValue() in here; we don't want to generate code at
04850       // this point if it hasn't been done yet.
04851       SDValue N = NodeMap[V];
04852       if (!N.getNode() && isa<Argument>(V))
04853         // Check unused arguments map.
04854         N = UnusedArgNodeMap[V];
04855       if (N.getNode()) {
04856         // A dbg.value for an alloca is always indirect.
04857         bool IsIndirect = isa<AllocaInst>(V) || Offset != 0;
04858         if (!EmitFuncArgumentDbgValue(V, Variable, Offset, IsIndirect, N)) {
04859           SDV = DAG.getDbgValue(Variable, N.getNode(),
04860                                 N.getResNo(), IsIndirect,
04861         Offset, dl, SDNodeOrder);
04862           DAG.AddDbgValue(SDV, N.getNode(), false);
04863         }
04864       } else if (!V->use_empty() ) {
04865         // Do not call getValue(V) yet, as we don't want to generate code.
04866         // Remember it for later.
04867         DanglingDebugInfo DDI(&DI, dl, SDNodeOrder);
04868         DanglingDebugInfoMap[V] = DDI;
04869       } else {
04870         // We may expand this to cover more cases.  One case where we have no
04871         // data available is an unreferenced parameter.
04872         DEBUG(dbgs() << "Dropping debug info for " << DI << "\n");
04873       }
04874     }
04875 
04876     // Build a debug info table entry.
04877     if (const BitCastInst *BCI = dyn_cast<BitCastInst>(V))
04878       V = BCI->getOperand(0);
04879     const AllocaInst *AI = dyn_cast<AllocaInst>(V);
04880     // Don't handle byval struct arguments or VLAs, for example.
04881     if (!AI) {
04882       DEBUG(dbgs() << "Dropping debug location info for:\n  " << DI << "\n");
04883       DEBUG(dbgs() << "  Last seen at:\n    " << *V << "\n");
04884       return nullptr;
04885     }
04886     DenseMap<const AllocaInst*, int>::iterator SI =
04887       FuncInfo.StaticAllocaMap.find(AI);
04888     if (SI == FuncInfo.StaticAllocaMap.end())
04889       return nullptr; // VLAs.
04890     return nullptr;
04891   }
04892 
04893   case Intrinsic::eh_typeid_for: {
04894     // Find the type id for the given typeinfo.
04895     GlobalVariable *GV = ExtractTypeInfo(I.getArgOperand(0));
04896     unsigned TypeID = DAG.getMachineFunction().getMMI().getTypeIDFor(GV);
04897     Res = DAG.getConstant(TypeID, MVT::i32);
04898     setValue(&I, Res);
04899     return nullptr;
04900   }
04901 
04902   case Intrinsic::eh_return_i32:
04903   case Intrinsic::eh_return_i64:
04904     DAG.getMachineFunction().getMMI().setCallsEHReturn(true);
04905     DAG.setRoot(DAG.getNode(ISD::EH_RETURN, sdl,
04906                             MVT::Other,
04907                             getControlRoot(),
04908                             getValue(I.getArgOperand(0)),
04909                             getValue(I.getArgOperand(1))));
04910     return nullptr;
04911   case Intrinsic::eh_unwind_init:
04912     DAG.getMachineFunction().getMMI().setCallsUnwindInit(true);
04913     return nullptr;
04914   case Intrinsic::eh_dwarf_cfa: {
04915     SDValue CfaArg = DAG.getSExtOrTrunc(getValue(I.getArgOperand(0)), sdl,
04916                                         TLI->getPointerTy());
04917     SDValue Offset = DAG.getNode(ISD::ADD, sdl,
04918                                  CfaArg.getValueType(),
04919                                  DAG.getNode(ISD::FRAME_TO_ARGS_OFFSET, sdl,
04920                                              CfaArg.getValueType()),
04921                                  CfaArg);
04922     SDValue FA = DAG.getNode(ISD::FRAMEADDR, sdl,
04923                              TLI->getPointerTy(),
04924                              DAG.getConstant(0, TLI->getPointerTy()));
04925     setValue(&I, DAG.getNode(ISD::ADD, sdl, FA.getValueType(),
04926                              FA, Offset));
04927     return nullptr;
04928   }
04929   case Intrinsic::eh_sjlj_callsite: {
04930     MachineModuleInfo &MMI = DAG.getMachineFunction().getMMI();
04931     ConstantInt *CI = dyn_cast<ConstantInt>(I.getArgOperand(0));
04932     assert(CI && "Non-constant call site value in eh.sjlj.callsite!");
04933     assert(MMI.getCurrentCallSite() == 0 && "Overlapping call sites!");
04934 
04935     MMI.setCurrentCallSite(CI->getZExtValue());
04936     return nullptr;
04937   }
04938   case Intrinsic::eh_sjlj_functioncontext: {
04939     // Get and store the index of the function context.
04940     MachineFrameInfo *MFI = DAG.getMachineFunction().getFrameInfo();
04941     AllocaInst *FnCtx =
04942       cast<AllocaInst>(I.getArgOperand(0)->stripPointerCasts());
04943     int FI = FuncInfo.StaticAllocaMap[FnCtx];
04944     MFI->setFunctionContextIndex(FI);
04945     return nullptr;
04946   }
04947   case Intrinsic::eh_sjlj_setjmp: {
04948     SDValue Ops[2];
04949     Ops[0] = getRoot();
04950     Ops[1] = getValue(I.getArgOperand(0));
04951     SDValue Op = DAG.getNode(ISD::EH_SJLJ_SETJMP, sdl,
04952                              DAG.getVTList(MVT::i32, MVT::Other), Ops);
04953     setValue(&I, Op.getValue(0));
04954     DAG.setRoot(Op.getValue(1));
04955     return nullptr;
04956   }
04957   case Intrinsic::eh_sjlj_longjmp: {
04958     DAG.setRoot(DAG.getNode(ISD::EH_SJLJ_LONGJMP, sdl, MVT::Other,
04959                             getRoot(), getValue(I.getArgOperand(0))));
04960     return nullptr;
04961   }
04962 
04963   case Intrinsic::x86_mmx_pslli_w:
04964   case Intrinsic::x86_mmx_pslli_d:
04965   case Intrinsic::x86_mmx_pslli_q:
04966   case Intrinsic::x86_mmx_psrli_w:
04967   case Intrinsic::x86_mmx_psrli_d:
04968   case Intrinsic::x86_mmx_psrli_q:
04969   case Intrinsic::x86_mmx_psrai_w:
04970   case Intrinsic::x86_mmx_psrai_d: {
04971     SDValue ShAmt = getValue(I.getArgOperand(1));
04972     if (isa<ConstantSDNode>(ShAmt)) {
04973       visitTargetIntrinsic(I, Intrinsic);
04974       return nullptr;
04975     }
04976     unsigned NewIntrinsic = 0;
04977     EVT ShAmtVT = MVT::v2i32;
04978     switch (Intrinsic) {
04979     case Intrinsic::x86_mmx_pslli_w:
04980       NewIntrinsic = Intrinsic::x86_mmx_psll_w;
04981       break;
04982     case Intrinsic::x86_mmx_pslli_d:
04983       NewIntrinsic = Intrinsic::x86_mmx_psll_d;
04984       break;
04985     case Intrinsic::x86_mmx_pslli_q:
04986       NewIntrinsic = Intrinsic::x86_mmx_psll_q;
04987       break;
04988     case Intrinsic::x86_mmx_psrli_w:
04989       NewIntrinsic = Intrinsic::x86_mmx_psrl_w;
04990       break;
04991     case Intrinsic::x86_mmx_psrli_d:
04992       NewIntrinsic = Intrinsic::x86_mmx_psrl_d;
04993       break;
04994     case Intrinsic::x86_mmx_psrli_q:
04995       NewIntrinsic = Intrinsic::x86_mmx_psrl_q;
04996       break;
04997     case Intrinsic::x86_mmx_psrai_w:
04998       NewIntrinsic = Intrinsic::x86_mmx_psra_w;
04999       break;
05000     case Intrinsic::x86_mmx_psrai_d:
05001       NewIntrinsic = Intrinsic::x86_mmx_psra_d;
05002       break;
05003     default: llvm_unreachable("Impossible intrinsic");  // Can't reach here.
05004     }
05005 
05006     // The vector shift intrinsics with scalars uses 32b shift amounts but
05007     // the sse2/mmx shift instructions reads 64 bits. Set the upper 32 bits
05008     // to be zero.
05009     // We must do this early because v2i32 is not a legal type.
05010     SDValue ShOps[2];
05011     ShOps[0] = ShAmt;
05012     ShOps[1] = DAG.getConstant(0, MVT::i32);
05013     ShAmt =  DAG.getNode(ISD::BUILD_VECTOR, sdl, ShAmtVT, ShOps);
05014     EVT DestVT = TLI->getValueType(I.getType());
05015     ShAmt = DAG.getNode(ISD::BITCAST, sdl, DestVT, ShAmt);
05016     Res = DAG.getNode(ISD::INTRINSIC_WO_CHAIN, sdl, DestVT,
05017                        DAG.getConstant(NewIntrinsic, MVT::i32),
05018                        getValue(I.getArgOperand(0)), ShAmt);
05019     setValue(&I, Res);
05020     return nullptr;
05021   }
05022   case Intrinsic::x86_avx_vinsertf128_pd_256:
05023   case Intrinsic::x86_avx_vinsertf128_ps_256:
05024   case Intrinsic::x86_avx_vinsertf128_si_256:
05025   case Intrinsic::x86_avx2_vinserti128: {
05026     EVT DestVT = TLI->getValueType(I.getType());
05027     EVT ElVT = TLI->getValueType(I.getArgOperand(1)->getType());
05028     uint64_t Idx = (cast<ConstantInt>(I.getArgOperand(2))->getZExtValue() & 1) *
05029                    ElVT.getVectorNumElements();
05030     Res = DAG.getNode(ISD::INSERT_SUBVECTOR, sdl, DestVT,
05031                       getValue(I.getArgOperand(0)),
05032                       getValue(I.getArgOperand(1)),
05033                       DAG.getConstant(Idx, TLI->getVectorIdxTy()));
05034     setValue(&I, Res);
05035     return nullptr;
05036   }
05037   case Intrinsic::x86_avx_vextractf128_pd_256:
05038   case Intrinsic::x86_avx_vextractf128_ps_256:
05039   case Intrinsic::x86_avx_vextractf128_si_256:
05040   case Intrinsic::x86_avx2_vextracti128: {
05041     EVT DestVT = TLI->getValueType(I.getType());
05042     uint64_t Idx = (cast<ConstantInt>(I.getArgOperand(1))->getZExtValue() & 1) *
05043                    DestVT.getVectorNumElements();
05044     Res = DAG.getNode(ISD::EXTRACT_SUBVECTOR, sdl, DestVT,
05045                       getValue(I.getArgOperand(0)),
05046                       DAG.getConstant(Idx, TLI->getVectorIdxTy()));
05047     setValue(&I, Res);
05048     return nullptr;
05049   }
05050   case Intrinsic::convertff:
05051   case Intrinsic::convertfsi:
05052   case Intrinsic::convertfui:
05053   case Intrinsic::convertsif:
05054   case Intrinsic::convertuif:
05055   case Intrinsic::convertss:
05056   case Intrinsic::convertsu:
05057   case Intrinsic::convertus:
05058   case Intrinsic::convertuu: {
05059     ISD::CvtCode Code = ISD::CVT_INVALID;
05060     switch (Intrinsic) {
05061     default: llvm_unreachable("Impossible intrinsic");  // Can't reach here.
05062     case Intrinsic::convertff:  Code = ISD::CVT_FF; break;
05063     case Intrinsic::convertfsi: Code = ISD::CVT_FS; break;
05064     case Intrinsic::convertfui: Code = ISD::CVT_FU; break;
05065     case Intrinsic::convertsif: Code = ISD::CVT_SF; break;
05066     case Intrinsic::convertuif: Code = ISD::CVT_UF; break;
05067     case Intrinsic::convertss:  Code = ISD::CVT_SS; break;
05068     case Intrinsic::convertsu:  Code = ISD::CVT_SU; break;
05069     case Intrinsic::convertus:  Code = ISD::CVT_US; break;
05070     case Intrinsic::convertuu:  Code = ISD::CVT_UU; break;
05071     }
05072     EVT DestVT = TLI->getValueType(I.getType());
05073     const Value *Op1 = I.getArgOperand(0);
05074     Res = DAG.getConvertRndSat(DestVT, sdl, getValue(Op1),
05075                                DAG.getValueType(DestVT),
05076                                DAG.getValueType(getValue(Op1).getValueType()),
05077                                getValue(I.getArgOperand(1)),
05078                                getValue(I.getArgOperand(2)),
05079                                Code);
05080     setValue(&I, Res);
05081     return nullptr;
05082   }
05083   case Intrinsic::powi:
05084     setValue(&I, ExpandPowI(sdl, getValue(I.getArgOperand(0)),
05085                             getValue(I.getArgOperand(1)), DAG));
05086     return nullptr;
05087   case Intrinsic::log:
05088     setValue(&I, expandLog(sdl, getValue(I.getArgOperand(0)), DAG, *TLI));
05089     return nullptr;
05090   case Intrinsic::log2:
05091     setValue(&I, expandLog2(sdl, getValue(I.getArgOperand(0)), DAG, *TLI));
05092     return nullptr;
05093   case Intrinsic::log10:
05094     setValue(&I, expandLog10(sdl, getValue(I.getArgOperand(0)), DAG, *TLI));
05095     return nullptr;
05096   case Intrinsic::exp:
05097     setValue(&I, expandExp(sdl, getValue(I.getArgOperand(0)), DAG, *TLI));
05098     return nullptr;
05099   case Intrinsic::exp2:
05100     setValue(&I, expandExp2(sdl, getValue(I.getArgOperand(0)), DAG, *TLI));
05101     return nullptr;
05102   case Intrinsic::pow:
05103     setValue(&I, expandPow(sdl, getValue(I.getArgOperand(0)),
05104                            getValue(I.getArgOperand(1)), DAG, *TLI));
05105     return nullptr;
05106   case Intrinsic::sqrt:
05107   case Intrinsic::fabs:
05108   case Intrinsic::sin:
05109   case Intrinsic::cos:
05110   case Intrinsic::floor:
05111   case Intrinsic::ceil:
05112   case Intrinsic::trunc:
05113   case Intrinsic::rint:
05114   case Intrinsic::nearbyint:
05115   case Intrinsic::round: {
05116     unsigned Opcode;
05117     switch (Intrinsic) {
05118     default: llvm_unreachable("Impossible intrinsic");  // Can't reach here.
05119     case Intrinsic::sqrt:      Opcode = ISD::FSQRT;      break;
05120     case Intrinsic::fabs:      Opcode = ISD::FABS;       break;
05121     case Intrinsic::sin:       Opcode = ISD::FSIN;       break;
05122     case Intrinsic::cos:       Opcode = ISD::FCOS;       break;
05123     case Intrinsic::floor:     Opcode = ISD::FFLOOR;     break;
05124     case Intrinsic::ceil:      Opcode = ISD::FCEIL;      break;
05125     case Intrinsic::trunc:     Opcode = ISD::FTRUNC;     break;
05126     case Intrinsic::rint:      Opcode = ISD::FRINT;      break;
05127     case Intrinsic::nearbyint: Opcode = ISD::FNEARBYINT; break;
05128     case Intrinsic::round:     Opcode = ISD::FROUND;     break;
05129     }
05130 
05131     setValue(&I, DAG.getNode(Opcode, sdl,
05132                              getValue(I.getArgOperand(0)).getValueType(),
05133                              getValue(I.getArgOperand(0))));
05134     return nullptr;
05135   }
05136   case Intrinsic::copysign:
05137     setValue(&I, DAG.getNode(ISD::FCOPYSIGN, sdl,
05138                              getValue(I.getArgOperand(0)).getValueType(),
05139                              getValue(I.getArgOperand(0)),
05140                              getValue(I.getArgOperand(1))));
05141     return nullptr;
05142   case Intrinsic::fma:
05143     setValue(&I, DAG.getNode(ISD::FMA, sdl,
05144                              getValue(I.getArgOperand(0)).getValueType(),
05145                              getValue(I.getArgOperand(0)),
05146                              getValue(I.getArgOperand(1)),
05147                              getValue(I.getArgOperand(2))));
05148     return nullptr;
05149   case Intrinsic::fmuladd: {
05150     EVT VT = TLI->getValueType(I.getType());
05151     if (TM.Options.AllowFPOpFusion != FPOpFusion::Strict &&
05152         TLI->isFMAFasterThanFMulAndFAdd(VT)) {
05153       setValue(&I, DAG.getNode(ISD::FMA, sdl,
05154                                getValue(I.getArgOperand(0)).getValueType(),
05155                                getValue(I.getArgOperand(0)),
05156                                getValue(I.getArgOperand(1)),
05157                                getValue(I.getArgOperand(2))));
05158     } else {
05159       SDValue Mul = DAG.getNode(ISD::FMUL, sdl,
05160                                 getValue(I.getArgOperand(0)).getValueType(),
05161                                 getValue(I.getArgOperand(0)),
05162                                 getValue(I.getArgOperand(1)));
05163       SDValue Add = DAG.getNode(ISD::FADD, sdl,
05164                                 getValue(I.getArgOperand(0)).getValueType(),
05165                                 Mul,
05166                                 getValue(I.getArgOperand(2)));
05167       setValue(&I, Add);
05168     }
05169     return nullptr;
05170   }
05171   case Intrinsic::convert_to_fp16:
05172     setValue(&I, DAG.getNode(ISD::BITCAST, sdl, MVT::i16,
05173                              DAG.getNode(ISD::FP_ROUND, sdl, MVT::f16,
05174                                          getValue(I.getArgOperand(0)),
05175                                          DAG.getTargetConstant(0, MVT::i32))));
05176     return nullptr;
05177   case Intrinsic::convert_from_fp16:
05178     setValue(&I,
05179              DAG.getNode(ISD::FP_EXTEND, sdl, TLI->getValueType(I.getType()),
05180                          DAG.getNode(ISD::BITCAST, sdl, MVT::f16,
05181                                      getValue(I.getArgOperand(0)))));
05182     return nullptr;
05183   case Intrinsic::pcmarker: {
05184     SDValue Tmp = getValue(I.getArgOperand(0));
05185     DAG.setRoot(DAG.getNode(ISD::PCMARKER, sdl, MVT::Other, getRoot(), Tmp));
05186     return nullptr;
05187   }
05188   case Intrinsic::readcyclecounter: {
05189     SDValue Op = getRoot();
05190     Res = DAG.getNode(ISD::READCYCLECOUNTER, sdl,
05191                       DAG.getVTList(MVT::i64, MVT::Other), Op);
05192     setValue(&I, Res);
05193     DAG.setRoot(Res.getValue(1));
05194     return nullptr;
05195   }
05196   case Intrinsic::bswap:
05197     setValue(&I, DAG.getNode(ISD::BSWAP, sdl,
05198                              getValue(I.getArgOperand(0)).getValueType(),
05199                              getValue(I.getArgOperand(0))));
05200     return nullptr;
05201   case Intrinsic::cttz: {
05202     SDValue Arg = getValue(I.getArgOperand(0));
05203     ConstantInt *CI = cast<ConstantInt>(I.getArgOperand(1));
05204     EVT Ty = Arg.getValueType();
05205     setValue(&I, DAG.getNode(CI->isZero() ? ISD::CTTZ : ISD::CTTZ_ZERO_UNDEF,
05206                              sdl, Ty, Arg));
05207     return nullptr;
05208   }
05209   case Intrinsic::ctlz: {
05210     SDValue Arg = getValue(I.getArgOperand(0));
05211     ConstantInt *CI = cast<ConstantInt>(I.getArgOperand(1));
05212     EVT Ty = Arg.getValueType();
05213     setValue(&I, DAG.getNode(CI->isZero() ? ISD::CTLZ : ISD::CTLZ_ZERO_UNDEF,
05214                              sdl, Ty, Arg));
05215     return nullptr;
05216   }
05217   case Intrinsic::ctpop: {
05218     SDValue Arg = getValue(I.getArgOperand(0));
05219     EVT Ty = Arg.getValueType();
05220     setValue(&I, DAG.getNode(ISD::CTPOP, sdl, Ty, Arg));
05221     return nullptr;
05222   }
05223   case Intrinsic::stacksave: {
05224     SDValue Op = getRoot();
05225     Res = DAG.getNode(ISD::STACKSAVE, sdl,
05226                       DAG.getVTList(TLI->getPointerTy(), MVT::Other), Op);
05227     setValue(&I, Res);
05228     DAG.setRoot(Res.getValue(1));
05229     return nullptr;
05230   }
05231   case Intrinsic::stackrestore: {
05232     Res = getValue(I.getArgOperand(0));
05233     DAG.setRoot(DAG.getNode(ISD::STACKRESTORE, sdl, MVT::Other, getRoot(), Res));
05234     return nullptr;
05235   }
05236   case Intrinsic::stackprotector: {
05237     // Emit code into the DAG to store the stack guard onto the stack.
05238     MachineFunction &MF = DAG.getMachineFunction();
05239     MachineFrameInfo *MFI = MF.getFrameInfo();
05240     EVT PtrTy = TLI->getPointerTy();
05241     SDValue Src, Chain = getRoot();
05242 
05243     if (TLI->useLoadStackGuardNode()) {
05244       // Emit a LOAD_STACK_GUARD node.
05245       MachineSDNode *Node = DAG.getMachineNode(TargetOpcode::LOAD_STACK_GUARD,
05246                                                sdl, PtrTy, Chain);
05247       LoadInst *LI = cast<LoadInst>(I.getArgOperand(0));
05248       MachinePointerInfo MPInfo(LI->getPointerOperand());
05249       MachineInstr::mmo_iterator MemRefs = MF.allocateMemRefsArray(1);
05250       unsigned Flags = MachineMemOperand::MOLoad |
05251                        MachineMemOperand::MOInvariant;
05252       *MemRefs = MF.getMachineMemOperand(MPInfo, Flags,
05253                                          PtrTy.getSizeInBits() / 8,
05254                                          DAG.getEVTAlignment(PtrTy));
05255       Node->setMemRefs(MemRefs, MemRefs + 1);
05256 
05257       // Copy the guard value to a virtual register so that it can be
05258       // retrieved in the epilogue.
05259       Src = SDValue(Node, 0);
05260       const TargetRegisterClass *RC =
05261           TLI->getRegClassFor(Src.getSimpleValueType());
05262       unsigned Reg = MF.getRegInfo().createVirtualRegister(RC);
05263 
05264       SPDescriptor.setGuardReg(Reg);
05265       Chain = DAG.getCopyToReg(Chain, sdl, Reg, Src);
05266     } else {
05267       Src = getValue(I.getArgOperand(0));   // The guard's value.
05268     }
05269 
05270     AllocaInst *Slot = cast<AllocaInst>(I.getArgOperand(1));
05271 
05272     int FI = FuncInfo.StaticAllocaMap[Slot];
05273     MFI->setStackProtectorIndex(FI);
05274 
05275     SDValue FIN = DAG.getFrameIndex(FI, PtrTy);
05276 
05277     // Store the stack protector onto the stack.
05278     Res = DAG.getStore(Chain, sdl, Src, FIN,
05279                        MachinePointerInfo::getFixedStack(FI),
05280                        true, false, 0);
05281     setValue(&I, Res);
05282     DAG.setRoot(Res);
05283     return nullptr;
05284   }
05285   case Intrinsic::objectsize: {
05286     // If we don't know by now, we're never going to know.
05287     ConstantInt *CI = dyn_cast<ConstantInt>(I.getArgOperand(1));
05288 
05289     assert(CI && "Non-constant type in __builtin_object_size?");
05290 
05291     SDValue Arg = getValue(I.getCalledValue());
05292     EVT Ty = Arg.getValueType();
05293 
05294     if (CI->isZero())
05295       Res = DAG.getConstant(-1ULL, Ty);
05296     else
05297       Res = DAG.getConstant(0, Ty);
05298 
05299     setValue(&I, Res);
05300     return nullptr;
05301   }
05302   case Intrinsic::annotation:
05303   case Intrinsic::ptr_annotation:
05304     // Drop the intrinsic, but forward the value
05305     setValue(&I, getValue(I.getOperand(0)));
05306     return nullptr;
05307   case Intrinsic::assume:
05308   case Intrinsic::var_annotation:
05309     // Discard annotate attributes and assumptions
05310     return nullptr;
05311 
05312   case Intrinsic::init_trampoline: {
05313     const Function *F = cast<Function>(I.getArgOperand(1)->stripPointerCasts());
05314 
05315     SDValue Ops[6];
05316     Ops[0] = getRoot();
05317     Ops[1] = getValue(I.getArgOperand(0));
05318     Ops[2] = getValue(I.getArgOperand(1));
05319     Ops[3] = getValue(I.getArgOperand(2));
05320     Ops[4] = DAG.getSrcValue(I.getArgOperand(0));
05321     Ops[5] = DAG.getSrcValue(F);
05322 
05323     Res = DAG.getNode(ISD::INIT_TRAMPOLINE, sdl, MVT::Other, Ops);
05324 
05325     DAG.setRoot(Res);
05326     return nullptr;
05327   }
05328   case Intrinsic::adjust_trampoline: {
05329     setValue(&I, DAG.getNode(ISD::ADJUST_TRAMPOLINE, sdl,
05330                              TLI->getPointerTy(),
05331                              getValue(I.getArgOperand(0))));
05332     return nullptr;
05333   }
05334   case Intrinsic::gcroot:
05335     if (GFI) {
05336       const Value *Alloca = I.getArgOperand(0)->stripPointerCasts();
05337       const Constant *TypeMap = cast<Constant>(I.getArgOperand(1));
05338 
05339       FrameIndexSDNode *FI = cast<FrameIndexSDNode>(getValue(Alloca).getNode());
05340       GFI->addStackRoot(FI->getIndex(), TypeMap);
05341     }
05342     return nullptr;
05343   case Intrinsic::gcread:
05344   case Intrinsic::gcwrite:
05345     llvm_unreachable("GC failed to lower gcread/gcwrite intrinsics!");
05346   case Intrinsic::flt_rounds:
05347     setValue(&I, DAG.getNode(ISD::FLT_ROUNDS_, sdl, MVT::i32));
05348     return nullptr;
05349 
05350   case Intrinsic::expect: {
05351     // Just replace __builtin_expect(exp, c) with EXP.
05352     setValue(&I, getValue(I.getArgOperand(0)));
05353     return nullptr;
05354   }
05355 
05356   case Intrinsic::debugtrap:
05357   case Intrinsic::trap: {
05358     StringRef TrapFuncName = TM.Options.getTrapFunctionName();
05359     if (TrapFuncName.empty()) {
05360       ISD::NodeType Op = (Intrinsic == Intrinsic::trap) ?
05361         ISD::TRAP : ISD::DEBUGTRAP;
05362       DAG.setRoot(DAG.getNode(Op, sdl,MVT::Other, getRoot()));
05363       return nullptr;
05364     }
05365     TargetLowering::ArgListTy Args;
05366 
05367     TargetLowering::CallLoweringInfo CLI(DAG);
05368     CLI.setDebugLoc(sdl).setChain(getRoot())
05369       .setCallee(CallingConv::C, I.getType(),
05370                  DAG.getExternalSymbol(TrapFuncName.data(), TLI->getPointerTy()),
05371                  std::move(Args), 0);
05372 
05373     std::pair<SDValue, SDValue> Result = TLI->LowerCallTo(CLI);
05374     DAG.setRoot(Result.second);
05375     return nullptr;
05376   }
05377 
05378   case Intrinsic::uadd_with_overflow:
05379   case Intrinsic::sadd_with_overflow:
05380   case Intrinsic::usub_with_overflow:
05381   case Intrinsic::ssub_with_overflow:
05382   case Intrinsic::umul_with_overflow:
05383   case Intrinsic::smul_with_overflow: {
05384     ISD::NodeType Op;
05385     switch (Intrinsic) {
05386     default: llvm_unreachable("Impossible intrinsic");  // Can't reach here.
05387     case Intrinsic::uadd_with_overflow: Op = ISD::UADDO; break;
05388     case Intrinsic::sadd_with_overflow: Op = ISD::SADDO; break;
05389     case Intrinsic::usub_with_overflow: Op = ISD::USUBO; break;
05390     case Intrinsic::ssub_with_overflow: Op = ISD::SSUBO; break;
05391     case Intrinsic::umul_with_overflow: Op = ISD::UMULO; break;
05392     case Intrinsic::smul_with_overflow: Op = ISD::SMULO; break;
05393     }
05394     SDValue Op1 = getValue(I.getArgOperand(0));
05395     SDValue Op2 = getValue(I.getArgOperand(1));
05396 
05397     SDVTList VTs = DAG.getVTList(Op1.getValueType(), MVT::i1);
05398     setValue(&I, DAG.getNode(Op, sdl, VTs, Op1, Op2));
05399     return nullptr;
05400   }
05401   case Intrinsic::prefetch: {
05402     SDValue Ops[5];
05403     unsigned rw = cast<ConstantInt>(I.getArgOperand(1))->getZExtValue();
05404     Ops[0] = getRoot();
05405     Ops[1] = getValue(I.getArgOperand(0));
05406     Ops[2] = getValue(I.getArgOperand(1));
05407     Ops[3] = getValue(I.getArgOperand(2));
05408     Ops[4] = getValue(I.getArgOperand(3));
05409     DAG.setRoot(DAG.getMemIntrinsicNode(ISD::PREFETCH, sdl,
05410                                         DAG.getVTList(MVT::Other), Ops,
05411                                         EVT::getIntegerVT(*Context, 8),
05412                                         MachinePointerInfo(I.getArgOperand(0)),
05413                                         0, /* align */
05414                                         false, /* volatile */
05415                                         rw==0, /* read */
05416                                         rw==1)); /* write */
05417     return nullptr;
05418   }
05419   case Intrinsic::lifetime_start:
05420   case Intrinsic::lifetime_end: {
05421     bool IsStart = (Intrinsic == Intrinsic::lifetime_start);
05422     // Stack coloring is not enabled in O0, discard region information.
05423     if (TM.getOptLevel() == CodeGenOpt::None)
05424       return nullptr;
05425 
05426     SmallVector<Value *, 4> Allocas;
05427     GetUnderlyingObjects(I.getArgOperand(1), Allocas, DL);
05428 
05429     for (SmallVectorImpl<Value*>::iterator Object = Allocas.begin(),
05430            E = Allocas.end(); Object != E; ++Object) {
05431       AllocaInst *LifetimeObject = dyn_cast_or_null<AllocaInst>(*Object);
05432 
05433       // Could not find an Alloca.
05434       if (!LifetimeObject)
05435         continue;
05436 
05437       int FI = FuncInfo.StaticAllocaMap[LifetimeObject];
05438 
05439       SDValue Ops[2];
05440       Ops[0] = getRoot();
05441       Ops[1] = DAG.getFrameIndex(FI, TLI->getPointerTy(), true);
05442       unsigned Opcode = (IsStart ? ISD::LIFETIME_START : ISD::LIFETIME_END);
05443 
05444       Res = DAG.getNode(Opcode, sdl, MVT::Other, Ops);
05445       DAG.setRoot(Res);
05446     }
05447     return nullptr;
05448   }
05449   case Intrinsic::invariant_start:
05450     // Discard region information.
05451     setValue(&I, DAG.getUNDEF(TLI->getPointerTy()));
05452     return nullptr;
05453   case Intrinsic::invariant_end:
05454     // Discard region information.
05455     return nullptr;
05456   case Intrinsic::stackprotectorcheck: {
05457     // Do not actually emit anything for this basic block. Instead we initialize
05458     // the stack protector descriptor and export the guard variable so we can
05459     // access it in FinishBasicBlock.
05460     const BasicBlock *BB = I.getParent();
05461     SPDescriptor.initialize(BB, FuncInfo.MBBMap[BB], I);
05462     ExportFromCurrentBlock(SPDescriptor.getGuard());
05463 
05464     // Flush our exports since we are going to process a terminator.
05465     (void)getControlRoot();
05466     return nullptr;
05467   }
05468   case Intrinsic::clear_cache:
05469     return TLI->getClearCacheBuiltinName();
05470   case Intrinsic::donothing:
05471     // ignore
05472     return nullptr;
05473   case Intrinsic::experimental_stackmap: {
05474     visitStackmap(I);
05475     return nullptr;
05476   }
05477   case Intrinsic::experimental_patchpoint_void:
05478   case Intrinsic::experimental_patchpoint_i64: {
05479     visitPatchpoint(I);
05480     return nullptr;
05481   }
05482   }
05483 }
05484 
05485 void SelectionDAGBuilder::LowerCallTo(ImmutableCallSite CS, SDValue Callee,
05486                                       bool isTailCall,
05487                                       MachineBasicBlock *LandingPad) {
05488   const TargetLowering *TLI = TM.getTargetLowering();
05489   PointerType *PT = cast<PointerType>(CS.getCalledValue()->getType());
05490   FunctionType *FTy = cast<FunctionType>(PT->getElementType());
05491   Type *RetTy = FTy->getReturnType();
05492   MachineModuleInfo &MMI = DAG.getMachineFunction().getMMI();
05493   MCSymbol *BeginLabel = nullptr;
05494 
05495   TargetLowering::ArgListTy Args;
05496   TargetLowering::ArgListEntry Entry;
05497   Args.reserve(CS.arg_size());
05498 
05499   for (ImmutableCallSite::arg_iterator i = CS.arg_begin(), e = CS.arg_end();
05500        i != e; ++i) {
05501     const Value *V = *i;
05502 
05503     // Skip empty types
05504     if (V->getType()->isEmptyTy())
05505       continue;
05506 
05507     SDValue ArgNode = getValue(V);
05508     Entry.Node = ArgNode; Entry.Ty = V->getType();
05509 
05510     // Skip the first return-type Attribute to get to params.
05511     Entry.setAttributes(&CS, i - CS.arg_begin() + 1);
05512     Args.push_back(Entry);
05513   }
05514 
05515   if (LandingPad) {
05516     // Insert a label before the invoke call to mark the try range.  This can be
05517     // used to detect deletion of the invoke via the MachineModuleInfo.
05518     BeginLabel = MMI.getContext().CreateTempSymbol();
05519 
05520     // For SjLj, keep track of which landing pads go with which invokes
05521     // so as to maintain the ordering of pads in the LSDA.
05522     unsigned CallSiteIndex = MMI.getCurrentCallSite();
05523     if (CallSiteIndex) {
05524       MMI.setCallSiteBeginLabel(BeginLabel, CallSiteIndex);
05525       LPadToCallSiteMap[LandingPad].push_back(CallSiteIndex);
05526 
05527       // Now that the call site is handled, stop tracking it.
05528       MMI.setCurrentCallSite(0);
05529     }
05530 
05531     // Both PendingLoads and PendingExports must be flushed here;
05532     // this call might not return.
05533     (void)getRoot();
05534     DAG.setRoot(DAG.getEHLabel(getCurSDLoc(), getControlRoot(), BeginLabel));
05535   }
05536 
05537   // Check if target-independent constraints permit a tail call here.
05538   // Target-dependent constraints are checked within TLI->LowerCallTo.
05539   if (isTailCall && !isInTailCallPosition(CS, DAG.getTarget()))
05540     isTailCall = false;
05541 
05542   TargetLowering::CallLoweringInfo CLI(DAG);
05543   CLI.setDebugLoc(getCurSDLoc()).setChain(getRoot())
05544     .setCallee(RetTy, FTy, Callee, std::move(Args), CS).setTailCall(isTailCall);
05545 
05546   std::pair<SDValue,SDValue> Result = TLI->LowerCallTo(CLI);
05547   assert((isTailCall || Result.second.getNode()) &&
05548          "Non-null chain expected with non-tail call!");
05549   assert((Result.second.getNode() || !Result.first.getNode()) &&
05550          "Null value expected with tail call!");
05551   if (Result.first.getNode())
05552     setValue(CS.getInstruction(), Result.first);
05553 
05554   if (!Result.second.getNode()) {
05555     // As a special case, a null chain means that a tail call has been emitted
05556     // and the DAG root is already updated.
05557     HasTailCall = true;
05558 
05559     // Since there's no actual continuation from this block, nothing can be
05560     // relying on us setting vregs for them.
05561     PendingExports.clear();
05562   } else {
05563     DAG.setRoot(Result.second);
05564   }
05565 
05566   if (LandingPad) {
05567     // Insert a label at the end of the invoke call to mark the try range.  This
05568     // can be used to detect deletion of the invoke via the MachineModuleInfo.
05569     MCSymbol *EndLabel = MMI.getContext().CreateTempSymbol();
05570     DAG.setRoot(DAG.getEHLabel(getCurSDLoc(), getRoot(), EndLabel));
05571 
05572     // Inform MachineModuleInfo of range.
05573     MMI.addInvoke(LandingPad, BeginLabel, EndLabel);
05574   }
05575 }
05576 
05577 /// IsOnlyUsedInZeroEqualityComparison - Return true if it only matters that the
05578 /// value is equal or not-equal to zero.
05579 static bool IsOnlyUsedInZeroEqualityComparison(const Value *V) {
05580   for (const User *U : V->users()) {
05581     if (const ICmpInst *IC = dyn_cast<ICmpInst>(U))
05582       if (IC->isEquality())
05583         if (const Constant *C = dyn_cast<Constant>(IC->getOperand(1)))
05584           if (C->isNullValue())
05585             continue;
05586     // Unknown instruction.
05587     return false;
05588   }
05589   return true;
05590 }
05591 
05592 static SDValue getMemCmpLoad(const Value *PtrVal, MVT LoadVT,
05593                              Type *LoadTy,
05594                              SelectionDAGBuilder &Builder) {
05595 
05596   // Check to see if this load can be trivially constant folded, e.g. if the
05597   // input is from a string literal.
05598   if (const Constant *LoadInput = dyn_cast<Constant>(PtrVal)) {
05599     // Cast pointer to the type we really want to load.
05600     LoadInput = ConstantExpr::getBitCast(const_cast<Constant *>(LoadInput),
05601                                          PointerType::getUnqual(LoadTy));
05602 
05603     if (const Constant *LoadCst =
05604           ConstantFoldLoadFromConstPtr(const_cast<Constant *>(LoadInput),
05605                                        Builder.DL))
05606       return Builder.getValue(LoadCst);
05607   }
05608 
05609   // Otherwise, we have to emit the load.  If the pointer is to unfoldable but
05610   // still constant memory, the input chain can be the entry node.
05611   SDValue Root;
05612   bool ConstantMemory = false;
05613 
05614   // Do not serialize (non-volatile) loads of constant memory with anything.
05615   if (Builder.AA->pointsToConstantMemory(PtrVal)) {
05616     Root = Builder.DAG.getEntryNode();
05617     ConstantMemory = true;
05618   } else {
05619     // Do not serialize non-volatile loads against each other.
05620     Root = Builder.DAG.getRoot();
05621   }
05622 
05623   SDValue Ptr = Builder.getValue(PtrVal);
05624   SDValue LoadVal = Builder.DAG.getLoad(LoadVT, Builder.getCurSDLoc(), Root,
05625                                         Ptr, MachinePointerInfo(PtrVal),
05626                                         false /*volatile*/,
05627                                         false /*nontemporal*/,
05628                                         false /*isinvariant*/, 1 /* align=1 */);
05629 
05630   if (!ConstantMemory)
05631     Builder.PendingLoads.push_back(LoadVal.getValue(1));
05632   return LoadVal;
05633 }
05634 
05635 /// processIntegerCallValue - Record the value for an instruction that
05636 /// produces an integer result, converting the type where necessary.
05637 void SelectionDAGBuilder::processIntegerCallValue(const Instruction &I,
05638                                                   SDValue Value,
05639                                                   bool IsSigned) {
05640   EVT VT = TM.getTargetLowering()->getValueType(I.getType(), true);
05641   if (IsSigned)
05642     Value = DAG.getSExtOrTrunc(Value, getCurSDLoc(), VT);
05643   else
05644     Value = DAG.getZExtOrTrunc(Value, getCurSDLoc(), VT);
05645   setValue(&I, Value);
05646 }
05647 
05648 /// visitMemCmpCall - See if we can lower a call to memcmp in an optimized form.
05649 /// If so, return true and lower it, otherwise return false and it will be
05650 /// lowered like a normal call.
05651 bool SelectionDAGBuilder::visitMemCmpCall(const CallInst &I) {
05652   // Verify that the prototype makes sense.  int memcmp(void*,void*,size_t)
05653   if (I.getNumArgOperands() != 3)
05654     return false;
05655 
05656   const Value *LHS = I.getArgOperand(0), *RHS = I.getArgOperand(1);
05657   if (!LHS->getType()->isPointerTy() || !RHS->getType()->isPointerTy() ||
05658       !I.getArgOperand(2)->getType()->isIntegerTy() ||
05659       !I.getType()->isIntegerTy())
05660     return false;
05661 
05662   const Value *Size = I.getArgOperand(2);
05663   const ConstantInt *CSize = dyn_cast<ConstantInt>(Size);
05664   if (CSize && CSize->getZExtValue() == 0) {
05665     EVT CallVT = TM.getTargetLowering()->getValueType(I.getType(), true);
05666     setValue(&I, DAG.getConstant(0, CallVT));
05667     return true;
05668   }
05669 
05670   const TargetSelectionDAGInfo &TSI = DAG.getSelectionDAGInfo();
05671   std::pair<SDValue, SDValue> Res =
05672     TSI.EmitTargetCodeForMemcmp(DAG, getCurSDLoc(), DAG.getRoot(),
05673                                 getValue(LHS), getValue(RHS), getValue(Size),
05674                                 MachinePointerInfo(LHS),
05675                                 MachinePointerInfo(RHS));
05676   if (Res.first.getNode()) {
05677     processIntegerCallValue(I, Res.first, true);
05678     PendingLoads.push_back(Res.second);
05679     return true;
05680   }
05681 
05682   // memcmp(S1,S2,2) != 0 -> (*(short*)LHS != *(short*)RHS)  != 0
05683   // memcmp(S1,S2,4) != 0 -> (*(int*)LHS != *(int*)RHS)  != 0
05684   if (CSize && IsOnlyUsedInZeroEqualityComparison(&I)) {
05685     bool ActuallyDoIt = true;
05686     MVT LoadVT;
05687     Type *LoadTy;
05688     switch (CSize->getZExtValue()) {
05689     default:
05690       LoadVT = MVT::Other;
05691       LoadTy = nullptr;
05692       ActuallyDoIt = false;
05693       break;
05694     case 2:
05695       LoadVT = MVT::i16;
05696       LoadTy = Type::getInt16Ty(CSize->getContext());
05697       break;
05698     case 4:
05699       LoadVT = MVT::i32;
05700       LoadTy = Type::getInt32Ty(CSize->getContext());
05701       break;
05702     case 8:
05703       LoadVT = MVT::i64;
05704       LoadTy = Type::getInt64Ty(CSize->getContext());
05705       break;
05706         /*
05707     case 16:
05708       LoadVT = MVT::v4i32;
05709       LoadTy = Type::getInt32Ty(CSize->getContext());
05710       LoadTy = VectorType::get(LoadTy, 4);
05711       break;
05712          */
05713     }
05714 
05715     // This turns into unaligned loads.  We only do this if the target natively
05716     // supports the MVT we'll be loading or if it is small enough (<= 4) that
05717     // we'll only produce a small number of byte loads.
05718 
05719     // Require that we can find a legal MVT, and only do this if the target
05720     // supports unaligned loads of that type.  Expanding into byte loads would
05721     // bloat the code.
05722     const TargetLowering *TLI = TM.getTargetLowering();
05723     if (ActuallyDoIt && CSize->getZExtValue() > 4) {
05724       unsigned DstAS = LHS->getType()->getPointerAddressSpace();
05725       unsigned SrcAS = RHS->getType()->getPointerAddressSpace();
05726       // TODO: Handle 5 byte compare as 4-byte + 1 byte.
05727       // TODO: Handle 8 byte compare on x86-32 as two 32-bit loads.
05728       // TODO: Check alignment of src and dest ptrs.
05729       if (!TLI->isTypeLegal(LoadVT) ||
05730           !TLI->allowsMisalignedMemoryAccesses(LoadVT, SrcAS) ||
05731           !TLI->allowsMisalignedMemoryAccesses(LoadVT, DstAS))
05732         ActuallyDoIt = false;
05733     }
05734 
05735     if (ActuallyDoIt) {
05736       SDValue LHSVal = getMemCmpLoad(LHS, LoadVT, LoadTy, *this);
05737       SDValue RHSVal = getMemCmpLoad(RHS, LoadVT, LoadTy, *this);
05738 
05739       SDValue Res = DAG.getSetCC(getCurSDLoc(), MVT::i1, LHSVal, RHSVal,
05740                                  ISD::SETNE);
05741       processIntegerCallValue(I, Res, false);
05742       return true;
05743     }
05744   }
05745 
05746 
05747   return false;
05748 }
05749 
05750 /// visitMemChrCall -- See if we can lower a memchr call into an optimized
05751 /// form.  If so, return true and lower it, otherwise return false and it
05752 /// will be lowered like a normal call.
05753 bool SelectionDAGBuilder::visitMemChrCall(const CallInst &I) {
05754   // Verify that the prototype makes sense.  void *memchr(void *, int, size_t)
05755   if (I.getNumArgOperands() != 3)
05756     return false;
05757 
05758   const Value *Src = I.getArgOperand(0);
05759   const Value *Char = I.getArgOperand(1);
05760   const Value *Length = I.getArgOperand(2);
05761   if (!Src->getType()->isPointerTy() ||
05762       !Char->getType()->isIntegerTy() ||
05763       !Length->getType()->isIntegerTy() ||
05764       !I.getType()->isPointerTy())
05765     return false;
05766 
05767   const TargetSelectionDAGInfo &TSI = DAG.getSelectionDAGInfo();
05768   std::pair<SDValue, SDValue> Res =
05769     TSI.EmitTargetCodeForMemchr(DAG, getCurSDLoc(), DAG.getRoot(),
05770                                 getValue(Src), getValue(Char), getValue(Length),
05771                                 MachinePointerInfo(Src));
05772   if (Res.first.getNode()) {
05773     setValue(&I, Res.first);
05774     PendingLoads.push_back(Res.second);
05775     return true;
05776   }
05777 
05778   return false;
05779 }
05780 
05781 /// visitStrCpyCall -- See if we can lower a strcpy or stpcpy call into an
05782 /// optimized form.  If so, return true and lower it, otherwise return false
05783 /// and it will be lowered like a normal call.
05784 bool SelectionDAGBuilder::visitStrCpyCall(const CallInst &I, bool isStpcpy) {
05785   // Verify that the prototype makes sense.  char *strcpy(char *, char *)
05786   if (I.getNumArgOperands() != 2)
05787     return false;
05788 
05789   const Value *Arg0 = I.getArgOperand(0), *Arg1 = I.getArgOperand(1);
05790   if (!Arg0->getType()->isPointerTy() ||
05791       !Arg1->getType()->isPointerTy() ||
05792       !I.getType()->isPointerTy())
05793     return false;
05794 
05795   const TargetSelectionDAGInfo &TSI = DAG.getSelectionDAGInfo();
05796   std::pair<SDValue, SDValue> Res =
05797     TSI.EmitTargetCodeForStrcpy(DAG, getCurSDLoc(), getRoot(),
05798                                 getValue(Arg0), getValue(Arg1),
05799                                 MachinePointerInfo(Arg0),
05800                                 MachinePointerInfo(Arg1), isStpcpy);
05801   if (Res.first.getNode()) {
05802     setValue(&I, Res.first);
05803     DAG.setRoot(Res.second);
05804     return true;
05805   }
05806 
05807   return false;
05808 }
05809 
05810 /// visitStrCmpCall - See if we can lower a call to strcmp in an optimized form.
05811 /// If so, return true and lower it, otherwise return false and it will be
05812 /// lowered like a normal call.
05813 bool SelectionDAGBuilder::visitStrCmpCall(const CallInst &I) {
05814   // Verify that the prototype makes sense.  int strcmp(void*,void*)
05815   if (I.getNumArgOperands() != 2)
05816     return false;
05817 
05818   const Value *Arg0 = I.getArgOperand(0), *Arg1 = I.getArgOperand(1);
05819   if (!Arg0->getType()->isPointerTy() ||
05820       !Arg1->getType()->isPointerTy() ||
05821       !I.getType()->isIntegerTy())
05822     return false;
05823 
05824   const TargetSelectionDAGInfo &TSI = DAG.getSelectionDAGInfo();
05825   std::pair<SDValue, SDValue> Res =
05826     TSI.EmitTargetCodeForStrcmp(DAG, getCurSDLoc(), DAG.getRoot(),
05827                                 getValue(Arg0), getValue(Arg1),
05828                                 MachinePointerInfo(Arg0),
05829                                 MachinePointerInfo(Arg1));
05830   if (Res.first.getNode()) {
05831     processIntegerCallValue(I, Res.first, true);
05832     PendingLoads.push_back(Res.second);
05833     return true;
05834   }
05835 
05836   return false;
05837 }
05838 
05839 /// visitStrLenCall -- See if we can lower a strlen call into an optimized
05840 /// form.  If so, return true and lower it, otherwise return false and it
05841 /// will be lowered like a normal call.
05842 bool SelectionDAGBuilder::visitStrLenCall(const CallInst &I) {
05843   // Verify that the prototype makes sense.  size_t strlen(char *)
05844   if (I.getNumArgOperands() != 1)
05845     return false;
05846 
05847   const Value *Arg0 = I.getArgOperand(0);
05848   if (!Arg0->getType()->isPointerTy() || !I.getType()->isIntegerTy())
05849     return false;
05850 
05851   const TargetSelectionDAGInfo &TSI = DAG.getSelectionDAGInfo();
05852   std::pair<SDValue, SDValue> Res =
05853     TSI.EmitTargetCodeForStrlen(DAG, getCurSDLoc(), DAG.getRoot(),
05854                                 getValue(Arg0), MachinePointerInfo(Arg0));
05855   if (Res.first.getNode()) {
05856     processIntegerCallValue(I, Res.first, false);
05857     PendingLoads.push_back(Res.second);
05858     return true;
05859   }
05860 
05861   return false;
05862 }
05863 
05864 /// visitStrNLenCall -- See if we can lower a strnlen call into an optimized
05865 /// form.  If so, return true and lower it, otherwise return false and it
05866 /// will be lowered like a normal call.
05867 bool SelectionDAGBuilder::visitStrNLenCall(const CallInst &I) {
05868   // Verify that the prototype makes sense.  size_t strnlen(char *, size_t)
05869   if (I.getNumArgOperands() != 2)
05870     return false;
05871 
05872   const Value *Arg0 = I.getArgOperand(0), *Arg1 = I.getArgOperand(1);
05873   if (!Arg0->getType()->isPointerTy() ||
05874       !Arg1->getType()->isIntegerTy() ||
05875       !I.getType()->isIntegerTy())
05876     return false;
05877 
05878   const TargetSelectionDAGInfo &TSI = DAG.getSelectionDAGInfo();
05879   std::pair<SDValue, SDValue> Res =
05880     TSI.EmitTargetCodeForStrnlen(DAG, getCurSDLoc(), DAG.getRoot(),
05881                                  getValue(Arg0), getValue(Arg1),
05882                                  MachinePointerInfo(Arg0));
05883   if (Res.first.getNode()) {
05884     processIntegerCallValue(I, Res.first, false);
05885     PendingLoads.push_back(Res.second);
05886     return true;
05887   }
05888 
05889   return false;
05890 }
05891 
05892 /// visitUnaryFloatCall - If a call instruction is a unary floating-point
05893 /// operation (as expected), translate it to an SDNode with the specified opcode
05894 /// and return true.
05895 bool SelectionDAGBuilder::visitUnaryFloatCall(const CallInst &I,
05896                                               unsigned Opcode) {
05897   // Sanity check that it really is a unary floating-point call.
05898   if (I.getNumArgOperands() != 1 ||
05899       !I.getArgOperand(0)->getType()->isFloatingPointTy() ||
05900       I.getType() != I.getArgOperand(0)->getType() ||
05901       !I.onlyReadsMemory())
05902     return false;
05903 
05904   SDValue Tmp = getValue(I.getArgOperand(0));
05905   setValue(&I, DAG.getNode(Opcode, getCurSDLoc(), Tmp.getValueType(), Tmp));
05906   return true;
05907 }
05908 
05909 void SelectionDAGBuilder::visitCall(const CallInst &I) {
05910   // Handle inline assembly differently.
05911   if (isa<InlineAsm>(I.getCalledValue())) {
05912     visitInlineAsm(&I);
05913     return;
05914   }
05915 
05916   MachineModuleInfo &MMI = DAG.getMachineFunction().getMMI();
05917   ComputeUsesVAFloatArgument(I, &MMI);
05918 
05919   const char *RenameFn = nullptr;
05920   if (Function *F = I.getCalledFunction()) {
05921     if (F->isDeclaration()) {
05922       if (const TargetIntrinsicInfo *II = TM.getIntrinsicInfo()) {
05923         if (unsigned IID = II->getIntrinsicID(F)) {
05924           RenameFn = visitIntrinsicCall(I, IID);
05925           if (!RenameFn)
05926             return;
05927         }
05928       }
05929       if (unsigned IID = F->getIntrinsicID()) {
05930         RenameFn = visitIntrinsicCall(I, IID);
05931         if (!RenameFn)
05932           return;
05933       }
05934     }
05935 
05936     // Check for well-known libc/libm calls.  If the function is internal, it
05937     // can't be a library call.
05938     LibFunc::Func Func;
05939     if (!F->hasLocalLinkage() && F->hasName() &&
05940         LibInfo->getLibFunc(F->getName(), Func) &&
05941         LibInfo->hasOptimizedCodeGen(Func)) {
05942       switch (Func) {
05943       default: break;
05944       case LibFunc::copysign:
05945       case LibFunc::copysignf:
05946       case LibFunc::copysignl:
05947         if (I.getNumArgOperands() == 2 &&   // Basic sanity checks.
05948             I.getArgOperand(0)->getType()->isFloatingPointTy() &&
05949             I.getType() == I.getArgOperand(0)->getType() &&
05950             I.getType() == I.getArgOperand(1)->getType() &&
05951             I.onlyReadsMemory()) {
05952           SDValue LHS = getValue(I.getArgOperand(0));
05953           SDValue RHS = getValue(I.getArgOperand(1));
05954           setValue(&I, DAG.getNode(ISD::FCOPYSIGN, getCurSDLoc(),
05955                                    LHS.getValueType(), LHS, RHS));
05956           return;
05957         }
05958         break;
05959       case LibFunc::fabs:
05960       case LibFunc::fabsf:
05961       case LibFunc::fabsl:
05962         if (visitUnaryFloatCall(I, ISD::FABS))
05963           return;
05964         break;
05965       case LibFunc::sin:
05966       case LibFunc::sinf:
05967       case LibFunc::sinl:
05968         if (visitUnaryFloatCall(I, ISD::FSIN))
05969           return;
05970         break;
05971       case LibFunc::cos:
05972       case LibFunc::cosf:
05973       case LibFunc::cosl:
05974         if (visitUnaryFloatCall(I, ISD::FCOS))
05975           return;
05976         break;
05977       case LibFunc::sqrt:
05978       case LibFunc::sqrtf:
05979       case LibFunc::sqrtl:
05980       case LibFunc::sqrt_finite:
05981       case LibFunc::sqrtf_finite:
05982       case LibFunc::sqrtl_finite:
05983         if (visitUnaryFloatCall(I, ISD::FSQRT))
05984           return;
05985         break;
05986       case LibFunc::floor:
05987       case LibFunc::floorf:
05988       case LibFunc::floorl:
05989         if (visitUnaryFloatCall(I, ISD::FFLOOR))
05990           return;
05991         break;
05992       case LibFunc::nearbyint:
05993       case LibFunc::nearbyintf:
05994       case LibFunc::nearbyintl:
05995         if (visitUnaryFloatCall(I, ISD::FNEARBYINT))
05996           return;
05997         break;
05998       case LibFunc::ceil:
05999       case LibFunc::ceilf:
06000       case LibFunc::ceill:
06001         if (visitUnaryFloatCall(I, ISD::FCEIL))
06002           return;
06003         break;
06004       case LibFunc::rint:
06005       case LibFunc::rintf:
06006       case LibFunc::rintl:
06007         if (visitUnaryFloatCall(I, ISD::FRINT))
06008           return;
06009         break;
06010       case LibFunc::round:
06011       case LibFunc::roundf:
06012       case LibFunc::roundl:
06013         if (visitUnaryFloatCall(I, ISD::FROUND))
06014           return;
06015         break;
06016       case LibFunc::trunc:
06017       case LibFunc::truncf:
06018       case LibFunc::truncl:
06019         if (visitUnaryFloatCall(I, ISD::FTRUNC))
06020           return;
06021         break;
06022       case LibFunc::log2:
06023       case LibFunc::log2f:
06024       case LibFunc::log2l:
06025         if (visitUnaryFloatCall(I, ISD::FLOG2))
06026           return;
06027         break;
06028       case LibFunc::exp2:
06029       case LibFunc::exp2f:
06030       case LibFunc::exp2l:
06031         if (visitUnaryFloatCall(I, ISD::FEXP2))
06032           return;
06033         break;
06034       case LibFunc::memcmp:
06035         if (visitMemCmpCall(I))
06036           return;
06037         break;
06038       case LibFunc::memchr:
06039         if (visitMemChrCall(I))
06040           return;
06041         break;
06042       case LibFunc::strcpy:
06043         if (visitStrCpyCall(I, false))
06044           return;
06045         break;
06046       case LibFunc::stpcpy:
06047         if (visitStrCpyCall(I, true))
06048           return;
06049         break;
06050       case LibFunc::strcmp:
06051         if (visitStrCmpCall(I))
06052           return;
06053         break;
06054       case LibFunc::strlen:
06055         if (visitStrLenCall(I))
06056           return;
06057         break;
06058       case LibFunc::strnlen:
06059         if (visitStrNLenCall(I))
06060           return;
06061         break;
06062       }
06063     }
06064   }
06065 
06066   SDValue Callee;
06067   if (!RenameFn)
06068     Callee = getValue(I.getCalledValue());
06069   else
06070     Callee = DAG.getExternalSymbol(RenameFn,
06071                                    TM.getTargetLowering()->getPointerTy());
06072 
06073   // Check if we can potentially perform a tail call. More detailed checking is
06074   // be done within LowerCallTo, after more information about the call is known.
06075   LowerCallTo(&I, Callee, I.isTailCall());
06076 }
06077 
06078 namespace {
06079 
06080 /// AsmOperandInfo - This contains information for each constraint that we are
06081 /// lowering.
06082 class SDISelAsmOperandInfo : public TargetLowering::AsmOperandInfo {
06083 public:
06084   /// CallOperand - If this is the result output operand or a clobber
06085   /// this is null, otherwise it is the incoming operand to the CallInst.
06086   /// This gets modified as the asm is processed.
06087   SDValue CallOperand;
06088 
06089   /// AssignedRegs - If this is a register or register class operand, this
06090   /// contains the set of register corresponding to the operand.
06091   RegsForValue AssignedRegs;
06092 
06093   explicit SDISelAsmOperandInfo(const TargetLowering::AsmOperandInfo &info)
06094     : TargetLowering::AsmOperandInfo(info), CallOperand(nullptr,0) {
06095   }
06096 
06097   /// getCallOperandValEVT - Return the EVT of the Value* that this operand
06098   /// corresponds to.  If there is no Value* for this operand, it returns
06099   /// MVT::Other.
06100   EVT getCallOperandValEVT(LLVMContext &Context,
06101                            const TargetLowering &TLI,
06102                            const DataLayout *DL) const {
06103     if (!CallOperandVal) return MVT::Other;
06104 
06105     if (isa<BasicBlock>(CallOperandVal))
06106       return TLI.getPointerTy();
06107 
06108     llvm::Type *OpTy = CallOperandVal->getType();
06109 
06110     // FIXME: code duplicated from TargetLowering::ParseConstraints().
06111     // If this is an indirect operand, the operand is a pointer to the
06112     // accessed type.
06113     if (isIndirect) {
06114       llvm::PointerType *PtrTy = dyn_cast<PointerType>(OpTy);
06115       if (!PtrTy)
06116         report_fatal_error("Indirect operand for inline asm not a pointer!");
06117       OpTy = PtrTy->getElementType();
06118     }
06119 
06120     // Look for vector wrapped in a struct. e.g. { <16 x i8> }.
06121     if (StructType *STy = dyn_cast<StructType>(OpTy))
06122       if (STy->getNumElements() == 1)
06123         OpTy = STy->getElementType(0);
06124 
06125     // If OpTy is not a single value, it may be a struct/union that we
06126     // can tile with integers.
06127     if (!OpTy->isSingleValueType() && OpTy->isSized()) {
06128       unsigned BitSize = DL->getTypeSizeInBits(OpTy);
06129       switch (BitSize) {
06130       default: break;
06131       case 1:
06132       case 8:
06133       case 16:
06134       case 32:
06135       case 64:
06136       case 128:
06137         OpTy = IntegerType::get(Context, BitSize);
06138         break;
06139       }
06140     }
06141 
06142     return TLI.getValueType(OpTy, true);
06143   }
06144 };
06145 
06146 typedef SmallVector<SDISelAsmOperandInfo,16> SDISelAsmOperandInfoVector;
06147 
06148 } // end anonymous namespace
06149 
06150 /// GetRegistersForValue - Assign registers (virtual or physical) for the
06151 /// specified operand.  We prefer to assign virtual registers, to allow the
06152 /// register allocator to handle the assignment process.  However, if the asm
06153 /// uses features that we can't model on machineinstrs, we have SDISel do the
06154 /// allocation.  This produces generally horrible, but correct, code.
06155 ///
06156 ///   OpInfo describes the operand.
06157 ///
06158 static void GetRegistersForValue(SelectionDAG &DAG,
06159                                  const TargetLowering &TLI,
06160                                  SDLoc DL,
06161                                  SDISelAsmOperandInfo &OpInfo) {
06162   LLVMContext &Context = *DAG.getContext();
06163 
06164   MachineFunction &MF = DAG.getMachineFunction();
06165   SmallVector<unsigned, 4> Regs;
06166 
06167   // If this is a constraint for a single physreg, or a constraint for a
06168   // register class, find it.
06169   std::pair<unsigned, const TargetRegisterClass*> PhysReg =
06170     TLI.getRegForInlineAsmConstraint(OpInfo.ConstraintCode,
06171                                      OpInfo.ConstraintVT);
06172 
06173   unsigned NumRegs = 1;
06174   if (OpInfo.ConstraintVT != MVT::Other) {
06175     // If this is a FP input in an integer register (or visa versa) insert a bit
06176     // cast of the input value.  More generally, handle any case where the input
06177     // value disagrees with the register class we plan to stick this in.
06178     if (OpInfo.Type == InlineAsm::isInput &&
06179         PhysReg.second && !PhysReg.second->hasType(OpInfo.ConstraintVT)) {
06180       // Try to convert to the first EVT that the reg class contains.  If the
06181       // types are identical size, use a bitcast to convert (e.g. two differing
06182       // vector types).
06183       MVT RegVT = *PhysReg.second->vt_begin();
06184       if (RegVT.getSizeInBits() == OpInfo.CallOperand.getValueSizeInBits()) {
06185         OpInfo.CallOperand = DAG.getNode(ISD::BITCAST, DL,
06186                                          RegVT, OpInfo.CallOperand);
06187         OpInfo.ConstraintVT = RegVT;
06188       } else if (RegVT.isInteger() && OpInfo.ConstraintVT.isFloatingPoint()) {
06189         // If the input is a FP value and we want it in FP registers, do a
06190         // bitcast to the corresponding integer type.  This turns an f64 value
06191         // into i64, which can be passed with two i32 values on a 32-bit
06192         // machine.
06193         RegVT = MVT::getIntegerVT(OpInfo.ConstraintVT.getSizeInBits());
06194         OpInfo.CallOperand = DAG.getNode(ISD::BITCAST, DL,
06195                                          RegVT, OpInfo.CallOperand);
06196         OpInfo.ConstraintVT = RegVT;
06197       }
06198     }
06199 
06200     NumRegs = TLI.getNumRegisters(Context, OpInfo.ConstraintVT);
06201   }
06202 
06203   MVT RegVT;
06204   EVT ValueVT = OpInfo.ConstraintVT;
06205 
06206   // If this is a constraint for a specific physical register, like {r17},
06207   // assign it now.
06208   if (unsigned AssignedReg = PhysReg.first) {
06209     const TargetRegisterClass *RC = PhysReg.second;
06210     if (OpInfo.ConstraintVT == MVT::Other)
06211       ValueVT = *RC->vt_begin();
06212 
06213     // Get the actual register value type.  This is important, because the user
06214     // may have asked for (e.g.) the AX register in i32 type.  We need to
06215     // remember that AX is actually i16 to get the right extension.
06216     RegVT = *RC->vt_begin();
06217 
06218     // This is a explicit reference to a physical register.
06219     Regs.push_back(AssignedReg);
06220 
06221     // If this is an expanded reference, add the rest of the regs to Regs.
06222     if (NumRegs != 1) {
06223       TargetRegisterClass::iterator I = RC->begin();
06224       for (; *I != AssignedReg; ++I)
06225         assert(I != RC->end() && "Didn't find reg!");
06226 
06227       // Already added the first reg.
06228       --NumRegs; ++I;
06229       for (; NumRegs; --NumRegs, ++I) {
06230         assert(I != RC->end() && "Ran out of registers to allocate!");
06231         Regs.push_back(*I);
06232       }
06233     }
06234 
06235     OpInfo.AssignedRegs = RegsForValue(Regs, RegVT, ValueVT);
06236     return;
06237   }
06238 
06239   // Otherwise, if this was a reference to an LLVM register class, create vregs
06240   // for this reference.
06241   if (