LLVM API Documentation

SelectionDAGBuilder.cpp
Go to the documentation of this file.
00001 //===-- SelectionDAGBuilder.cpp - Selection-DAG building ------------------===//
00002 //
00003 //                     The LLVM Compiler Infrastructure
00004 //
00005 // This file is distributed under the University of Illinois Open Source
00006 // License. See LICENSE.TXT for details.
00007 //
00008 //===----------------------------------------------------------------------===//
00009 //
00010 // This implements routines for translating from LLVM IR into SelectionDAG IR.
00011 //
00012 //===----------------------------------------------------------------------===//
00013 
00014 #include "SelectionDAGBuilder.h"
00015 #include "SDNodeDbgValue.h"
00016 #include "llvm/ADT/BitVector.h"
00017 #include "llvm/ADT/Optional.h"
00018 #include "llvm/ADT/SmallSet.h"
00019 #include "llvm/Analysis/AliasAnalysis.h"
00020 #include "llvm/Analysis/BranchProbabilityInfo.h"
00021 #include "llvm/Analysis/ConstantFolding.h"
00022 #include "llvm/Analysis/ValueTracking.h"
00023 #include "llvm/CodeGen/Analysis.h"
00024 #include "llvm/CodeGen/FastISel.h"
00025 #include "llvm/CodeGen/FunctionLoweringInfo.h"
00026 #include "llvm/CodeGen/GCMetadata.h"
00027 #include "llvm/CodeGen/GCStrategy.h"
00028 #include "llvm/CodeGen/MachineFrameInfo.h"
00029 #include "llvm/CodeGen/MachineFunction.h"
00030 #include "llvm/CodeGen/MachineInstrBuilder.h"
00031 #include "llvm/CodeGen/MachineJumpTableInfo.h"
00032 #include "llvm/CodeGen/MachineModuleInfo.h"
00033 #include "llvm/CodeGen/MachineRegisterInfo.h"
00034 #include "llvm/CodeGen/SelectionDAG.h"
00035 #include "llvm/CodeGen/StackMaps.h"
00036 #include "llvm/IR/CallingConv.h"
00037 #include "llvm/IR/Constants.h"
00038 #include "llvm/IR/DataLayout.h"
00039 #include "llvm/IR/DebugInfo.h"
00040 #include "llvm/IR/DerivedTypes.h"
00041 #include "llvm/IR/Function.h"
00042 #include "llvm/IR/GlobalVariable.h"
00043 #include "llvm/IR/InlineAsm.h"
00044 #include "llvm/IR/Instructions.h"
00045 #include "llvm/IR/IntrinsicInst.h"
00046 #include "llvm/IR/Intrinsics.h"
00047 #include "llvm/IR/LLVMContext.h"
00048 #include "llvm/IR/Module.h"
00049 #include "llvm/Support/CommandLine.h"
00050 #include "llvm/Support/Debug.h"
00051 #include "llvm/Support/ErrorHandling.h"
00052 #include "llvm/Support/MathExtras.h"
00053 #include "llvm/Support/raw_ostream.h"
00054 #include "llvm/Target/TargetFrameLowering.h"
00055 #include "llvm/Target/TargetInstrInfo.h"
00056 #include "llvm/Target/TargetIntrinsicInfo.h"
00057 #include "llvm/Target/TargetLibraryInfo.h"
00058 #include "llvm/Target/TargetLowering.h"
00059 #include "llvm/Target/TargetOptions.h"
00060 #include "llvm/Target/TargetSelectionDAGInfo.h"
00061 #include <algorithm>
00062 using namespace llvm;
00063 
00064 #define DEBUG_TYPE "isel"
00065 
00066 /// LimitFloatPrecision - Generate low-precision inline sequences for
00067 /// some float libcalls (6, 8 or 12 bits).
00068 static unsigned LimitFloatPrecision;
00069 
00070 static cl::opt<unsigned, true>
00071 LimitFPPrecision("limit-float-precision",
00072                  cl::desc("Generate low-precision inline sequences "
00073                           "for some float libcalls"),
00074                  cl::location(LimitFloatPrecision),
00075                  cl::init(0));
00076 
00077 // Limit the width of DAG chains. This is important in general to prevent
00078 // prevent DAG-based analysis from blowing up. For example, alias analysis and
00079 // load clustering may not complete in reasonable time. It is difficult to
00080 // recognize and avoid this situation within each individual analysis, and
00081 // future analyses are likely to have the same behavior. Limiting DAG width is
00082 // the safe approach, and will be especially important with global DAGs.
00083 //
00084 // MaxParallelChains default is arbitrarily high to avoid affecting
00085 // optimization, but could be lowered to improve compile time. Any ld-ld-st-st
00086 // sequence over this should have been converted to llvm.memcpy by the
00087 // frontend. It easy to induce this behavior with .ll code such as:
00088 // %buffer = alloca [4096 x i8]
00089 // %data = load [4096 x i8]* %argPtr
00090 // store [4096 x i8] %data, [4096 x i8]* %buffer
00091 static const unsigned MaxParallelChains = 64;
00092 
00093 static SDValue getCopyFromPartsVector(SelectionDAG &DAG, SDLoc DL,
00094                                       const SDValue *Parts, unsigned NumParts,
00095                                       MVT PartVT, EVT ValueVT, const Value *V);
00096 
00097 /// getCopyFromParts - Create a value that contains the specified legal parts
00098 /// combined into the value they represent.  If the parts combine to a type
00099 /// larger then ValueVT then AssertOp can be used to specify whether the extra
00100 /// bits are known to be zero (ISD::AssertZext) or sign extended from ValueVT
00101 /// (ISD::AssertSext).
00102 static SDValue getCopyFromParts(SelectionDAG &DAG, SDLoc DL,
00103                                 const SDValue *Parts,
00104                                 unsigned NumParts, MVT PartVT, EVT ValueVT,
00105                                 const Value *V,
00106                                 ISD::NodeType AssertOp = ISD::DELETED_NODE) {
00107   if (ValueVT.isVector())
00108     return getCopyFromPartsVector(DAG, DL, Parts, NumParts,
00109                                   PartVT, ValueVT, V);
00110 
00111   assert(NumParts > 0 && "No parts to assemble!");
00112   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
00113   SDValue Val = Parts[0];
00114 
00115   if (NumParts > 1) {
00116     // Assemble the value from multiple parts.
00117     if (ValueVT.isInteger()) {
00118       unsigned PartBits = PartVT.getSizeInBits();
00119       unsigned ValueBits = ValueVT.getSizeInBits();
00120 
00121       // Assemble the power of 2 part.
00122       unsigned RoundParts = NumParts & (NumParts - 1) ?
00123         1 << Log2_32(NumParts) : NumParts;
00124       unsigned RoundBits = PartBits * RoundParts;
00125       EVT RoundVT = RoundBits == ValueBits ?
00126         ValueVT : EVT::getIntegerVT(*DAG.getContext(), RoundBits);
00127       SDValue Lo, Hi;
00128 
00129       EVT HalfVT = EVT::getIntegerVT(*DAG.getContext(), RoundBits/2);
00130 
00131       if (RoundParts > 2) {
00132         Lo = getCopyFromParts(DAG, DL, Parts, RoundParts / 2,
00133                               PartVT, HalfVT, V);
00134         Hi = getCopyFromParts(DAG, DL, Parts + RoundParts / 2,
00135                               RoundParts / 2, PartVT, HalfVT, V);
00136       } else {
00137         Lo = DAG.getNode(ISD::BITCAST, DL, HalfVT, Parts[0]);
00138         Hi = DAG.getNode(ISD::BITCAST, DL, HalfVT, Parts[1]);
00139       }
00140 
00141       if (TLI.isBigEndian())
00142         std::swap(Lo, Hi);
00143 
00144       Val = DAG.getNode(ISD::BUILD_PAIR, DL, RoundVT, Lo, Hi);
00145 
00146       if (RoundParts < NumParts) {
00147         // Assemble the trailing non-power-of-2 part.
00148         unsigned OddParts = NumParts - RoundParts;
00149         EVT OddVT = EVT::getIntegerVT(*DAG.getContext(), OddParts * PartBits);
00150         Hi = getCopyFromParts(DAG, DL,
00151                               Parts + RoundParts, OddParts, PartVT, OddVT, V);
00152 
00153         // Combine the round and odd parts.
00154         Lo = Val;
00155         if (TLI.isBigEndian())
00156           std::swap(Lo, Hi);
00157         EVT TotalVT = EVT::getIntegerVT(*DAG.getContext(), NumParts * PartBits);
00158         Hi = DAG.getNode(ISD::ANY_EXTEND, DL, TotalVT, Hi);
00159         Hi = DAG.getNode(ISD::SHL, DL, TotalVT, Hi,
00160                          DAG.getConstant(Lo.getValueType().getSizeInBits(),
00161                                          TLI.getPointerTy()));
00162         Lo = DAG.getNode(ISD::ZERO_EXTEND, DL, TotalVT, Lo);
00163         Val = DAG.getNode(ISD::OR, DL, TotalVT, Lo, Hi);
00164       }
00165     } else if (PartVT.isFloatingPoint()) {
00166       // FP split into multiple FP parts (for ppcf128)
00167       assert(ValueVT == EVT(MVT::ppcf128) && PartVT == MVT::f64 &&
00168              "Unexpected split");
00169       SDValue Lo, Hi;
00170       Lo = DAG.getNode(ISD::BITCAST, DL, EVT(MVT::f64), Parts[0]);
00171       Hi = DAG.getNode(ISD::BITCAST, DL, EVT(MVT::f64), Parts[1]);
00172       if (TLI.hasBigEndianPartOrdering(ValueVT))
00173         std::swap(Lo, Hi);
00174       Val = DAG.getNode(ISD::BUILD_PAIR, DL, ValueVT, Lo, Hi);
00175     } else {
00176       // FP split into integer parts (soft fp)
00177       assert(ValueVT.isFloatingPoint() && PartVT.isInteger() &&
00178              !PartVT.isVector() && "Unexpected split");
00179       EVT IntVT = EVT::getIntegerVT(*DAG.getContext(), ValueVT.getSizeInBits());
00180       Val = getCopyFromParts(DAG, DL, Parts, NumParts, PartVT, IntVT, V);
00181     }
00182   }
00183 
00184   // There is now one part, held in Val.  Correct it to match ValueVT.
00185   EVT PartEVT = Val.getValueType();
00186 
00187   if (PartEVT == ValueVT)
00188     return Val;
00189 
00190   if (PartEVT.isInteger() && ValueVT.isInteger()) {
00191     if (ValueVT.bitsLT(PartEVT)) {
00192       // For a truncate, see if we have any information to
00193       // indicate whether the truncated bits will always be
00194       // zero or sign-extension.
00195       if (AssertOp != ISD::DELETED_NODE)
00196         Val = DAG.getNode(AssertOp, DL, PartEVT, Val,
00197                           DAG.getValueType(ValueVT));
00198       return DAG.getNode(ISD::TRUNCATE, DL, ValueVT, Val);
00199     }
00200     return DAG.getNode(ISD::ANY_EXTEND, DL, ValueVT, Val);
00201   }
00202 
00203   if (PartEVT.isFloatingPoint() && ValueVT.isFloatingPoint()) {
00204     // FP_ROUND's are always exact here.
00205     if (ValueVT.bitsLT(Val.getValueType()))
00206       return DAG.getNode(ISD::FP_ROUND, DL, ValueVT, Val,
00207                          DAG.getTargetConstant(1, TLI.getPointerTy()));
00208 
00209     return DAG.getNode(ISD::FP_EXTEND, DL, ValueVT, Val);
00210   }
00211 
00212   if (PartEVT.getSizeInBits() == ValueVT.getSizeInBits())
00213     return DAG.getNode(ISD::BITCAST, DL, ValueVT, Val);
00214 
00215   llvm_unreachable("Unknown mismatch!");
00216 }
00217 
00218 static void diagnosePossiblyInvalidConstraint(LLVMContext &Ctx, const Value *V,
00219                                               const Twine &ErrMsg) {
00220   const Instruction *I = dyn_cast_or_null<Instruction>(V);
00221   if (!V)
00222     return Ctx.emitError(ErrMsg);
00223 
00224   const char *AsmError = ", possible invalid constraint for vector type";
00225   if (const CallInst *CI = dyn_cast<CallInst>(I))
00226     if (isa<InlineAsm>(CI->getCalledValue()))
00227       return Ctx.emitError(I, ErrMsg + AsmError);
00228 
00229   return Ctx.emitError(I, ErrMsg);
00230 }
00231 
00232 /// getCopyFromPartsVector - Create a value that contains the specified legal
00233 /// parts combined into the value they represent.  If the parts combine to a
00234 /// type larger then ValueVT then AssertOp can be used to specify whether the
00235 /// extra bits are known to be zero (ISD::AssertZext) or sign extended from
00236 /// ValueVT (ISD::AssertSext).
00237 static SDValue getCopyFromPartsVector(SelectionDAG &DAG, SDLoc DL,
00238                                       const SDValue *Parts, unsigned NumParts,
00239                                       MVT PartVT, EVT ValueVT, const Value *V) {
00240   assert(ValueVT.isVector() && "Not a vector value");
00241   assert(NumParts > 0 && "No parts to assemble!");
00242   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
00243   SDValue Val = Parts[0];
00244 
00245   // Handle a multi-element vector.
00246   if (NumParts > 1) {
00247     EVT IntermediateVT;
00248     MVT RegisterVT;
00249     unsigned NumIntermediates;
00250     unsigned NumRegs =
00251     TLI.getVectorTypeBreakdown(*DAG.getContext(), ValueVT, IntermediateVT,
00252                                NumIntermediates, RegisterVT);
00253     assert(NumRegs == NumParts && "Part count doesn't match vector breakdown!");
00254     NumParts = NumRegs; // Silence a compiler warning.
00255     assert(RegisterVT == PartVT && "Part type doesn't match vector breakdown!");
00256     assert(RegisterVT == Parts[0].getSimpleValueType() &&
00257            "Part type doesn't match part!");
00258 
00259     // Assemble the parts into intermediate operands.
00260     SmallVector<SDValue, 8> Ops(NumIntermediates);
00261     if (NumIntermediates == NumParts) {
00262       // If the register was not expanded, truncate or copy the value,
00263       // as appropriate.
00264       for (unsigned i = 0; i != NumParts; ++i)
00265         Ops[i] = getCopyFromParts(DAG, DL, &Parts[i], 1,
00266                                   PartVT, IntermediateVT, V);
00267     } else if (NumParts > 0) {
00268       // If the intermediate type was expanded, build the intermediate
00269       // operands from the parts.
00270       assert(NumParts % NumIntermediates == 0 &&
00271              "Must expand into a divisible number of parts!");
00272       unsigned Factor = NumParts / NumIntermediates;
00273       for (unsigned i = 0; i != NumIntermediates; ++i)
00274         Ops[i] = getCopyFromParts(DAG, DL, &Parts[i * Factor], Factor,
00275                                   PartVT, IntermediateVT, V);
00276     }
00277 
00278     // Build a vector with BUILD_VECTOR or CONCAT_VECTORS from the
00279     // intermediate operands.
00280     Val = DAG.getNode(IntermediateVT.isVector() ? ISD::CONCAT_VECTORS
00281                                                 : ISD::BUILD_VECTOR,
00282                       DL, ValueVT, Ops);
00283   }
00284 
00285   // There is now one part, held in Val.  Correct it to match ValueVT.
00286   EVT PartEVT = Val.getValueType();
00287 
00288   if (PartEVT == ValueVT)
00289     return Val;
00290 
00291   if (PartEVT.isVector()) {
00292     // If the element type of the source/dest vectors are the same, but the
00293     // parts vector has more elements than the value vector, then we have a
00294     // vector widening case (e.g. <2 x float> -> <4 x float>).  Extract the
00295     // elements we want.
00296     if (PartEVT.getVectorElementType() == ValueVT.getVectorElementType()) {
00297       assert(PartEVT.getVectorNumElements() > ValueVT.getVectorNumElements() &&
00298              "Cannot narrow, it would be a lossy transformation");
00299       return DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, ValueVT, Val,
00300                          DAG.getConstant(0, TLI.getVectorIdxTy()));
00301     }
00302 
00303     // Vector/Vector bitcast.
00304     if (ValueVT.getSizeInBits() == PartEVT.getSizeInBits())
00305       return DAG.getNode(ISD::BITCAST, DL, ValueVT, Val);
00306 
00307     assert(PartEVT.getVectorNumElements() == ValueVT.getVectorNumElements() &&
00308       "Cannot handle this kind of promotion");
00309     // Promoted vector extract
00310     bool Smaller = ValueVT.bitsLE(PartEVT);
00311     return DAG.getNode((Smaller ? ISD::TRUNCATE : ISD::ANY_EXTEND),
00312                        DL, ValueVT, Val);
00313 
00314   }
00315 
00316   // Trivial bitcast if the types are the same size and the destination
00317   // vector type is legal.
00318   if (PartEVT.getSizeInBits() == ValueVT.getSizeInBits() &&
00319       TLI.isTypeLegal(ValueVT))
00320     return DAG.getNode(ISD::BITCAST, DL, ValueVT, Val);
00321 
00322   // Handle cases such as i8 -> <1 x i1>
00323   if (ValueVT.getVectorNumElements() != 1) {
00324     diagnosePossiblyInvalidConstraint(*DAG.getContext(), V,
00325                                       "non-trivial scalar-to-vector conversion");
00326     return DAG.getUNDEF(ValueVT);
00327   }
00328 
00329   if (ValueVT.getVectorNumElements() == 1 &&
00330       ValueVT.getVectorElementType() != PartEVT) {
00331     bool Smaller = ValueVT.bitsLE(PartEVT);
00332     Val = DAG.getNode((Smaller ? ISD::TRUNCATE : ISD::ANY_EXTEND),
00333                        DL, ValueVT.getScalarType(), Val);
00334   }
00335 
00336   return DAG.getNode(ISD::BUILD_VECTOR, DL, ValueVT, Val);
00337 }
00338 
00339 static void getCopyToPartsVector(SelectionDAG &DAG, SDLoc dl,
00340                                  SDValue Val, SDValue *Parts, unsigned NumParts,
00341                                  MVT PartVT, const Value *V);
00342 
00343 /// getCopyToParts - Create a series of nodes that contain the specified value
00344 /// split into legal parts.  If the parts contain more bits than Val, then, for
00345 /// integers, ExtendKind can be used to specify how to generate the extra bits.
00346 static void getCopyToParts(SelectionDAG &DAG, SDLoc DL,
00347                            SDValue Val, SDValue *Parts, unsigned NumParts,
00348                            MVT PartVT, const Value *V,
00349                            ISD::NodeType ExtendKind = ISD::ANY_EXTEND) {
00350   EVT ValueVT = Val.getValueType();
00351 
00352   // Handle the vector case separately.
00353   if (ValueVT.isVector())
00354     return getCopyToPartsVector(DAG, DL, Val, Parts, NumParts, PartVT, V);
00355 
00356   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
00357   unsigned PartBits = PartVT.getSizeInBits();
00358   unsigned OrigNumParts = NumParts;
00359   assert(TLI.isTypeLegal(PartVT) && "Copying to an illegal type!");
00360 
00361   if (NumParts == 0)
00362     return;
00363 
00364   assert(!ValueVT.isVector() && "Vector case handled elsewhere");
00365   EVT PartEVT = PartVT;
00366   if (PartEVT == ValueVT) {
00367     assert(NumParts == 1 && "No-op copy with multiple parts!");
00368     Parts[0] = Val;
00369     return;
00370   }
00371 
00372   if (NumParts * PartBits > ValueVT.getSizeInBits()) {
00373     // If the parts cover more bits than the value has, promote the value.
00374     if (PartVT.isFloatingPoint() && ValueVT.isFloatingPoint()) {
00375       assert(NumParts == 1 && "Do not know what to promote to!");
00376       Val = DAG.getNode(ISD::FP_EXTEND, DL, PartVT, Val);
00377     } else {
00378       assert((PartVT.isInteger() || PartVT == MVT::x86mmx) &&
00379              ValueVT.isInteger() &&
00380              "Unknown mismatch!");
00381       ValueVT = EVT::getIntegerVT(*DAG.getContext(), NumParts * PartBits);
00382       Val = DAG.getNode(ExtendKind, DL, ValueVT, Val);
00383       if (PartVT == MVT::x86mmx)
00384         Val = DAG.getNode(ISD::BITCAST, DL, PartVT, Val);
00385     }
00386   } else if (PartBits == ValueVT.getSizeInBits()) {
00387     // Different types of the same size.
00388     assert(NumParts == 1 && PartEVT != ValueVT);
00389     Val = DAG.getNode(ISD::BITCAST, DL, PartVT, Val);
00390   } else if (NumParts * PartBits < ValueVT.getSizeInBits()) {
00391     // If the parts cover less bits than value has, truncate the value.
00392     assert((PartVT.isInteger() || PartVT == MVT::x86mmx) &&
00393            ValueVT.isInteger() &&
00394            "Unknown mismatch!");
00395     ValueVT = EVT::getIntegerVT(*DAG.getContext(), NumParts * PartBits);
00396     Val = DAG.getNode(ISD::TRUNCATE, DL, ValueVT, Val);
00397     if (PartVT == MVT::x86mmx)
00398       Val = DAG.getNode(ISD::BITCAST, DL, PartVT, Val);
00399   }
00400 
00401   // The value may have changed - recompute ValueVT.
00402   ValueVT = Val.getValueType();
00403   assert(NumParts * PartBits == ValueVT.getSizeInBits() &&
00404          "Failed to tile the value with PartVT!");
00405 
00406   if (NumParts == 1) {
00407     if (PartEVT != ValueVT)
00408       diagnosePossiblyInvalidConstraint(*DAG.getContext(), V,
00409                                         "scalar-to-vector conversion failed");
00410 
00411     Parts[0] = Val;
00412     return;
00413   }
00414 
00415   // Expand the value into multiple parts.
00416   if (NumParts & (NumParts - 1)) {
00417     // The number of parts is not a power of 2.  Split off and copy the tail.
00418     assert(PartVT.isInteger() && ValueVT.isInteger() &&
00419            "Do not know what to expand to!");
00420     unsigned RoundParts = 1 << Log2_32(NumParts);
00421     unsigned RoundBits = RoundParts * PartBits;
00422     unsigned OddParts = NumParts - RoundParts;
00423     SDValue OddVal = DAG.getNode(ISD::SRL, DL, ValueVT, Val,
00424                                  DAG.getIntPtrConstant(RoundBits));
00425     getCopyToParts(DAG, DL, OddVal, Parts + RoundParts, OddParts, PartVT, V);
00426 
00427     if (TLI.isBigEndian())
00428       // The odd parts were reversed by getCopyToParts - unreverse them.
00429       std::reverse(Parts + RoundParts, Parts + NumParts);
00430 
00431     NumParts = RoundParts;
00432     ValueVT = EVT::getIntegerVT(*DAG.getContext(), NumParts * PartBits);
00433     Val = DAG.getNode(ISD::TRUNCATE, DL, ValueVT, Val);
00434   }
00435 
00436   // The number of parts is a power of 2.  Repeatedly bisect the value using
00437   // EXTRACT_ELEMENT.
00438   Parts[0] = DAG.getNode(ISD::BITCAST, DL,
00439                          EVT::getIntegerVT(*DAG.getContext(),
00440                                            ValueVT.getSizeInBits()),
00441                          Val);
00442 
00443   for (unsigned StepSize = NumParts; StepSize > 1; StepSize /= 2) {
00444     for (unsigned i = 0; i < NumParts; i += StepSize) {
00445       unsigned ThisBits = StepSize * PartBits / 2;
00446       EVT ThisVT = EVT::getIntegerVT(*DAG.getContext(), ThisBits);
00447       SDValue &Part0 = Parts[i];
00448       SDValue &Part1 = Parts[i+StepSize/2];
00449 
00450       Part1 = DAG.getNode(ISD::EXTRACT_ELEMENT, DL,
00451                           ThisVT, Part0, DAG.getIntPtrConstant(1));
00452       Part0 = DAG.getNode(ISD::EXTRACT_ELEMENT, DL,
00453                           ThisVT, Part0, DAG.getIntPtrConstant(0));
00454 
00455       if (ThisBits == PartBits && ThisVT != PartVT) {
00456         Part0 = DAG.getNode(ISD::BITCAST, DL, PartVT, Part0);
00457         Part1 = DAG.getNode(ISD::BITCAST, DL, PartVT, Part1);
00458       }
00459     }
00460   }
00461 
00462   if (TLI.isBigEndian())
00463     std::reverse(Parts, Parts + OrigNumParts);
00464 }
00465 
00466 
00467 /// getCopyToPartsVector - Create a series of nodes that contain the specified
00468 /// value split into legal parts.
00469 static void getCopyToPartsVector(SelectionDAG &DAG, SDLoc DL,
00470                                  SDValue Val, SDValue *Parts, unsigned NumParts,
00471                                  MVT PartVT, const Value *V) {
00472   EVT ValueVT = Val.getValueType();
00473   assert(ValueVT.isVector() && "Not a vector");
00474   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
00475 
00476   if (NumParts == 1) {
00477     EVT PartEVT = PartVT;
00478     if (PartEVT == ValueVT) {
00479       // Nothing to do.
00480     } else if (PartVT.getSizeInBits() == ValueVT.getSizeInBits()) {
00481       // Bitconvert vector->vector case.
00482       Val = DAG.getNode(ISD::BITCAST, DL, PartVT, Val);
00483     } else if (PartVT.isVector() &&
00484                PartEVT.getVectorElementType() == ValueVT.getVectorElementType() &&
00485                PartEVT.getVectorNumElements() > ValueVT.getVectorNumElements()) {
00486       EVT ElementVT = PartVT.getVectorElementType();
00487       // Vector widening case, e.g. <2 x float> -> <4 x float>.  Shuffle in
00488       // undef elements.
00489       SmallVector<SDValue, 16> Ops;
00490       for (unsigned i = 0, e = ValueVT.getVectorNumElements(); i != e; ++i)
00491         Ops.push_back(DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL,
00492                                   ElementVT, Val, DAG.getConstant(i,
00493                                                   TLI.getVectorIdxTy())));
00494 
00495       for (unsigned i = ValueVT.getVectorNumElements(),
00496            e = PartVT.getVectorNumElements(); i != e; ++i)
00497         Ops.push_back(DAG.getUNDEF(ElementVT));
00498 
00499       Val = DAG.getNode(ISD::BUILD_VECTOR, DL, PartVT, Ops);
00500 
00501       // FIXME: Use CONCAT for 2x -> 4x.
00502 
00503       //SDValue UndefElts = DAG.getUNDEF(VectorTy);
00504       //Val = DAG.getNode(ISD::CONCAT_VECTORS, DL, PartVT, Val, UndefElts);
00505     } else if (PartVT.isVector() &&
00506                PartEVT.getVectorElementType().bitsGE(
00507                  ValueVT.getVectorElementType()) &&
00508                PartEVT.getVectorNumElements() == ValueVT.getVectorNumElements()) {
00509 
00510       // Promoted vector extract
00511       bool Smaller = PartEVT.bitsLE(ValueVT);
00512       Val = DAG.getNode((Smaller ? ISD::TRUNCATE : ISD::ANY_EXTEND),
00513                         DL, PartVT, Val);
00514     } else{
00515       // Vector -> scalar conversion.
00516       assert(ValueVT.getVectorNumElements() == 1 &&
00517              "Only trivial vector-to-scalar conversions should get here!");
00518       Val = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL,
00519                         PartVT, Val, DAG.getConstant(0, TLI.getVectorIdxTy()));
00520 
00521       bool Smaller = ValueVT.bitsLE(PartVT);
00522       Val = DAG.getNode((Smaller ? ISD::TRUNCATE : ISD::ANY_EXTEND),
00523                          DL, PartVT, Val);
00524     }
00525 
00526     Parts[0] = Val;
00527     return;
00528   }
00529 
00530   // Handle a multi-element vector.
00531   EVT IntermediateVT;
00532   MVT RegisterVT;
00533   unsigned NumIntermediates;
00534   unsigned NumRegs = TLI.getVectorTypeBreakdown(*DAG.getContext(), ValueVT,
00535                                                 IntermediateVT,
00536                                                 NumIntermediates, RegisterVT);
00537   unsigned NumElements = ValueVT.getVectorNumElements();
00538 
00539   assert(NumRegs == NumParts && "Part count doesn't match vector breakdown!");
00540   NumParts = NumRegs; // Silence a compiler warning.
00541   assert(RegisterVT == PartVT && "Part type doesn't match vector breakdown!");
00542 
00543   // Split the vector into intermediate operands.
00544   SmallVector<SDValue, 8> Ops(NumIntermediates);
00545   for (unsigned i = 0; i != NumIntermediates; ++i) {
00546     if (IntermediateVT.isVector())
00547       Ops[i] = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL,
00548                            IntermediateVT, Val,
00549                    DAG.getConstant(i * (NumElements / NumIntermediates),
00550                                    TLI.getVectorIdxTy()));
00551     else
00552       Ops[i] = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL,
00553                            IntermediateVT, Val,
00554                            DAG.getConstant(i, TLI.getVectorIdxTy()));
00555   }
00556 
00557   // Split the intermediate operands into legal parts.
00558   if (NumParts == NumIntermediates) {
00559     // If the register was not expanded, promote or copy the value,
00560     // as appropriate.
00561     for (unsigned i = 0; i != NumParts; ++i)
00562       getCopyToParts(DAG, DL, Ops[i], &Parts[i], 1, PartVT, V);
00563   } else if (NumParts > 0) {
00564     // If the intermediate type was expanded, split each the value into
00565     // legal parts.
00566     assert(NumParts % NumIntermediates == 0 &&
00567            "Must expand into a divisible number of parts!");
00568     unsigned Factor = NumParts / NumIntermediates;
00569     for (unsigned i = 0; i != NumIntermediates; ++i)
00570       getCopyToParts(DAG, DL, Ops[i], &Parts[i*Factor], Factor, PartVT, V);
00571   }
00572 }
00573 
00574 namespace {
00575   /// RegsForValue - This struct represents the registers (physical or virtual)
00576   /// that a particular set of values is assigned, and the type information
00577   /// about the value. The most common situation is to represent one value at a
00578   /// time, but struct or array values are handled element-wise as multiple
00579   /// values.  The splitting of aggregates is performed recursively, so that we
00580   /// never have aggregate-typed registers. The values at this point do not
00581   /// necessarily have legal types, so each value may require one or more
00582   /// registers of some legal type.
00583   ///
00584   struct RegsForValue {
00585     /// ValueVTs - The value types of the values, which may not be legal, and
00586     /// may need be promoted or synthesized from one or more registers.
00587     ///
00588     SmallVector<EVT, 4> ValueVTs;
00589 
00590     /// RegVTs - The value types of the registers. This is the same size as
00591     /// ValueVTs and it records, for each value, what the type of the assigned
00592     /// register or registers are. (Individual values are never synthesized
00593     /// from more than one type of register.)
00594     ///
00595     /// With virtual registers, the contents of RegVTs is redundant with TLI's
00596     /// getRegisterType member function, however when with physical registers
00597     /// it is necessary to have a separate record of the types.
00598     ///
00599     SmallVector<MVT, 4> RegVTs;
00600 
00601     /// Regs - This list holds the registers assigned to the values.
00602     /// Each legal or promoted value requires one register, and each
00603     /// expanded value requires multiple registers.
00604     ///
00605     SmallVector<unsigned, 4> Regs;
00606 
00607     RegsForValue() {}
00608 
00609     RegsForValue(const SmallVector<unsigned, 4> &regs,
00610                  MVT regvt, EVT valuevt)
00611       : ValueVTs(1, valuevt), RegVTs(1, regvt), Regs(regs) {}
00612 
00613     RegsForValue(LLVMContext &Context, const TargetLowering &tli,
00614                  unsigned Reg, Type *Ty) {
00615       ComputeValueVTs(tli, Ty, ValueVTs);
00616 
00617       for (unsigned Value = 0, e = ValueVTs.size(); Value != e; ++Value) {
00618         EVT ValueVT = ValueVTs[Value];
00619         unsigned NumRegs = tli.getNumRegisters(Context, ValueVT);
00620         MVT RegisterVT = tli.getRegisterType(Context, ValueVT);
00621         for (unsigned i = 0; i != NumRegs; ++i)
00622           Regs.push_back(Reg + i);
00623         RegVTs.push_back(RegisterVT);
00624         Reg += NumRegs;
00625       }
00626     }
00627 
00628     /// append - Add the specified values to this one.
00629     void append(const RegsForValue &RHS) {
00630       ValueVTs.append(RHS.ValueVTs.begin(), RHS.ValueVTs.end());
00631       RegVTs.append(RHS.RegVTs.begin(), RHS.RegVTs.end());
00632       Regs.append(RHS.Regs.begin(), RHS.Regs.end());
00633     }
00634 
00635     /// getCopyFromRegs - Emit a series of CopyFromReg nodes that copies from
00636     /// this value and returns the result as a ValueVTs value.  This uses
00637     /// Chain/Flag as the input and updates them for the output Chain/Flag.
00638     /// If the Flag pointer is NULL, no flag is used.
00639     SDValue getCopyFromRegs(SelectionDAG &DAG, FunctionLoweringInfo &FuncInfo,
00640                             SDLoc dl,
00641                             SDValue &Chain, SDValue *Flag,
00642                             const Value *V = nullptr) const;
00643 
00644     /// getCopyToRegs - Emit a series of CopyToReg nodes that copies the
00645     /// specified value into the registers specified by this object.  This uses
00646     /// Chain/Flag as the input and updates them for the output Chain/Flag.
00647     /// If the Flag pointer is NULL, no flag is used.
00648     void getCopyToRegs(SDValue Val, SelectionDAG &DAG, SDLoc dl,
00649                        SDValue &Chain, SDValue *Flag, const Value *V) const;
00650 
00651     /// AddInlineAsmOperands - Add this value to the specified inlineasm node
00652     /// operand list.  This adds the code marker, matching input operand index
00653     /// (if applicable), and includes the number of values added into it.
00654     void AddInlineAsmOperands(unsigned Kind,
00655                               bool HasMatching, unsigned MatchingIdx,
00656                               SelectionDAG &DAG,
00657                               std::vector<SDValue> &Ops) const;
00658   };
00659 }
00660 
00661 /// getCopyFromRegs - Emit a series of CopyFromReg nodes that copies from
00662 /// this value and returns the result as a ValueVT value.  This uses
00663 /// Chain/Flag as the input and updates them for the output Chain/Flag.
00664 /// If the Flag pointer is NULL, no flag is used.
00665 SDValue RegsForValue::getCopyFromRegs(SelectionDAG &DAG,
00666                                       FunctionLoweringInfo &FuncInfo,
00667                                       SDLoc dl,
00668                                       SDValue &Chain, SDValue *Flag,
00669                                       const Value *V) const {
00670   // A Value with type {} or [0 x %t] needs no registers.
00671   if (ValueVTs.empty())
00672     return SDValue();
00673 
00674   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
00675 
00676   // Assemble the legal parts into the final values.
00677   SmallVector<SDValue, 4> Values(ValueVTs.size());
00678   SmallVector<SDValue, 8> Parts;
00679   for (unsigned Value = 0, Part = 0, e = ValueVTs.size(); Value != e; ++Value) {
00680     // Copy the legal parts from the registers.
00681     EVT ValueVT = ValueVTs[Value];
00682     unsigned NumRegs = TLI.getNumRegisters(*DAG.getContext(), ValueVT);
00683     MVT RegisterVT = RegVTs[Value];
00684 
00685     Parts.resize(NumRegs);
00686     for (unsigned i = 0; i != NumRegs; ++i) {
00687       SDValue P;
00688       if (!Flag) {
00689         P = DAG.getCopyFromReg(Chain, dl, Regs[Part+i], RegisterVT);
00690       } else {
00691         P = DAG.getCopyFromReg(Chain, dl, Regs[Part+i], RegisterVT, *Flag);
00692         *Flag = P.getValue(2);
00693       }
00694 
00695       Chain = P.getValue(1);
00696       Parts[i] = P;
00697 
00698       // If the source register was virtual and if we know something about it,
00699       // add an assert node.
00700       if (!TargetRegisterInfo::isVirtualRegister(Regs[Part+i]) ||
00701           !RegisterVT.isInteger() || RegisterVT.isVector())
00702         continue;
00703 
00704       const FunctionLoweringInfo::LiveOutInfo *LOI =
00705         FuncInfo.GetLiveOutRegInfo(Regs[Part+i]);
00706       if (!LOI)
00707         continue;
00708 
00709       unsigned RegSize = RegisterVT.getSizeInBits();
00710       unsigned NumSignBits = LOI->NumSignBits;
00711       unsigned NumZeroBits = LOI->KnownZero.countLeadingOnes();
00712 
00713       if (NumZeroBits == RegSize) {
00714         // The current value is a zero.
00715         // Explicitly express that as it would be easier for
00716         // optimizations to kick in.
00717         Parts[i] = DAG.getConstant(0, RegisterVT);
00718         continue;
00719       }
00720 
00721       // FIXME: We capture more information than the dag can represent.  For
00722       // now, just use the tightest assertzext/assertsext possible.
00723       bool isSExt = true;
00724       EVT FromVT(MVT::Other);
00725       if (NumSignBits == RegSize)
00726         isSExt = true, FromVT = MVT::i1;   // ASSERT SEXT 1
00727       else if (NumZeroBits >= RegSize-1)
00728         isSExt = false, FromVT = MVT::i1;  // ASSERT ZEXT 1
00729       else if (NumSignBits > RegSize-8)
00730         isSExt = true, FromVT = MVT::i8;   // ASSERT SEXT 8
00731       else if (NumZeroBits >= RegSize-8)
00732         isSExt = false, FromVT = MVT::i8;  // ASSERT ZEXT 8
00733       else if (NumSignBits > RegSize-16)
00734         isSExt = true, FromVT = MVT::i16;  // ASSERT SEXT 16
00735       else if (NumZeroBits >= RegSize-16)
00736         isSExt = false, FromVT = MVT::i16; // ASSERT ZEXT 16
00737       else if (NumSignBits > RegSize-32)
00738         isSExt = true, FromVT = MVT::i32;  // ASSERT SEXT 32
00739       else if (NumZeroBits >= RegSize-32)
00740         isSExt = false, FromVT = MVT::i32; // ASSERT ZEXT 32
00741       else
00742         continue;
00743 
00744       // Add an assertion node.
00745       assert(FromVT != MVT::Other);
00746       Parts[i] = DAG.getNode(isSExt ? ISD::AssertSext : ISD::AssertZext, dl,
00747                              RegisterVT, P, DAG.getValueType(FromVT));
00748     }
00749 
00750     Values[Value] = getCopyFromParts(DAG, dl, Parts.begin(),
00751                                      NumRegs, RegisterVT, ValueVT, V);
00752     Part += NumRegs;
00753     Parts.clear();
00754   }
00755 
00756   return DAG.getNode(ISD::MERGE_VALUES, dl, DAG.getVTList(ValueVTs), Values);
00757 }
00758 
00759 /// getCopyToRegs - Emit a series of CopyToReg nodes that copies the
00760 /// specified value into the registers specified by this object.  This uses
00761 /// Chain/Flag as the input and updates them for the output Chain/Flag.
00762 /// If the Flag pointer is NULL, no flag is used.
00763 void RegsForValue::getCopyToRegs(SDValue Val, SelectionDAG &DAG, SDLoc dl,
00764                                  SDValue &Chain, SDValue *Flag,
00765                                  const Value *V) const {
00766   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
00767 
00768   // Get the list of the values's legal parts.
00769   unsigned NumRegs = Regs.size();
00770   SmallVector<SDValue, 8> Parts(NumRegs);
00771   for (unsigned Value = 0, Part = 0, e = ValueVTs.size(); Value != e; ++Value) {
00772     EVT ValueVT = ValueVTs[Value];
00773     unsigned NumParts = TLI.getNumRegisters(*DAG.getContext(), ValueVT);
00774     MVT RegisterVT = RegVTs[Value];
00775     ISD::NodeType ExtendKind =
00776       TLI.isZExtFree(Val, RegisterVT)? ISD::ZERO_EXTEND: ISD::ANY_EXTEND;
00777 
00778     getCopyToParts(DAG, dl, Val.getValue(Val.getResNo() + Value),
00779                    &Parts[Part], NumParts, RegisterVT, V, ExtendKind);
00780     Part += NumParts;
00781   }
00782 
00783   // Copy the parts into the registers.
00784   SmallVector<SDValue, 8> Chains(NumRegs);
00785   for (unsigned i = 0; i != NumRegs; ++i) {
00786     SDValue Part;
00787     if (!Flag) {
00788       Part = DAG.getCopyToReg(Chain, dl, Regs[i], Parts[i]);
00789     } else {
00790       Part = DAG.getCopyToReg(Chain, dl, Regs[i], Parts[i], *Flag);
00791       *Flag = Part.getValue(1);
00792     }
00793 
00794     Chains[i] = Part.getValue(0);
00795   }
00796 
00797   if (NumRegs == 1 || Flag)
00798     // If NumRegs > 1 && Flag is used then the use of the last CopyToReg is
00799     // flagged to it. That is the CopyToReg nodes and the user are considered
00800     // a single scheduling unit. If we create a TokenFactor and return it as
00801     // chain, then the TokenFactor is both a predecessor (operand) of the
00802     // user as well as a successor (the TF operands are flagged to the user).
00803     // c1, f1 = CopyToReg
00804     // c2, f2 = CopyToReg
00805     // c3     = TokenFactor c1, c2
00806     // ...
00807     //        = op c3, ..., f2
00808     Chain = Chains[NumRegs-1];
00809   else
00810     Chain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Chains);
00811 }
00812 
00813 /// AddInlineAsmOperands - Add this value to the specified inlineasm node
00814 /// operand list.  This adds the code marker and includes the number of
00815 /// values added into it.
00816 void RegsForValue::AddInlineAsmOperands(unsigned Code, bool HasMatching,
00817                                         unsigned MatchingIdx,
00818                                         SelectionDAG &DAG,
00819                                         std::vector<SDValue> &Ops) const {
00820   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
00821 
00822   unsigned Flag = InlineAsm::getFlagWord(Code, Regs.size());
00823   if (HasMatching)
00824     Flag = InlineAsm::getFlagWordForMatchingOp(Flag, MatchingIdx);
00825   else if (!Regs.empty() &&
00826            TargetRegisterInfo::isVirtualRegister(Regs.front())) {
00827     // Put the register class of the virtual registers in the flag word.  That
00828     // way, later passes can recompute register class constraints for inline
00829     // assembly as well as normal instructions.
00830     // Don't do this for tied operands that can use the regclass information
00831     // from the def.
00832     const MachineRegisterInfo &MRI = DAG.getMachineFunction().getRegInfo();
00833     const TargetRegisterClass *RC = MRI.getRegClass(Regs.front());
00834     Flag = InlineAsm::getFlagWordForRegClass(Flag, RC->getID());
00835   }
00836 
00837   SDValue Res = DAG.getTargetConstant(Flag, MVT::i32);
00838   Ops.push_back(Res);
00839 
00840   unsigned SP = TLI.getStackPointerRegisterToSaveRestore();
00841   for (unsigned Value = 0, Reg = 0, e = ValueVTs.size(); Value != e; ++Value) {
00842     unsigned NumRegs = TLI.getNumRegisters(*DAG.getContext(), ValueVTs[Value]);
00843     MVT RegisterVT = RegVTs[Value];
00844     for (unsigned i = 0; i != NumRegs; ++i) {
00845       assert(Reg < Regs.size() && "Mismatch in # registers expected");
00846       unsigned TheReg = Regs[Reg++];
00847       Ops.push_back(DAG.getRegister(TheReg, RegisterVT));
00848 
00849       if (TheReg == SP && Code == InlineAsm::Kind_Clobber) {
00850         // If we clobbered the stack pointer, MFI should know about it.
00851         assert(DAG.getMachineFunction().getFrameInfo()->
00852             hasInlineAsmWithSPAdjust());
00853       }
00854     }
00855   }
00856 }
00857 
00858 void SelectionDAGBuilder::init(GCFunctionInfo *gfi, AliasAnalysis &aa,
00859                                const TargetLibraryInfo *li) {
00860   AA = &aa;
00861   GFI = gfi;
00862   LibInfo = li;
00863   DL = DAG.getTarget().getDataLayout();
00864   Context = DAG.getContext();
00865   LPadToCallSiteMap.clear();
00866 }
00867 
00868 /// clear - Clear out the current SelectionDAG and the associated
00869 /// state and prepare this SelectionDAGBuilder object to be used
00870 /// for a new block. This doesn't clear out information about
00871 /// additional blocks that are needed to complete switch lowering
00872 /// or PHI node updating; that information is cleared out as it is
00873 /// consumed.
00874 void SelectionDAGBuilder::clear() {
00875   NodeMap.clear();
00876   UnusedArgNodeMap.clear();
00877   PendingLoads.clear();
00878   PendingExports.clear();
00879   CurInst = nullptr;
00880   HasTailCall = false;
00881   SDNodeOrder = LowestSDNodeOrder;
00882 }
00883 
00884 /// clearDanglingDebugInfo - Clear the dangling debug information
00885 /// map. This function is separated from the clear so that debug
00886 /// information that is dangling in a basic block can be properly
00887 /// resolved in a different basic block. This allows the
00888 /// SelectionDAG to resolve dangling debug information attached
00889 /// to PHI nodes.
00890 void SelectionDAGBuilder::clearDanglingDebugInfo() {
00891   DanglingDebugInfoMap.clear();
00892 }
00893 
00894 /// getRoot - Return the current virtual root of the Selection DAG,
00895 /// flushing any PendingLoad items. This must be done before emitting
00896 /// a store or any other node that may need to be ordered after any
00897 /// prior load instructions.
00898 ///
00899 SDValue SelectionDAGBuilder::getRoot() {
00900   if (PendingLoads.empty())
00901     return DAG.getRoot();
00902 
00903   if (PendingLoads.size() == 1) {
00904     SDValue Root = PendingLoads[0];
00905     DAG.setRoot(Root);
00906     PendingLoads.clear();
00907     return Root;
00908   }
00909 
00910   // Otherwise, we have to make a token factor node.
00911   SDValue Root = DAG.getNode(ISD::TokenFactor, getCurSDLoc(), MVT::Other,
00912                              PendingLoads);
00913   PendingLoads.clear();
00914   DAG.setRoot(Root);
00915   return Root;
00916 }
00917 
00918 /// getControlRoot - Similar to getRoot, but instead of flushing all the
00919 /// PendingLoad items, flush all the PendingExports items. It is necessary
00920 /// to do this before emitting a terminator instruction.
00921 ///
00922 SDValue SelectionDAGBuilder::getControlRoot() {
00923   SDValue Root = DAG.getRoot();
00924 
00925   if (PendingExports.empty())
00926     return Root;
00927 
00928   // Turn all of the CopyToReg chains into one factored node.
00929   if (Root.getOpcode() != ISD::EntryToken) {
00930     unsigned i = 0, e = PendingExports.size();
00931     for (; i != e; ++i) {
00932       assert(PendingExports[i].getNode()->getNumOperands() > 1);
00933       if (PendingExports[i].getNode()->getOperand(0) == Root)
00934         break;  // Don't add the root if we already indirectly depend on it.
00935     }
00936 
00937     if (i == e)
00938       PendingExports.push_back(Root);
00939   }
00940 
00941   Root = DAG.getNode(ISD::TokenFactor, getCurSDLoc(), MVT::Other,
00942                      PendingExports);
00943   PendingExports.clear();
00944   DAG.setRoot(Root);
00945   return Root;
00946 }
00947 
00948 void SelectionDAGBuilder::visit(const Instruction &I) {
00949   // Set up outgoing PHI node register values before emitting the terminator.
00950   if (isa<TerminatorInst>(&I))
00951     HandlePHINodesInSuccessorBlocks(I.getParent());
00952 
00953   ++SDNodeOrder;
00954 
00955   CurInst = &I;
00956 
00957   visit(I.getOpcode(), I);
00958 
00959   if (!isa<TerminatorInst>(&I) && !HasTailCall)
00960     CopyToExportRegsIfNeeded(&I);
00961 
00962   CurInst = nullptr;
00963 }
00964 
00965 void SelectionDAGBuilder::visitPHI(const PHINode &) {
00966   llvm_unreachable("SelectionDAGBuilder shouldn't visit PHI nodes!");
00967 }
00968 
00969 void SelectionDAGBuilder::visit(unsigned Opcode, const User &I) {
00970   // Note: this doesn't use InstVisitor, because it has to work with
00971   // ConstantExpr's in addition to instructions.
00972   switch (Opcode) {
00973   default: llvm_unreachable("Unknown instruction type encountered!");
00974     // Build the switch statement using the Instruction.def file.
00975 #define HANDLE_INST(NUM, OPCODE, CLASS) \
00976     case Instruction::OPCODE: visit##OPCODE((const CLASS&)I); break;
00977 #include "llvm/IR/Instruction.def"
00978   }
00979 }
00980 
00981 // resolveDanglingDebugInfo - if we saw an earlier dbg_value referring to V,
00982 // generate the debug data structures now that we've seen its definition.
00983 void SelectionDAGBuilder::resolveDanglingDebugInfo(const Value *V,
00984                                                    SDValue Val) {
00985   DanglingDebugInfo &DDI = DanglingDebugInfoMap[V];
00986   if (DDI.getDI()) {
00987     const DbgValueInst *DI = DDI.getDI();
00988     DebugLoc dl = DDI.getdl();
00989     unsigned DbgSDNodeOrder = DDI.getSDNodeOrder();
00990     MDNode *Variable = DI->getVariable();
00991     uint64_t Offset = DI->getOffset();
00992     // A dbg.value for an alloca is always indirect.
00993     bool IsIndirect = isa<AllocaInst>(V) || Offset != 0;
00994     SDDbgValue *SDV;
00995     if (Val.getNode()) {
00996       if (!EmitFuncArgumentDbgValue(V, Variable, Offset, IsIndirect, Val)) {
00997         SDV = DAG.getDbgValue(Variable, Val.getNode(),
00998                               Val.getResNo(), IsIndirect,
00999             Offset, dl, DbgSDNodeOrder);
01000         DAG.AddDbgValue(SDV, Val.getNode(), false);
01001       }
01002     } else
01003       DEBUG(dbgs() << "Dropping debug info for " << *DI << "\n");
01004     DanglingDebugInfoMap[V] = DanglingDebugInfo();
01005   }
01006 }
01007 
01008 /// getValue - Return an SDValue for the given Value.
01009 SDValue SelectionDAGBuilder::getValue(const Value *V) {
01010   // If we already have an SDValue for this value, use it. It's important
01011   // to do this first, so that we don't create a CopyFromReg if we already
01012   // have a regular SDValue.
01013   SDValue &N = NodeMap[V];
01014   if (N.getNode()) return N;
01015 
01016   // If there's a virtual register allocated and initialized for this
01017   // value, use it.
01018   DenseMap<const Value *, unsigned>::iterator It = FuncInfo.ValueMap.find(V);
01019   if (It != FuncInfo.ValueMap.end()) {
01020     unsigned InReg = It->second;
01021     RegsForValue RFV(*DAG.getContext(), *TM.getTargetLowering(),
01022                      InReg, V->getType());
01023     SDValue Chain = DAG.getEntryNode();
01024     N = RFV.getCopyFromRegs(DAG, FuncInfo, getCurSDLoc(), Chain, nullptr, V);
01025     resolveDanglingDebugInfo(V, N);
01026     return N;
01027   }
01028 
01029   // Otherwise create a new SDValue and remember it.
01030   SDValue Val = getValueImpl(V);
01031   NodeMap[V] = Val;
01032   resolveDanglingDebugInfo(V, Val);
01033   return Val;
01034 }
01035 
01036 /// getNonRegisterValue - Return an SDValue for the given Value, but
01037 /// don't look in FuncInfo.ValueMap for a virtual register.
01038 SDValue SelectionDAGBuilder::getNonRegisterValue(const Value *V) {
01039   // If we already have an SDValue for this value, use it.
01040   SDValue &N = NodeMap[V];
01041   if (N.getNode()) return N;
01042 
01043   // Otherwise create a new SDValue and remember it.
01044   SDValue Val = getValueImpl(V);
01045   NodeMap[V] = Val;
01046   resolveDanglingDebugInfo(V, Val);
01047   return Val;
01048 }
01049 
01050 /// getValueImpl - Helper function for getValue and getNonRegisterValue.
01051 /// Create an SDValue for the given value.
01052 SDValue SelectionDAGBuilder::getValueImpl(const Value *V) {
01053   const TargetLowering *TLI = TM.getTargetLowering();
01054 
01055   if (const Constant *C = dyn_cast<Constant>(V)) {
01056     EVT VT = TLI->getValueType(V->getType(), true);
01057 
01058     if (const ConstantInt *CI = dyn_cast<ConstantInt>(C))
01059       return DAG.getConstant(*CI, VT);
01060 
01061     if (const GlobalValue *GV = dyn_cast<GlobalValue>(C))
01062       return DAG.getGlobalAddress(GV, getCurSDLoc(), VT);
01063 
01064     if (isa<ConstantPointerNull>(C)) {
01065       unsigned AS = V->getType()->getPointerAddressSpace();
01066       return DAG.getConstant(0, TLI->getPointerTy(AS));
01067     }
01068 
01069     if (const ConstantFP *CFP = dyn_cast<ConstantFP>(C))
01070       return DAG.getConstantFP(*CFP, VT);
01071 
01072     if (isa<UndefValue>(C) && !V->getType()->isAggregateType())
01073       return DAG.getUNDEF(VT);
01074 
01075     if (const ConstantExpr *CE = dyn_cast<ConstantExpr>(C)) {
01076       visit(CE->getOpcode(), *CE);
01077       SDValue N1 = NodeMap[V];
01078       assert(N1.getNode() && "visit didn't populate the NodeMap!");
01079       return N1;
01080     }
01081 
01082     if (isa<ConstantStruct>(C) || isa<ConstantArray>(C)) {
01083       SmallVector<SDValue, 4> Constants;
01084       for (User::const_op_iterator OI = C->op_begin(), OE = C->op_end();
01085            OI != OE; ++OI) {
01086         SDNode *Val = getValue(*OI).getNode();
01087         // If the operand is an empty aggregate, there are no values.
01088         if (!Val) continue;
01089         // Add each leaf value from the operand to the Constants list
01090         // to form a flattened list of all the values.
01091         for (unsigned i = 0, e = Val->getNumValues(); i != e; ++i)
01092           Constants.push_back(SDValue(Val, i));
01093       }
01094 
01095       return DAG.getMergeValues(Constants, getCurSDLoc());
01096     }
01097 
01098     if (const ConstantDataSequential *CDS =
01099           dyn_cast<ConstantDataSequential>(C)) {
01100       SmallVector<SDValue, 4> Ops;
01101       for (unsigned i = 0, e = CDS->getNumElements(); i != e; ++i) {
01102         SDNode *Val = getValue(CDS->getElementAsConstant(i)).getNode();
01103         // Add each leaf value from the operand to the Constants list
01104         // to form a flattened list of all the values.
01105         for (unsigned i = 0, e = Val->getNumValues(); i != e; ++i)
01106           Ops.push_back(SDValue(Val, i));
01107       }
01108 
01109       if (isa<ArrayType>(CDS->getType()))
01110         return DAG.getMergeValues(Ops, getCurSDLoc());
01111       return NodeMap[V] = DAG.getNode(ISD::BUILD_VECTOR, getCurSDLoc(),
01112                                       VT, Ops);
01113     }
01114 
01115     if (C->getType()->isStructTy() || C->getType()->isArrayTy()) {
01116       assert((isa<ConstantAggregateZero>(C) || isa<UndefValue>(C)) &&
01117              "Unknown struct or array constant!");
01118 
01119       SmallVector<EVT, 4> ValueVTs;
01120       ComputeValueVTs(*TLI, C->getType(), ValueVTs);
01121       unsigned NumElts = ValueVTs.size();
01122       if (NumElts == 0)
01123         return SDValue(); // empty struct
01124       SmallVector<SDValue, 4> Constants(NumElts);
01125       for (unsigned i = 0; i != NumElts; ++i) {
01126         EVT EltVT = ValueVTs[i];
01127         if (isa<UndefValue>(C))
01128           Constants[i] = DAG.getUNDEF(EltVT);
01129         else if (EltVT.isFloatingPoint())
01130           Constants[i] = DAG.getConstantFP(0, EltVT);
01131         else
01132           Constants[i] = DAG.getConstant(0, EltVT);
01133       }
01134 
01135       return DAG.getMergeValues(Constants, getCurSDLoc());
01136     }
01137 
01138     if (const BlockAddress *BA = dyn_cast<BlockAddress>(C))
01139       return DAG.getBlockAddress(BA, VT);
01140 
01141     VectorType *VecTy = cast<VectorType>(V->getType());
01142     unsigned NumElements = VecTy->getNumElements();
01143 
01144     // Now that we know the number and type of the elements, get that number of
01145     // elements into the Ops array based on what kind of constant it is.
01146     SmallVector<SDValue, 16> Ops;
01147     if (const ConstantVector *CV = dyn_cast<ConstantVector>(C)) {
01148       for (unsigned i = 0; i != NumElements; ++i)
01149         Ops.push_back(getValue(CV->getOperand(i)));
01150     } else {
01151       assert(isa<ConstantAggregateZero>(C) && "Unknown vector constant!");
01152       EVT EltVT = TLI->getValueType(VecTy->getElementType());
01153 
01154       SDValue Op;
01155       if (EltVT.isFloatingPoint())
01156         Op = DAG.getConstantFP(0, EltVT);
01157       else
01158         Op = DAG.getConstant(0, EltVT);
01159       Ops.assign(NumElements, Op);
01160     }
01161 
01162     // Create a BUILD_VECTOR node.
01163     return NodeMap[V] = DAG.getNode(ISD::BUILD_VECTOR, getCurSDLoc(), VT, Ops);
01164   }
01165 
01166   // If this is a static alloca, generate it as the frameindex instead of
01167   // computation.
01168   if (const AllocaInst *AI = dyn_cast<AllocaInst>(V)) {
01169     DenseMap<const AllocaInst*, int>::iterator SI =
01170       FuncInfo.StaticAllocaMap.find(AI);
01171     if (SI != FuncInfo.StaticAllocaMap.end())
01172       return DAG.getFrameIndex(SI->second, TLI->getPointerTy());
01173   }
01174 
01175   // If this is an instruction which fast-isel has deferred, select it now.
01176   if (const Instruction *Inst = dyn_cast<Instruction>(V)) {
01177     unsigned InReg = FuncInfo.InitializeRegForValue(Inst);
01178     RegsForValue RFV(*DAG.getContext(), *TLI, InReg, Inst->getType());
01179     SDValue Chain = DAG.getEntryNode();
01180     return RFV.getCopyFromRegs(DAG, FuncInfo, getCurSDLoc(), Chain, nullptr, V);
01181   }
01182 
01183   llvm_unreachable("Can't get register for value!");
01184 }
01185 
01186 void SelectionDAGBuilder::visitRet(const ReturnInst &I) {
01187   const TargetLowering *TLI = TM.getTargetLowering();
01188   SDValue Chain = getControlRoot();
01189   SmallVector<ISD::OutputArg, 8> Outs;
01190   SmallVector<SDValue, 8> OutVals;
01191 
01192   if (!FuncInfo.CanLowerReturn) {
01193     unsigned DemoteReg = FuncInfo.DemoteRegister;
01194     const Function *F = I.getParent()->getParent();
01195 
01196     // Emit a store of the return value through the virtual register.
01197     // Leave Outs empty so that LowerReturn won't try to load return
01198     // registers the usual way.
01199     SmallVector<EVT, 1> PtrValueVTs;
01200     ComputeValueVTs(*TLI, PointerType::getUnqual(F->getReturnType()),
01201                     PtrValueVTs);
01202 
01203     SDValue RetPtr = DAG.getRegister(DemoteReg, PtrValueVTs[0]);
01204     SDValue RetOp = getValue(I.getOperand(0));
01205 
01206     SmallVector<EVT, 4> ValueVTs;
01207     SmallVector<uint64_t, 4> Offsets;
01208     ComputeValueVTs(*TLI, I.getOperand(0)->getType(), ValueVTs, &Offsets);
01209     unsigned NumValues = ValueVTs.size();
01210 
01211     SmallVector<SDValue, 4> Chains(NumValues);
01212     for (unsigned i = 0; i != NumValues; ++i) {
01213       SDValue Add = DAG.getNode(ISD::ADD, getCurSDLoc(),
01214                                 RetPtr.getValueType(), RetPtr,
01215                                 DAG.getIntPtrConstant(Offsets[i]));
01216       Chains[i] =
01217         DAG.getStore(Chain, getCurSDLoc(),
01218                      SDValue(RetOp.getNode(), RetOp.getResNo() + i),
01219                      // FIXME: better loc info would be nice.
01220                      Add, MachinePointerInfo(), false, false, 0);
01221     }
01222 
01223     Chain = DAG.getNode(ISD::TokenFactor, getCurSDLoc(),
01224                         MVT::Other, Chains);
01225   } else if (I.getNumOperands() != 0) {
01226     SmallVector<EVT, 4> ValueVTs;
01227     ComputeValueVTs(*TLI, I.getOperand(0)->getType(), ValueVTs);
01228     unsigned NumValues = ValueVTs.size();
01229     if (NumValues) {
01230       SDValue RetOp = getValue(I.getOperand(0));
01231       for (unsigned j = 0, f = NumValues; j != f; ++j) {
01232         EVT VT = ValueVTs[j];
01233 
01234         ISD::NodeType ExtendKind = ISD::ANY_EXTEND;
01235 
01236         const Function *F = I.getParent()->getParent();
01237         if (F->getAttributes().hasAttribute(AttributeSet::ReturnIndex,
01238                                             Attribute::SExt))
01239           ExtendKind = ISD::SIGN_EXTEND;
01240         else if (F->getAttributes().hasAttribute(AttributeSet::ReturnIndex,
01241                                                  Attribute::ZExt))
01242           ExtendKind = ISD::ZERO_EXTEND;
01243 
01244         if (ExtendKind != ISD::ANY_EXTEND && VT.isInteger())
01245           VT = TLI->getTypeForExtArgOrReturn(VT.getSimpleVT(), ExtendKind);
01246 
01247         unsigned NumParts = TLI->getNumRegisters(*DAG.getContext(), VT);
01248         MVT PartVT = TLI->getRegisterType(*DAG.getContext(), VT);
01249         SmallVector<SDValue, 4> Parts(NumParts);
01250         getCopyToParts(DAG, getCurSDLoc(),
01251                        SDValue(RetOp.getNode(), RetOp.getResNo() + j),
01252                        &Parts[0], NumParts, PartVT, &I, ExtendKind);
01253 
01254         // 'inreg' on function refers to return value
01255         ISD::ArgFlagsTy Flags = ISD::ArgFlagsTy();
01256         if (F->getAttributes().hasAttribute(AttributeSet::ReturnIndex,
01257                                             Attribute::InReg))
01258           Flags.setInReg();
01259 
01260         // Propagate extension type if any
01261         if (ExtendKind == ISD::SIGN_EXTEND)
01262           Flags.setSExt();
01263         else if (ExtendKind == ISD::ZERO_EXTEND)
01264           Flags.setZExt();
01265 
01266         for (unsigned i = 0; i < NumParts; ++i) {
01267           Outs.push_back(ISD::OutputArg(Flags, Parts[i].getValueType(),
01268                                         VT, /*isfixed=*/true, 0, 0));
01269           OutVals.push_back(Parts[i]);
01270         }
01271       }
01272     }
01273   }
01274 
01275   bool isVarArg = DAG.getMachineFunction().getFunction()->isVarArg();
01276   CallingConv::ID CallConv =
01277     DAG.getMachineFunction().getFunction()->getCallingConv();
01278   Chain = TM.getTargetLowering()->LowerReturn(Chain, CallConv, isVarArg,
01279                                               Outs, OutVals, getCurSDLoc(),
01280                                               DAG);
01281 
01282   // Verify that the target's LowerReturn behaved as expected.
01283   assert(Chain.getNode() && Chain.getValueType() == MVT::Other &&
01284          "LowerReturn didn't return a valid chain!");
01285 
01286   // Update the DAG with the new chain value resulting from return lowering.
01287   DAG.setRoot(Chain);
01288 }
01289 
01290 /// CopyToExportRegsIfNeeded - If the given value has virtual registers
01291 /// created for it, emit nodes to copy the value into the virtual
01292 /// registers.
01293 void SelectionDAGBuilder::CopyToExportRegsIfNeeded(const Value *V) {
01294   // Skip empty types
01295   if (V->getType()->isEmptyTy())
01296     return;
01297 
01298   DenseMap<const Value *, unsigned>::iterator VMI = FuncInfo.ValueMap.find(V);
01299   if (VMI != FuncInfo.ValueMap.end()) {
01300     assert(!V->use_empty() && "Unused value assigned virtual registers!");
01301     CopyValueToVirtualRegister(V, VMI->second);
01302   }
01303 }
01304 
01305 /// ExportFromCurrentBlock - If this condition isn't known to be exported from
01306 /// the current basic block, add it to ValueMap now so that we'll get a
01307 /// CopyTo/FromReg.
01308 void SelectionDAGBuilder::ExportFromCurrentBlock(const Value *V) {
01309   // No need to export constants.
01310   if (!isa<Instruction>(V) && !isa<Argument>(V)) return;
01311 
01312   // Already exported?
01313   if (FuncInfo.isExportedInst(V)) return;
01314 
01315   unsigned Reg = FuncInfo.InitializeRegForValue(V);
01316   CopyValueToVirtualRegister(V, Reg);
01317 }
01318 
01319 bool SelectionDAGBuilder::isExportableFromCurrentBlock(const Value *V,
01320                                                      const BasicBlock *FromBB) {
01321   // The operands of the setcc have to be in this block.  We don't know
01322   // how to export them from some other block.
01323   if (const Instruction *VI = dyn_cast<Instruction>(V)) {
01324     // Can export from current BB.
01325     if (VI->getParent() == FromBB)
01326       return true;
01327 
01328     // Is already exported, noop.
01329     return FuncInfo.isExportedInst(V);
01330   }
01331 
01332   // If this is an argument, we can export it if the BB is the entry block or
01333   // if it is already exported.
01334   if (isa<Argument>(V)) {
01335     if (FromBB == &FromBB->getParent()->getEntryBlock())
01336       return true;
01337 
01338     // Otherwise, can only export this if it is already exported.
01339     return FuncInfo.isExportedInst(V);
01340   }
01341 
01342   // Otherwise, constants can always be exported.
01343   return true;
01344 }
01345 
01346 /// Return branch probability calculated by BranchProbabilityInfo for IR blocks.
01347 uint32_t SelectionDAGBuilder::getEdgeWeight(const MachineBasicBlock *Src,
01348                                             const MachineBasicBlock *Dst) const {
01349   BranchProbabilityInfo *BPI = FuncInfo.BPI;
01350   if (!BPI)
01351     return 0;
01352   const BasicBlock *SrcBB = Src->getBasicBlock();
01353   const BasicBlock *DstBB = Dst->getBasicBlock();
01354   return BPI->getEdgeWeight(SrcBB, DstBB);
01355 }
01356 
01357 void SelectionDAGBuilder::
01358 addSuccessorWithWeight(MachineBasicBlock *Src, MachineBasicBlock *Dst,
01359                        uint32_t Weight /* = 0 */) {
01360   if (!Weight)
01361     Weight = getEdgeWeight(Src, Dst);
01362   Src->addSuccessor(Dst, Weight);
01363 }
01364 
01365 
01366 static bool InBlock(const Value *V, const BasicBlock *BB) {
01367   if (const Instruction *I = dyn_cast<Instruction>(V))
01368     return I->getParent() == BB;
01369   return true;
01370 }
01371 
01372 /// EmitBranchForMergedCondition - Helper method for FindMergedConditions.
01373 /// This function emits a branch and is used at the leaves of an OR or an
01374 /// AND operator tree.
01375 ///
01376 void
01377 SelectionDAGBuilder::EmitBranchForMergedCondition(const Value *Cond,
01378                                                   MachineBasicBlock *TBB,
01379                                                   MachineBasicBlock *FBB,
01380                                                   MachineBasicBlock *CurBB,
01381                                                   MachineBasicBlock *SwitchBB,
01382                                                   uint32_t TWeight,
01383                                                   uint32_t FWeight) {
01384   const BasicBlock *BB = CurBB->getBasicBlock();
01385 
01386   // If the leaf of the tree is a comparison, merge the condition into
01387   // the caseblock.
01388   if (const CmpInst *BOp = dyn_cast<CmpInst>(Cond)) {
01389     // The operands of the cmp have to be in this block.  We don't know
01390     // how to export them from some other block.  If this is the first block
01391     // of the sequence, no exporting is needed.
01392     if (CurBB == SwitchBB ||
01393         (isExportableFromCurrentBlock(BOp->getOperand(0), BB) &&
01394          isExportableFromCurrentBlock(BOp->getOperand(1), BB))) {
01395       ISD::CondCode Condition;
01396       if (const ICmpInst *IC = dyn_cast<ICmpInst>(Cond)) {
01397         Condition = getICmpCondCode(IC->getPredicate());
01398       } else if (const FCmpInst *FC = dyn_cast<FCmpInst>(Cond)) {
01399         Condition = getFCmpCondCode(FC->getPredicate());
01400         if (TM.Options.NoNaNsFPMath)
01401           Condition = getFCmpCodeWithoutNaN(Condition);
01402       } else {
01403         Condition = ISD::SETEQ; // silence warning.
01404         llvm_unreachable("Unknown compare instruction");
01405       }
01406 
01407       CaseBlock CB(Condition, BOp->getOperand(0), BOp->getOperand(1), nullptr,
01408                    TBB, FBB, CurBB, TWeight, FWeight);
01409       SwitchCases.push_back(CB);
01410       return;
01411     }
01412   }
01413 
01414   // Create a CaseBlock record representing this branch.
01415   CaseBlock CB(ISD::SETEQ, Cond, ConstantInt::getTrue(*DAG.getContext()),
01416                nullptr, TBB, FBB, CurBB, TWeight, FWeight);
01417   SwitchCases.push_back(CB);
01418 }
01419 
01420 /// Scale down both weights to fit into uint32_t.
01421 static void ScaleWeights(uint64_t &NewTrue, uint64_t &NewFalse) {
01422   uint64_t NewMax = (NewTrue > NewFalse) ? NewTrue : NewFalse;
01423   uint32_t Scale = (NewMax / UINT32_MAX) + 1;
01424   NewTrue = NewTrue / Scale;
01425   NewFalse = NewFalse / Scale;
01426 }
01427 
01428 /// FindMergedConditions - If Cond is an expression like
01429 void SelectionDAGBuilder::FindMergedConditions(const Value *Cond,
01430                                                MachineBasicBlock *TBB,
01431                                                MachineBasicBlock *FBB,
01432                                                MachineBasicBlock *CurBB,
01433                                                MachineBasicBlock *SwitchBB,
01434                                                unsigned Opc, uint32_t TWeight,
01435                                                uint32_t FWeight) {
01436   // If this node is not part of the or/and tree, emit it as a branch.
01437   const Instruction *BOp = dyn_cast<Instruction>(Cond);
01438   if (!BOp || !(isa<BinaryOperator>(BOp) || isa<CmpInst>(BOp)) ||
01439       (unsigned)BOp->getOpcode() != Opc || !BOp->hasOneUse() ||
01440       BOp->getParent() != CurBB->getBasicBlock() ||
01441       !InBlock(BOp->getOperand(0), CurBB->getBasicBlock()) ||
01442       !InBlock(BOp->getOperand(1), CurBB->getBasicBlock())) {
01443     EmitBranchForMergedCondition(Cond, TBB, FBB, CurBB, SwitchBB,
01444                                  TWeight, FWeight);
01445     return;
01446   }
01447 
01448   //  Create TmpBB after CurBB.
01449   MachineFunction::iterator BBI = CurBB;
01450   MachineFunction &MF = DAG.getMachineFunction();
01451   MachineBasicBlock *TmpBB = MF.CreateMachineBasicBlock(CurBB->getBasicBlock());
01452   CurBB->getParent()->insert(++BBI, TmpBB);
01453 
01454   if (Opc == Instruction::Or) {
01455     // Codegen X | Y as:
01456     // BB1:
01457     //   jmp_if_X TBB
01458     //   jmp TmpBB
01459     // TmpBB:
01460     //   jmp_if_Y TBB
01461     //   jmp FBB
01462     //
01463 
01464     // We have flexibility in setting Prob for BB1 and Prob for TmpBB.
01465     // The requirement is that
01466     //   TrueProb for BB1 + (FalseProb for BB1 * TrueProb for TmpBB)
01467     //     = TrueProb for orignal BB.
01468     // Assuming the orignal weights are A and B, one choice is to set BB1's
01469     // weights to A and A+2B, and set TmpBB's weights to A and 2B. This choice
01470     // assumes that
01471     //   TrueProb for BB1 == FalseProb for BB1 * TrueProb for TmpBB.
01472     // Another choice is to assume TrueProb for BB1 equals to TrueProb for
01473     // TmpBB, but the math is more complicated.
01474 
01475     uint64_t NewTrueWeight = TWeight;
01476     uint64_t NewFalseWeight = (uint64_t)TWeight + 2 * (uint64_t)FWeight;
01477     ScaleWeights(NewTrueWeight, NewFalseWeight);
01478     // Emit the LHS condition.
01479     FindMergedConditions(BOp->getOperand(0), TBB, TmpBB, CurBB, SwitchBB, Opc,
01480                          NewTrueWeight, NewFalseWeight);
01481 
01482     NewTrueWeight = TWeight;
01483     NewFalseWeight = 2 * (uint64_t)FWeight;
01484     ScaleWeights(NewTrueWeight, NewFalseWeight);
01485     // Emit the RHS condition into TmpBB.
01486     FindMergedConditions(BOp->getOperand(1), TBB, FBB, TmpBB, SwitchBB, Opc,
01487                          NewTrueWeight, NewFalseWeight);
01488   } else {
01489     assert(Opc == Instruction::And && "Unknown merge op!");
01490     // Codegen X & Y as:
01491     // BB1:
01492     //   jmp_if_X TmpBB
01493     //   jmp FBB
01494     // TmpBB:
01495     //   jmp_if_Y TBB
01496     //   jmp FBB
01497     //
01498     //  This requires creation of TmpBB after CurBB.
01499 
01500     // We have flexibility in setting Prob for BB1 and Prob for TmpBB.
01501     // The requirement is that
01502     //   FalseProb for BB1 + (TrueProb for BB1 * FalseProb for TmpBB)
01503     //     = FalseProb for orignal BB.
01504     // Assuming the orignal weights are A and B, one choice is to set BB1's
01505     // weights to 2A+B and B, and set TmpBB's weights to 2A and B. This choice
01506     // assumes that
01507     //   FalseProb for BB1 == TrueProb for BB1 * FalseProb for TmpBB.
01508 
01509     uint64_t NewTrueWeight = 2 * (uint64_t)TWeight + (uint64_t)FWeight;
01510     uint64_t NewFalseWeight = FWeight;
01511     ScaleWeights(NewTrueWeight, NewFalseWeight);
01512     // Emit the LHS condition.
01513     FindMergedConditions(BOp->getOperand(0), TmpBB, FBB, CurBB, SwitchBB, Opc,
01514                          NewTrueWeight, NewFalseWeight);
01515 
01516     NewTrueWeight = 2 * (uint64_t)TWeight;
01517     NewFalseWeight = FWeight;
01518     ScaleWeights(NewTrueWeight, NewFalseWeight);
01519     // Emit the RHS condition into TmpBB.
01520     FindMergedConditions(BOp->getOperand(1), TBB, FBB, TmpBB, SwitchBB, Opc,
01521                          NewTrueWeight, NewFalseWeight);
01522   }
01523 }
01524 
01525 /// If the set of cases should be emitted as a series of branches, return true.
01526 /// If we should emit this as a bunch of and/or'd together conditions, return
01527 /// false.
01528 bool
01529 SelectionDAGBuilder::ShouldEmitAsBranches(const std::vector<CaseBlock> &Cases) {
01530   if (Cases.size() != 2) return true;
01531 
01532   // If this is two comparisons of the same values or'd or and'd together, they
01533   // will get folded into a single comparison, so don't emit two blocks.
01534   if ((Cases[0].CmpLHS == Cases[1].CmpLHS &&
01535        Cases[0].CmpRHS == Cases[1].CmpRHS) ||
01536       (Cases[0].CmpRHS == Cases[1].CmpLHS &&
01537        Cases[0].CmpLHS == Cases[1].CmpRHS)) {
01538     return false;
01539   }
01540 
01541   // Handle: (X != null) | (Y != null) --> (X|Y) != 0
01542   // Handle: (X == null) & (Y == null) --> (X|Y) == 0
01543   if (Cases[0].CmpRHS == Cases[1].CmpRHS &&
01544       Cases[0].CC == Cases[1].CC &&
01545       isa<Constant>(Cases[0].CmpRHS) &&
01546       cast<Constant>(Cases[0].CmpRHS)->isNullValue()) {
01547     if (Cases[0].CC == ISD::SETEQ && Cases[0].TrueBB == Cases[1].ThisBB)
01548       return false;
01549     if (Cases[0].CC == ISD::SETNE && Cases[0].FalseBB == Cases[1].ThisBB)
01550       return false;
01551   }
01552 
01553   return true;
01554 }
01555 
01556 void SelectionDAGBuilder::visitBr(const BranchInst &I) {
01557   MachineBasicBlock *BrMBB = FuncInfo.MBB;
01558 
01559   // Update machine-CFG edges.
01560   MachineBasicBlock *Succ0MBB = FuncInfo.MBBMap[I.getSuccessor(0)];
01561 
01562   // Figure out which block is immediately after the current one.
01563   MachineBasicBlock *NextBlock = nullptr;
01564   MachineFunction::iterator BBI = BrMBB;
01565   if (++BBI != FuncInfo.MF->end())
01566     NextBlock = BBI;
01567 
01568   if (I.isUnconditional()) {
01569     // Update machine-CFG edges.
01570     BrMBB->addSuccessor(Succ0MBB);
01571 
01572     // If this is not a fall-through branch or optimizations are switched off,
01573     // emit the branch.
01574     if (Succ0MBB != NextBlock || TM.getOptLevel() == CodeGenOpt::None)
01575       DAG.setRoot(DAG.getNode(ISD::BR, getCurSDLoc(),
01576                               MVT::Other, getControlRoot(),
01577                               DAG.getBasicBlock(Succ0MBB)));
01578 
01579     return;
01580   }
01581 
01582   // If this condition is one of the special cases we handle, do special stuff
01583   // now.
01584   const Value *CondVal = I.getCondition();
01585   MachineBasicBlock *Succ1MBB = FuncInfo.MBBMap[I.getSuccessor(1)];
01586 
01587   // If this is a series of conditions that are or'd or and'd together, emit
01588   // this as a sequence of branches instead of setcc's with and/or operations.
01589   // As long as jumps are not expensive, this should improve performance.
01590   // For example, instead of something like:
01591   //     cmp A, B
01592   //     C = seteq
01593   //     cmp D, E
01594   //     F = setle
01595   //     or C, F
01596   //     jnz foo
01597   // Emit:
01598   //     cmp A, B
01599   //     je foo
01600   //     cmp D, E
01601   //     jle foo
01602   //
01603   if (const BinaryOperator *BOp = dyn_cast<BinaryOperator>(CondVal)) {
01604     if (!TM.getTargetLowering()->isJumpExpensive() &&
01605         BOp->hasOneUse() &&
01606         (BOp->getOpcode() == Instruction::And ||
01607          BOp->getOpcode() == Instruction::Or)) {
01608       FindMergedConditions(BOp, Succ0MBB, Succ1MBB, BrMBB, BrMBB,
01609                            BOp->getOpcode(), getEdgeWeight(BrMBB, Succ0MBB),
01610                            getEdgeWeight(BrMBB, Succ1MBB));
01611       // If the compares in later blocks need to use values not currently
01612       // exported from this block, export them now.  This block should always
01613       // be the first entry.
01614       assert(SwitchCases[0].ThisBB == BrMBB && "Unexpected lowering!");
01615 
01616       // Allow some cases to be rejected.
01617       if (ShouldEmitAsBranches(SwitchCases)) {
01618         for (unsigned i = 1, e = SwitchCases.size(); i != e; ++i) {
01619           ExportFromCurrentBlock(SwitchCases[i].CmpLHS);
01620           ExportFromCurrentBlock(SwitchCases[i].CmpRHS);
01621         }
01622 
01623         // Emit the branch for this block.
01624         visitSwitchCase(SwitchCases[0], BrMBB);
01625         SwitchCases.erase(SwitchCases.begin());
01626         return;
01627       }
01628 
01629       // Okay, we decided not to do this, remove any inserted MBB's and clear
01630       // SwitchCases.
01631       for (unsigned i = 1, e = SwitchCases.size(); i != e; ++i)
01632         FuncInfo.MF->erase(SwitchCases[i].ThisBB);
01633 
01634       SwitchCases.clear();
01635     }
01636   }
01637 
01638   // Create a CaseBlock record representing this branch.
01639   CaseBlock CB(ISD::SETEQ, CondVal, ConstantInt::getTrue(*DAG.getContext()),
01640                nullptr, Succ0MBB, Succ1MBB, BrMBB);
01641 
01642   // Use visitSwitchCase to actually insert the fast branch sequence for this
01643   // cond branch.
01644   visitSwitchCase(CB, BrMBB);
01645 }
01646 
01647 /// visitSwitchCase - Emits the necessary code to represent a single node in
01648 /// the binary search tree resulting from lowering a switch instruction.
01649 void SelectionDAGBuilder::visitSwitchCase(CaseBlock &CB,
01650                                           MachineBasicBlock *SwitchBB) {
01651   SDValue Cond;
01652   SDValue CondLHS = getValue(CB.CmpLHS);
01653   SDLoc dl = getCurSDLoc();
01654 
01655   // Build the setcc now.
01656   if (!CB.CmpMHS) {
01657     // Fold "(X == true)" to X and "(X == false)" to !X to
01658     // handle common cases produced by branch lowering.
01659     if (CB.CmpRHS == ConstantInt::getTrue(*DAG.getContext()) &&
01660         CB.CC == ISD::SETEQ)
01661       Cond = CondLHS;
01662     else if (CB.CmpRHS == ConstantInt::getFalse(*DAG.getContext()) &&
01663              CB.CC == ISD::SETEQ) {
01664       SDValue True = DAG.getConstant(1, CondLHS.getValueType());
01665       Cond = DAG.getNode(ISD::XOR, dl, CondLHS.getValueType(), CondLHS, True);
01666     } else
01667       Cond = DAG.getSetCC(dl, MVT::i1, CondLHS, getValue(CB.CmpRHS), CB.CC);
01668   } else {
01669     assert(CB.CC == ISD::SETLE && "Can handle only LE ranges now");
01670 
01671     const APInt& Low = cast<ConstantInt>(CB.CmpLHS)->getValue();
01672     const APInt& High  = cast<ConstantInt>(CB.CmpRHS)->getValue();
01673 
01674     SDValue CmpOp = getValue(CB.CmpMHS);
01675     EVT VT = CmpOp.getValueType();
01676 
01677     if (cast<ConstantInt>(CB.CmpLHS)->isMinValue(true)) {
01678       Cond = DAG.getSetCC(dl, MVT::i1, CmpOp, DAG.getConstant(High, VT),
01679                           ISD::SETLE);
01680     } else {
01681       SDValue SUB = DAG.getNode(ISD::SUB, dl,
01682                                 VT, CmpOp, DAG.getConstant(Low, VT));
01683       Cond = DAG.getSetCC(dl, MVT::i1, SUB,
01684                           DAG.getConstant(High-Low, VT), ISD::SETULE);
01685     }
01686   }
01687 
01688   // Update successor info
01689   addSuccessorWithWeight(SwitchBB, CB.TrueBB, CB.TrueWeight);
01690   // TrueBB and FalseBB are always different unless the incoming IR is
01691   // degenerate. This only happens when running llc on weird IR.
01692   if (CB.TrueBB != CB.FalseBB)
01693     addSuccessorWithWeight(SwitchBB, CB.FalseBB, CB.FalseWeight);
01694 
01695   // Set NextBlock to be the MBB immediately after the current one, if any.
01696   // This is used to avoid emitting unnecessary branches to the next block.
01697   MachineBasicBlock *NextBlock = nullptr;
01698   MachineFunction::iterator BBI = SwitchBB;
01699   if (++BBI != FuncInfo.MF->end())
01700     NextBlock = BBI;
01701 
01702   // If the lhs block is the next block, invert the condition so that we can
01703   // fall through to the lhs instead of the rhs block.
01704   if (CB.TrueBB == NextBlock) {
01705     std::swap(CB.TrueBB, CB.FalseBB);
01706     SDValue True = DAG.getConstant(1, Cond.getValueType());
01707     Cond = DAG.getNode(ISD::XOR, dl, Cond.getValueType(), Cond, True);
01708   }
01709 
01710   SDValue BrCond = DAG.getNode(ISD::BRCOND, dl,
01711                                MVT::Other, getControlRoot(), Cond,
01712                                DAG.getBasicBlock(CB.TrueBB));
01713 
01714   // Insert the false branch. Do this even if it's a fall through branch,
01715   // this makes it easier to do DAG optimizations which require inverting
01716   // the branch condition.
01717   BrCond = DAG.getNode(ISD::BR, dl, MVT::Other, BrCond,
01718                        DAG.getBasicBlock(CB.FalseBB));
01719 
01720   DAG.setRoot(BrCond);
01721 }
01722 
01723 /// visitJumpTable - Emit JumpTable node in the current MBB
01724 void SelectionDAGBuilder::visitJumpTable(JumpTable &JT) {
01725   // Emit the code for the jump table
01726   assert(JT.Reg != -1U && "Should lower JT Header first!");
01727   EVT PTy = TM.getTargetLowering()->getPointerTy();
01728   SDValue Index = DAG.getCopyFromReg(getControlRoot(), getCurSDLoc(),
01729                                      JT.Reg, PTy);
01730   SDValue Table = DAG.getJumpTable(JT.JTI, PTy);
01731   SDValue BrJumpTable = DAG.getNode(ISD::BR_JT, getCurSDLoc(),
01732                                     MVT::Other, Index.getValue(1),
01733                                     Table, Index);
01734   DAG.setRoot(BrJumpTable);
01735 }
01736 
01737 /// visitJumpTableHeader - This function emits necessary code to produce index
01738 /// in the JumpTable from switch case.
01739 void SelectionDAGBuilder::visitJumpTableHeader(JumpTable &JT,
01740                                                JumpTableHeader &JTH,
01741                                                MachineBasicBlock *SwitchBB) {
01742   // Subtract the lowest switch case value from the value being switched on and
01743   // conditional branch to default mbb if the result is greater than the
01744   // difference between smallest and largest cases.
01745   SDValue SwitchOp = getValue(JTH.SValue);
01746   EVT VT = SwitchOp.getValueType();
01747   SDValue Sub = DAG.getNode(ISD::SUB, getCurSDLoc(), VT, SwitchOp,
01748                             DAG.getConstant(JTH.First, VT));
01749 
01750   // The SDNode we just created, which holds the value being switched on minus
01751   // the smallest case value, needs to be copied to a virtual register so it
01752   // can be used as an index into the jump table in a subsequent basic block.
01753   // This value may be smaller or larger than the target's pointer type, and
01754   // therefore require extension or truncating.
01755   const TargetLowering *TLI = TM.getTargetLowering();
01756   SwitchOp = DAG.getZExtOrTrunc(Sub, getCurSDLoc(), TLI->getPointerTy());
01757 
01758   unsigned JumpTableReg = FuncInfo.CreateReg(TLI->getPointerTy());
01759   SDValue CopyTo = DAG.getCopyToReg(getControlRoot(), getCurSDLoc(),
01760                                     JumpTableReg, SwitchOp);
01761   JT.Reg = JumpTableReg;
01762 
01763   // Emit the range check for the jump table, and branch to the default block
01764   // for the switch statement if the value being switched on exceeds the largest
01765   // case in the switch.
01766   SDValue CMP = DAG.getSetCC(getCurSDLoc(),
01767                              TLI->getSetCCResultType(*DAG.getContext(),
01768                                                      Sub.getValueType()),
01769                              Sub,
01770                              DAG.getConstant(JTH.Last - JTH.First,VT),
01771                              ISD::SETUGT);
01772 
01773   // Set NextBlock to be the MBB immediately after the current one, if any.
01774   // This is used to avoid emitting unnecessary branches to the next block.
01775   MachineBasicBlock *NextBlock = nullptr;
01776   MachineFunction::iterator BBI = SwitchBB;
01777 
01778   if (++BBI != FuncInfo.MF->end())
01779     NextBlock = BBI;
01780 
01781   SDValue BrCond = DAG.getNode(ISD::BRCOND, getCurSDLoc(),
01782                                MVT::Other, CopyTo, CMP,
01783                                DAG.getBasicBlock(JT.Default));
01784 
01785   if (JT.MBB != NextBlock)
01786     BrCond = DAG.getNode(ISD::BR, getCurSDLoc(), MVT::Other, BrCond,
01787                          DAG.getBasicBlock(JT.MBB));
01788 
01789   DAG.setRoot(BrCond);
01790 }
01791 
01792 /// Codegen a new tail for a stack protector check ParentMBB which has had its
01793 /// tail spliced into a stack protector check success bb.
01794 ///
01795 /// For a high level explanation of how this fits into the stack protector
01796 /// generation see the comment on the declaration of class
01797 /// StackProtectorDescriptor.
01798 void SelectionDAGBuilder::visitSPDescriptorParent(StackProtectorDescriptor &SPD,
01799                                                   MachineBasicBlock *ParentBB) {
01800 
01801   // First create the loads to the guard/stack slot for the comparison.
01802   const TargetLowering *TLI = TM.getTargetLowering();
01803   EVT PtrTy = TLI->getPointerTy();
01804 
01805   MachineFrameInfo *MFI = ParentBB->getParent()->getFrameInfo();
01806   int FI = MFI->getStackProtectorIndex();
01807 
01808   const Value *IRGuard = SPD.getGuard();
01809   SDValue GuardPtr = getValue(IRGuard);
01810   SDValue StackSlotPtr = DAG.getFrameIndex(FI, PtrTy);
01811 
01812   unsigned Align =
01813     TLI->getDataLayout()->getPrefTypeAlignment(IRGuard->getType());
01814   SDValue Guard = DAG.getLoad(PtrTy, getCurSDLoc(), DAG.getEntryNode(),
01815                               GuardPtr, MachinePointerInfo(IRGuard, 0),
01816                               true, false, false, Align);
01817 
01818   SDValue StackSlot = DAG.getLoad(PtrTy, getCurSDLoc(), DAG.getEntryNode(),
01819                                   StackSlotPtr,
01820                                   MachinePointerInfo::getFixedStack(FI),
01821                                   true, false, false, Align);
01822 
01823   // Perform the comparison via a subtract/getsetcc.
01824   EVT VT = Guard.getValueType();
01825   SDValue Sub = DAG.getNode(ISD::SUB, getCurSDLoc(), VT, Guard, StackSlot);
01826 
01827   SDValue Cmp = DAG.getSetCC(getCurSDLoc(),
01828                              TLI->getSetCCResultType(*DAG.getContext(),
01829                                                      Sub.getValueType()),
01830                              Sub, DAG.getConstant(0, VT),
01831                              ISD::SETNE);
01832 
01833   // If the sub is not 0, then we know the guard/stackslot do not equal, so
01834   // branch to failure MBB.
01835   SDValue BrCond = DAG.getNode(ISD::BRCOND, getCurSDLoc(),
01836                                MVT::Other, StackSlot.getOperand(0),
01837                                Cmp, DAG.getBasicBlock(SPD.getFailureMBB()));
01838   // Otherwise branch to success MBB.
01839   SDValue Br = DAG.getNode(ISD::BR, getCurSDLoc(),
01840                            MVT::Other, BrCond,
01841                            DAG.getBasicBlock(SPD.getSuccessMBB()));
01842 
01843   DAG.setRoot(Br);
01844 }
01845 
01846 /// Codegen the failure basic block for a stack protector check.
01847 ///
01848 /// A failure stack protector machine basic block consists simply of a call to
01849 /// __stack_chk_fail().
01850 ///
01851 /// For a high level explanation of how this fits into the stack protector
01852 /// generation see the comment on the declaration of class
01853 /// StackProtectorDescriptor.
01854 void
01855 SelectionDAGBuilder::visitSPDescriptorFailure(StackProtectorDescriptor &SPD) {
01856   const TargetLowering *TLI = TM.getTargetLowering();
01857   SDValue Chain = TLI->makeLibCall(DAG, RTLIB::STACKPROTECTOR_CHECK_FAIL,
01858                                    MVT::isVoid, nullptr, 0, false,
01859                                    getCurSDLoc(), false, false).second;
01860   DAG.setRoot(Chain);
01861 }
01862 
01863 /// visitBitTestHeader - This function emits necessary code to produce value
01864 /// suitable for "bit tests"
01865 void SelectionDAGBuilder::visitBitTestHeader(BitTestBlock &B,
01866                                              MachineBasicBlock *SwitchBB) {
01867   // Subtract the minimum value
01868   SDValue SwitchOp = getValue(B.SValue);
01869   EVT VT = SwitchOp.getValueType();
01870   SDValue Sub = DAG.getNode(ISD::SUB, getCurSDLoc(), VT, SwitchOp,
01871                             DAG.getConstant(B.First, VT));
01872 
01873   // Check range
01874   const TargetLowering *TLI = TM.getTargetLowering();
01875   SDValue RangeCmp = DAG.getSetCC(getCurSDLoc(),
01876                                   TLI->getSetCCResultType(*DAG.getContext(),
01877                                                          Sub.getValueType()),
01878                                   Sub, DAG.getConstant(B.Range, VT),
01879                                   ISD::SETUGT);
01880 
01881   // Determine the type of the test operands.
01882   bool UsePtrType = false;
01883   if (!TLI->isTypeLegal(VT))
01884     UsePtrType = true;
01885   else {
01886     for (unsigned i = 0, e = B.Cases.size(); i != e; ++i)
01887       if (!isUIntN(VT.getSizeInBits(), B.Cases[i].Mask)) {
01888         // Switch table case range are encoded into series of masks.
01889         // Just use pointer type, it's guaranteed to fit.
01890         UsePtrType = true;
01891         break;
01892       }
01893   }
01894   if (UsePtrType) {
01895     VT = TLI->getPointerTy();
01896     Sub = DAG.getZExtOrTrunc(Sub, getCurSDLoc(), VT);
01897   }
01898 
01899   B.RegVT = VT.getSimpleVT();
01900   B.Reg = FuncInfo.CreateReg(B.RegVT);
01901   SDValue CopyTo = DAG.getCopyToReg(getControlRoot(), getCurSDLoc(),
01902                                     B.Reg, Sub);
01903 
01904   // Set NextBlock to be the MBB immediately after the current one, if any.
01905   // This is used to avoid emitting unnecessary branches to the next block.
01906   MachineBasicBlock *NextBlock = nullptr;
01907   MachineFunction::iterator BBI = SwitchBB;
01908   if (++BBI != FuncInfo.MF->end())
01909     NextBlock = BBI;
01910 
01911   MachineBasicBlock* MBB = B.Cases[0].ThisBB;
01912 
01913   addSuccessorWithWeight(SwitchBB, B.Default);
01914   addSuccessorWithWeight(SwitchBB, MBB);
01915 
01916   SDValue BrRange = DAG.getNode(ISD::BRCOND, getCurSDLoc(),
01917                                 MVT::Other, CopyTo, RangeCmp,
01918                                 DAG.getBasicBlock(B.Default));
01919 
01920   if (MBB != NextBlock)
01921     BrRange = DAG.getNode(ISD::BR, getCurSDLoc(), MVT::Other, CopyTo,
01922                           DAG.getBasicBlock(MBB));
01923 
01924   DAG.setRoot(BrRange);
01925 }
01926 
01927 /// visitBitTestCase - this function produces one "bit test"
01928 void SelectionDAGBuilder::visitBitTestCase(BitTestBlock &BB,
01929                                            MachineBasicBlock* NextMBB,
01930                                            uint32_t BranchWeightToNext,
01931                                            unsigned Reg,
01932                                            BitTestCase &B,
01933                                            MachineBasicBlock *SwitchBB) {
01934   MVT VT = BB.RegVT;
01935   SDValue ShiftOp = DAG.getCopyFromReg(getControlRoot(), getCurSDLoc(),
01936                                        Reg, VT);
01937   SDValue Cmp;
01938   unsigned PopCount = CountPopulation_64(B.Mask);
01939   const TargetLowering *TLI = TM.getTargetLowering();
01940   if (PopCount == 1) {
01941     // Testing for a single bit; just compare the shift count with what it
01942     // would need to be to shift a 1 bit in that position.
01943     Cmp = DAG.getSetCC(getCurSDLoc(),
01944                        TLI->getSetCCResultType(*DAG.getContext(), VT),
01945                        ShiftOp,
01946                        DAG.getConstant(countTrailingZeros(B.Mask), VT),
01947                        ISD::SETEQ);
01948   } else if (PopCount == BB.Range) {
01949     // There is only one zero bit in the range, test for it directly.
01950     Cmp = DAG.getSetCC(getCurSDLoc(),
01951                        TLI->getSetCCResultType(*DAG.getContext(), VT),
01952                        ShiftOp,
01953                        DAG.getConstant(CountTrailingOnes_64(B.Mask), VT),
01954                        ISD::SETNE);
01955   } else {
01956     // Make desired shift
01957     SDValue SwitchVal = DAG.getNode(ISD::SHL, getCurSDLoc(), VT,
01958                                     DAG.getConstant(1, VT), ShiftOp);
01959 
01960     // Emit bit tests and jumps
01961     SDValue AndOp = DAG.getNode(ISD::AND, getCurSDLoc(),
01962                                 VT, SwitchVal, DAG.getConstant(B.Mask, VT));
01963     Cmp = DAG.getSetCC(getCurSDLoc(),
01964                        TLI->getSetCCResultType(*DAG.getContext(), VT),
01965                        AndOp, DAG.getConstant(0, VT),
01966                        ISD::SETNE);
01967   }
01968 
01969   // The branch weight from SwitchBB to B.TargetBB is B.ExtraWeight.
01970   addSuccessorWithWeight(SwitchBB, B.TargetBB, B.ExtraWeight);
01971   // The branch weight from SwitchBB to NextMBB is BranchWeightToNext.
01972   addSuccessorWithWeight(SwitchBB, NextMBB, BranchWeightToNext);
01973 
01974   SDValue BrAnd = DAG.getNode(ISD::BRCOND, getCurSDLoc(),
01975                               MVT::Other, getControlRoot(),
01976                               Cmp, DAG.getBasicBlock(B.TargetBB));
01977 
01978   // Set NextBlock to be the MBB immediately after the current one, if any.
01979   // This is used to avoid emitting unnecessary branches to the next block.
01980   MachineBasicBlock *NextBlock = nullptr;
01981   MachineFunction::iterator BBI = SwitchBB;
01982   if (++BBI != FuncInfo.MF->end())
01983     NextBlock = BBI;
01984 
01985   if (NextMBB != NextBlock)
01986     BrAnd = DAG.getNode(ISD::BR, getCurSDLoc(), MVT::Other, BrAnd,
01987                         DAG.getBasicBlock(NextMBB));
01988 
01989   DAG.setRoot(BrAnd);
01990 }
01991 
01992 void SelectionDAGBuilder::visitInvoke(const InvokeInst &I) {
01993   MachineBasicBlock *InvokeMBB = FuncInfo.MBB;
01994 
01995   // Retrieve successors.
01996   MachineBasicBlock *Return = FuncInfo.MBBMap[I.getSuccessor(0)];
01997   MachineBasicBlock *LandingPad = FuncInfo.MBBMap[I.getSuccessor(1)];
01998 
01999   const Value *Callee(I.getCalledValue());
02000   const Function *Fn = dyn_cast<Function>(Callee);
02001   if (isa<InlineAsm>(Callee))
02002     visitInlineAsm(&I);
02003   else if (Fn && Fn->isIntrinsic()) {
02004     assert(Fn->getIntrinsicID() == Intrinsic::donothing);
02005     // Ignore invokes to @llvm.donothing: jump directly to the next BB.
02006   } else
02007     LowerCallTo(&I, getValue(Callee), false, LandingPad);
02008 
02009   // If the value of the invoke is used outside of its defining block, make it
02010   // available as a virtual register.
02011   CopyToExportRegsIfNeeded(&I);
02012 
02013   // Update successor info
02014   addSuccessorWithWeight(InvokeMBB, Return);
02015   addSuccessorWithWeight(InvokeMBB, LandingPad);
02016 
02017   // Drop into normal successor.
02018   DAG.setRoot(DAG.getNode(ISD::BR, getCurSDLoc(),
02019                           MVT::Other, getControlRoot(),
02020                           DAG.getBasicBlock(Return)));
02021 }
02022 
02023 void SelectionDAGBuilder::visitResume(const ResumeInst &RI) {
02024   llvm_unreachable("SelectionDAGBuilder shouldn't visit resume instructions!");
02025 }
02026 
02027 void SelectionDAGBuilder::visitLandingPad(const LandingPadInst &LP) {
02028   assert(FuncInfo.MBB->isLandingPad() &&
02029          "Call to landingpad not in landing pad!");
02030 
02031   MachineBasicBlock *MBB = FuncInfo.MBB;
02032   MachineModuleInfo &MMI = DAG.getMachineFunction().getMMI();
02033   AddLandingPadInfo(LP, MMI, MBB);
02034 
02035   // If there aren't registers to copy the values into (e.g., during SjLj
02036   // exceptions), then don't bother to create these DAG nodes.
02037   const TargetLowering *TLI = TM.getTargetLowering();
02038   if (TLI->getExceptionPointerRegister() == 0 &&
02039       TLI->getExceptionSelectorRegister() == 0)
02040     return;
02041 
02042   SmallVector<EVT, 2> ValueVTs;
02043   ComputeValueVTs(*TLI, LP.getType(), ValueVTs);
02044   assert(ValueVTs.size() == 2 && "Only two-valued landingpads are supported");
02045 
02046   // Get the two live-in registers as SDValues. The physregs have already been
02047   // copied into virtual registers.
02048   SDValue Ops[2];
02049   Ops[0] = DAG.getZExtOrTrunc(
02050     DAG.getCopyFromReg(DAG.getEntryNode(), getCurSDLoc(),
02051                        FuncInfo.ExceptionPointerVirtReg, TLI->getPointerTy()),
02052     getCurSDLoc(), ValueVTs[0]);
02053   Ops[1] = DAG.getZExtOrTrunc(
02054     DAG.getCopyFromReg(DAG.getEntryNode(), getCurSDLoc(),
02055                        FuncInfo.ExceptionSelectorVirtReg, TLI->getPointerTy()),
02056     getCurSDLoc(), ValueVTs[1]);
02057 
02058   // Merge into one.
02059   SDValue Res = DAG.getNode(ISD::MERGE_VALUES, getCurSDLoc(),
02060                             DAG.getVTList(ValueVTs), Ops);
02061   setValue(&LP, Res);
02062 }
02063 
02064 /// handleSmallSwitchCaseRange - Emit a series of specific tests (suitable for
02065 /// small case ranges).
02066 bool SelectionDAGBuilder::handleSmallSwitchRange(CaseRec& CR,
02067                                                  CaseRecVector& WorkList,
02068                                                  const Value* SV,
02069                                                  MachineBasicBlock *Default,
02070                                                  MachineBasicBlock *SwitchBB) {
02071   // Size is the number of Cases represented by this range.
02072   size_t Size = CR.Range.second - CR.Range.first;
02073   if (Size > 3)
02074     return false;
02075 
02076   // Get the MachineFunction which holds the current MBB.  This is used when
02077   // inserting any additional MBBs necessary to represent the switch.
02078   MachineFunction *CurMF = FuncInfo.MF;
02079 
02080   // Figure out which block is immediately after the current one.
02081   MachineBasicBlock *NextBlock = nullptr;
02082   MachineFunction::iterator BBI = CR.CaseBB;
02083 
02084   if (++BBI != FuncInfo.MF->end())
02085     NextBlock = BBI;
02086 
02087   BranchProbabilityInfo *BPI = FuncInfo.BPI;
02088   // If any two of the cases has the same destination, and if one value
02089   // is the same as the other, but has one bit unset that the other has set,
02090   // use bit manipulation to do two compares at once.  For example:
02091   // "if (X == 6 || X == 4)" -> "if ((X|2) == 6)"
02092   // TODO: This could be extended to merge any 2 cases in switches with 3 cases.
02093   // TODO: Handle cases where CR.CaseBB != SwitchBB.
02094   if (Size == 2 && CR.CaseBB == SwitchBB) {
02095     Case &Small = *CR.Range.first;
02096     Case &Big = *(CR.Range.second-1);
02097 
02098     if (Small.Low == Small.High && Big.Low == Big.High && Small.BB == Big.BB) {
02099       const APInt& SmallValue = cast<ConstantInt>(Small.Low)->getValue();
02100       const APInt& BigValue = cast<ConstantInt>(Big.Low)->getValue();
02101 
02102       // Check that there is only one bit different.
02103       if (BigValue.countPopulation() == SmallValue.countPopulation() + 1 &&
02104           (SmallValue | BigValue) == BigValue) {
02105         // Isolate the common bit.
02106         APInt CommonBit = BigValue & ~SmallValue;
02107         assert((SmallValue | CommonBit) == BigValue &&
02108                CommonBit.countPopulation() == 1 && "Not a common bit?");
02109 
02110         SDValue CondLHS = getValue(SV);
02111         EVT VT = CondLHS.getValueType();
02112         SDLoc DL = getCurSDLoc();
02113 
02114         SDValue Or = DAG.getNode(ISD::OR, DL, VT, CondLHS,
02115                                  DAG.getConstant(CommonBit, VT));
02116         SDValue Cond = DAG.getSetCC(DL, MVT::i1,
02117                                     Or, DAG.getConstant(BigValue, VT),
02118                                     ISD::SETEQ);
02119 
02120         // Update successor info.
02121         // Both Small and Big will jump to Small.BB, so we sum up the weights.
02122         addSuccessorWithWeight(SwitchBB, Small.BB,
02123                                Small.ExtraWeight + Big.ExtraWeight);
02124         addSuccessorWithWeight(SwitchBB, Default,
02125           // The default destination is the first successor in IR.
02126           BPI ? BPI->getEdgeWeight(SwitchBB->getBasicBlock(), (unsigned)0) : 0);
02127 
02128         // Insert the true branch.
02129         SDValue BrCond = DAG.getNode(ISD::BRCOND, DL, MVT::Other,
02130                                      getControlRoot(), Cond,
02131                                      DAG.getBasicBlock(Small.BB));
02132 
02133         // Insert the false branch.
02134         BrCond = DAG.getNode(ISD::BR, DL, MVT::Other, BrCond,
02135                              DAG.getBasicBlock(Default));
02136 
02137         DAG.setRoot(BrCond);
02138         return true;
02139       }
02140     }
02141   }
02142 
02143   // Order cases by weight so the most likely case will be checked first.
02144   uint32_t UnhandledWeights = 0;
02145   if (BPI) {
02146     for (CaseItr I = CR.Range.first, IE = CR.Range.second; I != IE; ++I) {
02147       uint32_t IWeight = I->ExtraWeight;
02148       UnhandledWeights += IWeight;
02149       for (CaseItr J = CR.Range.first; J < I; ++J) {
02150         uint32_t JWeight = J->ExtraWeight;
02151         if (IWeight > JWeight)
02152           std::swap(*I, *J);
02153       }
02154     }
02155   }
02156   // Rearrange the case blocks so that the last one falls through if possible.
02157   Case &BackCase = *(CR.Range.second-1);
02158   if (Size > 1 &&
02159       NextBlock && Default != NextBlock && BackCase.BB != NextBlock) {
02160     // The last case block won't fall through into 'NextBlock' if we emit the
02161     // branches in this order.  See if rearranging a case value would help.
02162     // We start at the bottom as it's the case with the least weight.
02163     for (Case *I = &*(CR.Range.second-2), *E = &*CR.Range.first-1; I != E; --I)
02164       if (I->BB == NextBlock) {
02165         std::swap(*I, BackCase);
02166         break;
02167       }
02168   }
02169 
02170   // Create a CaseBlock record representing a conditional branch to
02171   // the Case's target mbb if the value being switched on SV is equal
02172   // to C.
02173   MachineBasicBlock *CurBlock = CR.CaseBB;
02174   for (CaseItr I = CR.Range.first, E = CR.Range.second; I != E; ++I) {
02175     MachineBasicBlock *FallThrough;
02176     if (I != E-1) {
02177       FallThrough = CurMF->CreateMachineBasicBlock(CurBlock->getBasicBlock());
02178       CurMF->insert(BBI, FallThrough);
02179 
02180       // Put SV in a virtual register to make it available from the new blocks.
02181       ExportFromCurrentBlock(SV);
02182     } else {
02183       // If the last case doesn't match, go to the default block.
02184       FallThrough = Default;
02185     }
02186 
02187     const Value *RHS, *LHS, *MHS;
02188     ISD::CondCode CC;
02189     if (I->High == I->Low) {
02190       // This is just small small case range :) containing exactly 1 case
02191       CC = ISD::SETEQ;
02192       LHS = SV; RHS = I->High; MHS = nullptr;
02193     } else {
02194       CC = ISD::SETLE;
02195       LHS = I->Low; MHS = SV; RHS = I->High;
02196     }
02197 
02198     // The false weight should be sum of all un-handled cases.
02199     UnhandledWeights -= I->ExtraWeight;
02200     CaseBlock CB(CC, LHS, RHS, MHS, /* truebb */ I->BB, /* falsebb */ FallThrough,
02201                  /* me */ CurBlock,
02202                  /* trueweight */ I->ExtraWeight,
02203                  /* falseweight */ UnhandledWeights);
02204 
02205     // If emitting the first comparison, just call visitSwitchCase to emit the
02206     // code into the current block.  Otherwise, push the CaseBlock onto the
02207     // vector to be later processed by SDISel, and insert the node's MBB
02208     // before the next MBB.
02209     if (CurBlock == SwitchBB)
02210       visitSwitchCase(CB, SwitchBB);
02211     else
02212       SwitchCases.push_back(CB);
02213 
02214     CurBlock = FallThrough;
02215   }
02216 
02217   return true;
02218 }
02219 
02220 static inline bool areJTsAllowed(const TargetLowering &TLI) {
02221   return TLI.supportJumpTables() &&
02222           (TLI.isOperationLegalOrCustom(ISD::BR_JT, MVT::Other) ||
02223            TLI.isOperationLegalOrCustom(ISD::BRIND, MVT::Other));
02224 }
02225 
02226 static APInt ComputeRange(const APInt &First, const APInt &Last) {
02227   uint32_t BitWidth = std::max(Last.getBitWidth(), First.getBitWidth()) + 1;
02228   APInt LastExt = Last.sext(BitWidth), FirstExt = First.sext(BitWidth);
02229   return (LastExt - FirstExt + 1ULL);
02230 }
02231 
02232 /// handleJTSwitchCase - Emit jumptable for current switch case range
02233 bool SelectionDAGBuilder::handleJTSwitchCase(CaseRec &CR,
02234                                              CaseRecVector &WorkList,
02235                                              const Value *SV,
02236                                              MachineBasicBlock *Default,
02237                                              MachineBasicBlock *SwitchBB) {
02238   Case& FrontCase = *CR.Range.first;
02239   Case& BackCase  = *(CR.Range.second-1);
02240 
02241   const APInt &First = cast<ConstantInt>(FrontCase.Low)->getValue();
02242   const APInt &Last  = cast<ConstantInt>(BackCase.High)->getValue();
02243 
02244   APInt TSize(First.getBitWidth(), 0);
02245   for (CaseItr I = CR.Range.first, E = CR.Range.second; I != E; ++I)
02246     TSize += I->size();
02247 
02248   const TargetLowering *TLI = TM.getTargetLowering();
02249   if (!areJTsAllowed(*TLI) || TSize.ult(TLI->getMinimumJumpTableEntries()))
02250     return false;
02251 
02252   APInt Range = ComputeRange(First, Last);
02253   // The density is TSize / Range. Require at least 40%.
02254   // It should not be possible for IntTSize to saturate for sane code, but make
02255   // sure we handle Range saturation correctly.
02256   uint64_t IntRange = Range.getLimitedValue(UINT64_MAX/10);
02257   uint64_t IntTSize = TSize.getLimitedValue(UINT64_MAX/10);
02258   if (IntTSize * 10 < IntRange * 4)
02259     return false;
02260 
02261   DEBUG(dbgs() << "Lowering jump table\n"
02262                << "First entry: " << First << ". Last entry: " << Last << '\n'
02263                << "Range: " << Range << ". Size: " << TSize << ".\n\n");
02264 
02265   // Get the MachineFunction which holds the current MBB.  This is used when
02266   // inserting any additional MBBs necessary to represent the switch.
02267   MachineFunction *CurMF = FuncInfo.MF;
02268 
02269   // Figure out which block is immediately after the current one.
02270   MachineFunction::iterator BBI = CR.CaseBB;
02271   ++BBI;
02272 
02273   const BasicBlock *LLVMBB = CR.CaseBB->getBasicBlock();
02274 
02275   // Create a new basic block to hold the code for loading the address
02276   // of the jump table, and jumping to it.  Update successor information;
02277   // we will either branch to the default case for the switch, or the jump
02278   // table.
02279   MachineBasicBlock *JumpTableBB = CurMF->CreateMachineBasicBlock(LLVMBB);
02280   CurMF->insert(BBI, JumpTableBB);
02281 
02282   addSuccessorWithWeight(CR.CaseBB, Default);
02283   addSuccessorWithWeight(CR.CaseBB, JumpTableBB);
02284 
02285   // Build a vector of destination BBs, corresponding to each target
02286   // of the jump table. If the value of the jump table slot corresponds to
02287   // a case statement, push the case's BB onto the vector, otherwise, push
02288   // the default BB.
02289   std::vector<MachineBasicBlock*> DestBBs;
02290   APInt TEI = First;
02291   for (CaseItr I = CR.Range.first, E = CR.Range.second; I != E; ++TEI) {
02292     const APInt &Low = cast<ConstantInt>(I->Low)->getValue();
02293     const APInt &High = cast<ConstantInt>(I->High)->getValue();
02294 
02295     if (Low.sle(TEI) && TEI.sle(High)) {
02296       DestBBs.push_back(I->BB);
02297       if (TEI==High)
02298         ++I;
02299     } else {
02300       DestBBs.push_back(Default);
02301     }
02302   }
02303 
02304   // Calculate weight for each unique destination in CR.
02305   DenseMap<MachineBasicBlock*, uint32_t> DestWeights;
02306   if (FuncInfo.BPI)
02307     for (CaseItr I = CR.Range.first, E = CR.Range.second; I != E; ++I) {
02308       DenseMap<MachineBasicBlock*, uint32_t>::iterator Itr =
02309           DestWeights.find(I->BB);
02310       if (Itr != DestWeights.end())
02311         Itr->second += I->ExtraWeight;
02312       else
02313         DestWeights[I->BB] = I->ExtraWeight;
02314     }
02315 
02316   // Update successor info. Add one edge to each unique successor.
02317   BitVector SuccsHandled(CR.CaseBB->getParent()->getNumBlockIDs());
02318   for (std::vector<MachineBasicBlock*>::iterator I = DestBBs.begin(),
02319          E = DestBBs.end(); I != E; ++I) {
02320     if (!SuccsHandled[(*I)->getNumber()]) {
02321       SuccsHandled[(*I)->getNumber()] = true;
02322       DenseMap<MachineBasicBlock*, uint32_t>::iterator Itr =
02323           DestWeights.find(*I);
02324       addSuccessorWithWeight(JumpTableBB, *I,
02325                              Itr != DestWeights.end() ? Itr->second : 0);
02326     }
02327   }
02328 
02329   // Create a jump table index for this jump table.
02330   unsigned JTEncoding = TLI->getJumpTableEncoding();
02331   unsigned JTI = CurMF->getOrCreateJumpTableInfo(JTEncoding)
02332                        ->createJumpTableIndex(DestBBs);
02333 
02334   // Set the jump table information so that we can codegen it as a second
02335   // MachineBasicBlock
02336   JumpTable JT(-1U, JTI, JumpTableBB, Default);
02337   JumpTableHeader JTH(First, Last, SV, CR.CaseBB, (CR.CaseBB == SwitchBB));
02338   if (CR.CaseBB == SwitchBB)
02339     visitJumpTableHeader(JT, JTH, SwitchBB);
02340 
02341   JTCases.push_back(JumpTableBlock(JTH, JT));
02342   return true;
02343 }
02344 
02345 /// handleBTSplitSwitchCase - emit comparison and split binary search tree into
02346 /// 2 subtrees.
02347 bool SelectionDAGBuilder::handleBTSplitSwitchCase(CaseRec& CR,
02348                                                   CaseRecVector& WorkList,
02349                                                   const Value* SV,
02350                                                   MachineBasicBlock* Default,
02351                                                   MachineBasicBlock* SwitchBB) {
02352   // Get the MachineFunction which holds the current MBB.  This is used when
02353   // inserting any additional MBBs necessary to represent the switch.
02354   MachineFunction *CurMF = FuncInfo.MF;
02355 
02356   // Figure out which block is immediately after the current one.
02357   MachineFunction::iterator BBI = CR.CaseBB;
02358   ++BBI;
02359 
02360   Case& FrontCase = *CR.Range.first;
02361   Case& BackCase  = *(CR.Range.second-1);
02362   const BasicBlock *LLVMBB = CR.CaseBB->getBasicBlock();
02363 
02364   // Size is the number of Cases represented by this range.
02365   unsigned Size = CR.Range.second - CR.Range.first;
02366 
02367   const APInt &First = cast<ConstantInt>(FrontCase.Low)->getValue();
02368   const APInt &Last  = cast<ConstantInt>(BackCase.High)->getValue();
02369   double FMetric = 0;
02370   CaseItr Pivot = CR.Range.first + Size/2;
02371 
02372   // Select optimal pivot, maximizing sum density of LHS and RHS. This will
02373   // (heuristically) allow us to emit JumpTable's later.
02374   APInt TSize(First.getBitWidth(), 0);
02375   for (CaseItr I = CR.Range.first, E = CR.Range.second;
02376        I!=E; ++I)
02377     TSize += I->size();
02378 
02379   APInt LSize = FrontCase.size();
02380   APInt RSize = TSize-LSize;
02381   DEBUG(dbgs() << "Selecting best pivot: \n"
02382                << "First: " << First << ", Last: " << Last <<'\n'
02383                << "LSize: " << LSize << ", RSize: " << RSize << '\n');
02384   for (CaseItr I = CR.Range.first, J=I+1, E = CR.Range.second;
02385        J!=E; ++I, ++J) {
02386     const APInt &LEnd = cast<ConstantInt>(I->High)->getValue();
02387     const APInt &RBegin = cast<ConstantInt>(J->Low)->getValue();
02388     APInt Range = ComputeRange(LEnd, RBegin);
02389     assert((Range - 2ULL).isNonNegative() &&
02390            "Invalid case distance");
02391     // Use volatile double here to avoid excess precision issues on some hosts,
02392     // e.g. that use 80-bit X87 registers.
02393     volatile double LDensity =
02394        (double)LSize.roundToDouble() /
02395                            (LEnd - First + 1ULL).roundToDouble();
02396     volatile double RDensity =
02397       (double)RSize.roundToDouble() /
02398                            (Last - RBegin + 1ULL).roundToDouble();
02399     volatile double Metric = Range.logBase2()*(LDensity+RDensity);
02400     // Should always split in some non-trivial place
02401     DEBUG(dbgs() <<"=>Step\n"
02402                  << "LEnd: " << LEnd << ", RBegin: " << RBegin << '\n'
02403                  << "LDensity: " << LDensity
02404                  << ", RDensity: " << RDensity << '\n'
02405                  << "Metric: " << Metric << '\n');
02406     if (FMetric < Metric) {
02407       Pivot = J;
02408       FMetric = Metric;
02409       DEBUG(dbgs() << "Current metric set to: " << FMetric << '\n');
02410     }
02411 
02412     LSize += J->size();
02413     RSize -= J->size();
02414   }
02415 
02416   const TargetLowering *TLI = TM.getTargetLowering();
02417   if (areJTsAllowed(*TLI)) {
02418     // If our case is dense we *really* should handle it earlier!
02419     assert((FMetric > 0) && "Should handle dense range earlier!");
02420   } else {
02421     Pivot = CR.Range.first + Size/2;
02422   }
02423 
02424   CaseRange LHSR(CR.Range.first, Pivot);
02425   CaseRange RHSR(Pivot, CR.Range.second);
02426   const Constant *C = Pivot->Low;
02427   MachineBasicBlock *FalseBB = nullptr, *TrueBB = nullptr;
02428 
02429   // We know that we branch to the LHS if the Value being switched on is
02430   // less than the Pivot value, C.  We use this to optimize our binary
02431   // tree a bit, by recognizing that if SV is greater than or equal to the
02432   // LHS's Case Value, and that Case Value is exactly one less than the
02433   // Pivot's Value, then we can branch directly to the LHS's Target,
02434   // rather than creating a leaf node for it.
02435   if ((LHSR.second - LHSR.first) == 1 &&
02436       LHSR.first->High == CR.GE &&
02437       cast<ConstantInt>(C)->getValue() ==
02438       (cast<ConstantInt>(CR.GE)->getValue() + 1LL)) {
02439     TrueBB = LHSR.first->BB;
02440   } else {
02441     TrueBB = CurMF->CreateMachineBasicBlock(LLVMBB);
02442     CurMF->insert(BBI, TrueBB);
02443     WorkList.push_back(CaseRec(TrueBB, C, CR.GE, LHSR));
02444 
02445     // Put SV in a virtual register to make it available from the new blocks.
02446     ExportFromCurrentBlock(SV);
02447   }
02448 
02449   // Similar to the optimization above, if the Value being switched on is
02450   // known to be less than the Constant CR.LT, and the current Case Value
02451   // is CR.LT - 1, then we can branch directly to the target block for
02452   // the current Case Value, rather than emitting a RHS leaf node for it.
02453   if ((RHSR.second - RHSR.first) == 1 && CR.LT &&
02454       cast<ConstantInt>(RHSR.first->Low)->getValue() ==
02455       (cast<ConstantInt>(CR.LT)->getValue() - 1LL)) {
02456     FalseBB = RHSR.first->BB;
02457   } else {
02458     FalseBB = CurMF->CreateMachineBasicBlock(LLVMBB);
02459     CurMF->insert(BBI, FalseBB);
02460     WorkList.push_back(CaseRec(FalseBB,CR.LT,C,RHSR));
02461 
02462     // Put SV in a virtual register to make it available from the new blocks.
02463     ExportFromCurrentBlock(SV);
02464   }
02465 
02466   // Create a CaseBlock record representing a conditional branch to
02467   // the LHS node if the value being switched on SV is less than C.
02468   // Otherwise, branch to LHS.
02469   CaseBlock CB(ISD::SETLT, SV, C, nullptr, TrueBB, FalseBB, CR.CaseBB);
02470 
02471   if (CR.CaseBB == SwitchBB)
02472     visitSwitchCase(CB, SwitchBB);
02473   else
02474     SwitchCases.push_back(CB);
02475 
02476   return true;
02477 }
02478 
02479 /// handleBitTestsSwitchCase - if current case range has few destination and
02480 /// range span less, than machine word bitwidth, encode case range into series
02481 /// of masks and emit bit tests with these masks.
02482 bool SelectionDAGBuilder::handleBitTestsSwitchCase(CaseRec& CR,
02483                                                    CaseRecVector& WorkList,
02484                                                    const Value* SV,
02485                                                    MachineBasicBlock* Default,
02486                                                    MachineBasicBlock* SwitchBB) {
02487   const TargetLowering *TLI = TM.getTargetLowering();
02488   EVT PTy = TLI->getPointerTy();
02489   unsigned IntPtrBits = PTy.getSizeInBits();
02490 
02491   Case& FrontCase = *CR.Range.first;
02492   Case& BackCase  = *(CR.Range.second-1);
02493 
02494   // Get the MachineFunction which holds the current MBB.  This is used when
02495   // inserting any additional MBBs necessary to represent the switch.
02496   MachineFunction *CurMF = FuncInfo.MF;
02497 
02498   // If target does not have legal shift left, do not emit bit tests at all.
02499   if (!TLI->isOperationLegal(ISD::SHL, PTy))
02500     return false;
02501 
02502   size_t numCmps = 0;
02503   for (CaseItr I = CR.Range.first, E = CR.Range.second;
02504        I!=E; ++I) {
02505     // Single case counts one, case range - two.
02506     numCmps += (I->Low == I->High ? 1 : 2);
02507   }
02508 
02509   // Count unique destinations
02510   SmallSet<MachineBasicBlock*, 4> Dests;
02511   for (CaseItr I = CR.Range.first, E = CR.Range.second; I!=E; ++I) {
02512     Dests.insert(I->BB);
02513     if (Dests.size() > 3)
02514       // Don't bother the code below, if there are too much unique destinations
02515       return false;
02516   }
02517   DEBUG(dbgs() << "Total number of unique destinations: "
02518         << Dests.size() << '\n'
02519         << "Total number of comparisons: " << numCmps << '\n');
02520 
02521   // Compute span of values.
02522   const APInt& minValue = cast<ConstantInt>(FrontCase.Low)->getValue();
02523   const APInt& maxValue = cast<ConstantInt>(BackCase.High)->getValue();
02524   APInt cmpRange = maxValue - minValue;
02525 
02526   DEBUG(dbgs() << "Compare range: " << cmpRange << '\n'
02527                << "Low bound: " << minValue << '\n'
02528                << "High bound: " << maxValue << '\n');
02529 
02530   if (cmpRange.uge(IntPtrBits) ||
02531       (!(Dests.size() == 1 && numCmps >= 3) &&
02532        !(Dests.size() == 2 && numCmps >= 5) &&
02533        !(Dests.size() >= 3 && numCmps >= 6)))
02534     return false;
02535 
02536   DEBUG(dbgs() << "Emitting bit tests\n");
02537   APInt lowBound = APInt::getNullValue(cmpRange.getBitWidth());
02538 
02539   // Optimize the case where all the case values fit in a
02540   // word without having to subtract minValue. In this case,
02541   // we can optimize away the subtraction.
02542   if (minValue.isNonNegative() && maxValue.slt(IntPtrBits)) {
02543     cmpRange = maxValue;
02544   } else {
02545     lowBound = minValue;
02546   }
02547 
02548   CaseBitsVector CasesBits;
02549   unsigned i, count = 0;
02550 
02551   for (CaseItr I = CR.Range.first, E = CR.Range.second; I!=E; ++I) {
02552     MachineBasicBlock* Dest = I->BB;
02553     for (i = 0; i < count; ++i)
02554       if (Dest == CasesBits[i].BB)
02555         break;
02556 
02557     if (i == count) {
02558       assert((count < 3) && "Too much destinations to test!");
02559       CasesBits.push_back(CaseBits(0, Dest, 0, 0/*Weight*/));
02560       count++;
02561     }
02562 
02563     const APInt& lowValue = cast<ConstantInt>(I->Low)->getValue();
02564     const APInt& highValue = cast<ConstantInt>(I->High)->getValue();
02565 
02566     uint64_t lo = (lowValue - lowBound).getZExtValue();
02567     uint64_t hi = (highValue - lowBound).getZExtValue();
02568     CasesBits[i].ExtraWeight += I->ExtraWeight;
02569 
02570     for (uint64_t j = lo; j <= hi; j++) {
02571       CasesBits[i].Mask |=  1ULL << j;
02572       CasesBits[i].Bits++;
02573     }
02574 
02575   }
02576   std::sort(CasesBits.begin(), CasesBits.end(), CaseBitsCmp());
02577 
02578   BitTestInfo BTC;
02579 
02580   // Figure out which block is immediately after the current one.
02581   MachineFunction::iterator BBI = CR.CaseBB;
02582   ++BBI;
02583 
02584   const BasicBlock *LLVMBB = CR.CaseBB->getBasicBlock();
02585 
02586   DEBUG(dbgs() << "Cases:\n");
02587   for (unsigned i = 0, e = CasesBits.size(); i!=e; ++i) {
02588     DEBUG(dbgs() << "Mask: " << CasesBits[i].Mask
02589                  << ", Bits: " << CasesBits[i].Bits
02590                  << ", BB: " << CasesBits[i].BB << '\n');
02591 
02592     MachineBasicBlock *CaseBB = CurMF->CreateMachineBasicBlock(LLVMBB);
02593     CurMF->insert(BBI, CaseBB);
02594     BTC.push_back(BitTestCase(CasesBits[i].Mask,
02595                               CaseBB,
02596                               CasesBits[i].BB, CasesBits[i].ExtraWeight));
02597 
02598     // Put SV in a virtual register to make it available from the new blocks.
02599     ExportFromCurrentBlock(SV);
02600   }
02601 
02602   BitTestBlock BTB(lowBound, cmpRange, SV,
02603                    -1U, MVT::Other, (CR.CaseBB == SwitchBB),
02604                    CR.CaseBB, Default, BTC);
02605 
02606   if (CR.CaseBB == SwitchBB)
02607     visitBitTestHeader(BTB, SwitchBB);
02608 
02609   BitTestCases.push_back(BTB);
02610 
02611   return true;
02612 }
02613 
02614 /// Clusterify - Transform simple list of Cases into list of CaseRange's
02615 size_t SelectionDAGBuilder::Clusterify(CaseVector& Cases,
02616                                        const SwitchInst& SI) {
02617   size_t numCmps = 0;
02618 
02619   BranchProbabilityInfo *BPI = FuncInfo.BPI;
02620   // Start with "simple" cases
02621   for (SwitchInst::ConstCaseIt i = SI.case_begin(), e = SI.case_end();
02622        i != e; ++i) {
02623     const BasicBlock *SuccBB = i.getCaseSuccessor();
02624     MachineBasicBlock *SMBB = FuncInfo.MBBMap[SuccBB];
02625 
02626     uint32_t ExtraWeight =
02627       BPI ? BPI->getEdgeWeight(SI.getParent(), i.getSuccessorIndex()) : 0;
02628 
02629     Cases.push_back(Case(i.getCaseValue(), i.getCaseValue(),
02630                          SMBB, ExtraWeight));
02631   }
02632   std::sort(Cases.begin(), Cases.end(), CaseCmp());
02633 
02634   // Merge case into clusters
02635   if (Cases.size() >= 2)
02636     // Must recompute end() each iteration because it may be
02637     // invalidated by erase if we hold on to it
02638     for (CaseItr I = Cases.begin(), J = std::next(Cases.begin());
02639          J != Cases.end(); ) {
02640       const APInt& nextValue = cast<ConstantInt>(J->Low)->getValue();
02641       const APInt& currentValue = cast<ConstantInt>(I->High)->getValue();
02642       MachineBasicBlock* nextBB = J->BB;
02643       MachineBasicBlock* currentBB = I->BB;
02644 
02645       // If the two neighboring cases go to the same destination, merge them
02646       // into a single case.
02647       if ((nextValue - currentValue == 1) && (currentBB == nextBB)) {
02648         I->High = J->High;
02649         I->ExtraWeight += J->ExtraWeight;
02650         J = Cases.erase(J);
02651       } else {
02652         I = J++;
02653       }
02654     }
02655 
02656   for (CaseItr I=Cases.begin(), E=Cases.end(); I!=E; ++I, ++numCmps) {
02657     if (I->Low != I->High)
02658       // A range counts double, since it requires two compares.
02659       ++numCmps;
02660   }
02661 
02662   return numCmps;
02663 }
02664 
02665 void SelectionDAGBuilder::UpdateSplitBlock(MachineBasicBlock *First,
02666                                            MachineBasicBlock *Last) {
02667   // Update JTCases.
02668   for (unsigned i = 0, e = JTCases.size(); i != e; ++i)
02669     if (JTCases[i].first.HeaderBB == First)
02670       JTCases[i].first.HeaderBB = Last;
02671 
02672   // Update BitTestCases.
02673   for (unsigned i = 0, e = BitTestCases.size(); i != e; ++i)
02674     if (BitTestCases[i].Parent == First)
02675       BitTestCases[i].Parent = Last;
02676 }
02677 
02678 void SelectionDAGBuilder::visitSwitch(const SwitchInst &SI) {
02679   MachineBasicBlock *SwitchMBB = FuncInfo.MBB;
02680 
02681   // Figure out which block is immediately after the current one.
02682   MachineBasicBlock *NextBlock = nullptr;
02683   MachineBasicBlock *Default = FuncInfo.MBBMap[SI.getDefaultDest()];
02684 
02685   // If there is only the default destination, branch to it if it is not the
02686   // next basic block.  Otherwise, just fall through.
02687   if (!SI.getNumCases()) {
02688     // Update machine-CFG edges.
02689 
02690     // If this is not a fall-through branch, emit the branch.
02691     SwitchMBB->addSuccessor(Default);
02692     if (Default != NextBlock)
02693       DAG.setRoot(DAG.getNode(ISD::BR, getCurSDLoc(),
02694                               MVT::Other, getControlRoot(),
02695                               DAG.getBasicBlock(Default)));
02696 
02697     return;
02698   }
02699 
02700   // If there are any non-default case statements, create a vector of Cases
02701   // representing each one, and sort the vector so that we can efficiently
02702   // create a binary search tree from them.
02703   CaseVector Cases;
02704   size_t numCmps = Clusterify(Cases, SI);
02705   DEBUG(dbgs() << "Clusterify finished. Total clusters: " << Cases.size()
02706                << ". Total compares: " << numCmps << '\n');
02707   (void)numCmps;
02708 
02709   // Get the Value to be switched on and default basic blocks, which will be
02710   // inserted into CaseBlock records, representing basic blocks in the binary
02711   // search tree.
02712   const Value *SV = SI.getCondition();
02713 
02714   // Push the initial CaseRec onto the worklist
02715   CaseRecVector WorkList;
02716   WorkList.push_back(CaseRec(SwitchMBB,nullptr,nullptr,
02717                              CaseRange(Cases.begin(),Cases.end())));
02718 
02719   while (!WorkList.empty()) {
02720     // Grab a record representing a case range to process off the worklist
02721     CaseRec CR = WorkList.back();
02722     WorkList.pop_back();
02723 
02724     if (handleBitTestsSwitchCase(CR, WorkList, SV, Default, SwitchMBB))
02725       continue;
02726 
02727     // If the range has few cases (two or less) emit a series of specific
02728     // tests.
02729     if (handleSmallSwitchRange(CR, WorkList, SV, Default, SwitchMBB))
02730       continue;
02731 
02732     // If the switch has more than N blocks, and is at least 40% dense, and the
02733     // target supports indirect branches, then emit a jump table rather than
02734     // lowering the switch to a binary tree of conditional branches.
02735     // N defaults to 4 and is controlled via TLS.getMinimumJumpTableEntries().
02736     if (handleJTSwitchCase(CR, WorkList, SV, Default, SwitchMBB))
02737       continue;
02738 
02739     // Emit binary tree. We need to pick a pivot, and push left and right ranges
02740     // onto the worklist. Leafs are handled via handleSmallSwitchRange() call.
02741     handleBTSplitSwitchCase(CR, WorkList, SV, Default, SwitchMBB);
02742   }
02743 }
02744 
02745 void SelectionDAGBuilder::visitIndirectBr(const IndirectBrInst &I) {
02746   MachineBasicBlock *IndirectBrMBB = FuncInfo.MBB;
02747 
02748   // Update machine-CFG edges with unique successors.
02749   SmallSet<BasicBlock*, 32> Done;
02750   for (unsigned i = 0, e = I.getNumSuccessors(); i != e; ++i) {
02751     BasicBlock *BB = I.getSuccessor(i);
02752     bool Inserted = Done.insert(BB);
02753     if (!Inserted)
02754         continue;
02755 
02756     MachineBasicBlock *Succ = FuncInfo.MBBMap[BB];
02757     addSuccessorWithWeight(IndirectBrMBB, Succ);
02758   }
02759 
02760   DAG.setRoot(DAG.getNode(ISD::BRIND, getCurSDLoc(),
02761                           MVT::Other, getControlRoot(),
02762                           getValue(I.getAddress())));
02763 }
02764 
02765 void SelectionDAGBuilder::visitUnreachable(const UnreachableInst &I) {
02766   if (DAG.getTarget().Options.TrapUnreachable)
02767     DAG.setRoot(DAG.getNode(ISD::TRAP, getCurSDLoc(), MVT::Other, DAG.getRoot()));
02768 }
02769 
02770 void SelectionDAGBuilder::visitFSub(const User &I) {
02771   // -0.0 - X --> fneg
02772   Type *Ty = I.getType();
02773   if (isa<Constant>(I.getOperand(0)) &&
02774       I.getOperand(0) == ConstantFP::getZeroValueForNegation(Ty)) {
02775     SDValue Op2 = getValue(I.getOperand(1));
02776     setValue(&I, DAG.getNode(ISD::FNEG, getCurSDLoc(),
02777                              Op2.getValueType(), Op2));
02778     return;
02779   }
02780 
02781   visitBinary(I, ISD::FSUB);
02782 }
02783 
02784 void SelectionDAGBuilder::visitBinary(const User &I, unsigned OpCode) {
02785   SDValue Op1 = getValue(I.getOperand(0));
02786   SDValue Op2 = getValue(I.getOperand(1));
02787 
02788   bool nuw = false;
02789   bool nsw = false;
02790   bool exact = false;
02791   if (const OverflowingBinaryOperator *OFBinOp =
02792           dyn_cast<const OverflowingBinaryOperator>(&I)) {
02793     nuw = OFBinOp->hasNoUnsignedWrap();
02794     nsw = OFBinOp->hasNoSignedWrap();
02795   }
02796   if (const PossiblyExactOperator *ExactOp =
02797           dyn_cast<const PossiblyExactOperator>(&I))
02798     exact = ExactOp->isExact();
02799 
02800   SDValue BinNodeValue = DAG.getNode(OpCode, getCurSDLoc(), Op1.getValueType(),
02801                                      Op1, Op2, nuw, nsw, exact);
02802   setValue(&I, BinNodeValue);
02803 }
02804 
02805 void SelectionDAGBuilder::visitShift(const User &I, unsigned Opcode) {
02806   SDValue Op1 = getValue(I.getOperand(0));
02807   SDValue Op2 = getValue(I.getOperand(1));
02808 
02809   EVT ShiftTy = TM.getTargetLowering()->getShiftAmountTy(Op2.getValueType());
02810 
02811   // Coerce the shift amount to the right type if we can.
02812   if (!I.getType()->isVectorTy() && Op2.getValueType() != ShiftTy) {
02813     unsigned ShiftSize = ShiftTy.getSizeInBits();
02814     unsigned Op2Size = Op2.getValueType().getSizeInBits();
02815     SDLoc DL = getCurSDLoc();
02816 
02817     // If the operand is smaller than the shift count type, promote it.
02818     if (ShiftSize > Op2Size)
02819       Op2 = DAG.getNode(ISD::ZERO_EXTEND, DL, ShiftTy, Op2);
02820 
02821     // If the operand is larger than the shift count type but the shift
02822     // count type has enough bits to represent any shift value, truncate
02823     // it now. This is a common case and it exposes the truncate to
02824     // optimization early.
02825     else if (ShiftSize >= Log2_32_Ceil(Op2.getValueType().getSizeInBits()))
02826       Op2 = DAG.getNode(ISD::TRUNCATE, DL, ShiftTy, Op2);
02827     // Otherwise we'll need to temporarily settle for some other convenient
02828     // type.  Type legalization will make adjustments once the shiftee is split.
02829     else
02830       Op2 = DAG.getZExtOrTrunc(Op2, DL, MVT::i32);
02831   }
02832 
02833   bool nuw = false;
02834   bool nsw = false;
02835   bool exact = false;
02836 
02837   if (Opcode == ISD::SRL || Opcode == ISD::SRA || Opcode == ISD::SHL) {
02838 
02839     if (const OverflowingBinaryOperator *OFBinOp =
02840             dyn_cast<const OverflowingBinaryOperator>(&I)) {
02841       nuw = OFBinOp->hasNoUnsignedWrap();
02842       nsw = OFBinOp->hasNoSignedWrap();
02843     }
02844     if (const PossiblyExactOperator *ExactOp =
02845             dyn_cast<const PossiblyExactOperator>(&I))
02846       exact = ExactOp->isExact();
02847   }
02848 
02849   SDValue Res = DAG.getNode(Opcode, getCurSDLoc(), Op1.getValueType(), Op1, Op2,
02850                             nuw, nsw, exact);
02851   setValue(&I, Res);
02852 }
02853 
02854 void SelectionDAGBuilder::visitSDiv(const User &I) {
02855   SDValue Op1 = getValue(I.getOperand(0));
02856   SDValue Op2 = getValue(I.getOperand(1));
02857 
02858   // Turn exact SDivs into multiplications.
02859   // FIXME: This should be in DAGCombiner, but it doesn't have access to the
02860   // exact bit.
02861   if (isa<BinaryOperator>(&I) && cast<BinaryOperator>(&I)->isExact() &&
02862       !isa<ConstantSDNode>(Op1) &&
02863       isa<ConstantSDNode>(Op2) && !cast<ConstantSDNode>(Op2)->isNullValue())
02864     setValue(&I, TM.getTargetLowering()->BuildExactSDIV(Op1, Op2,
02865                                                         getCurSDLoc(), DAG));
02866   else
02867     setValue(&I, DAG.getNode(ISD::SDIV, getCurSDLoc(), Op1.getValueType(),
02868                              Op1, Op2));
02869 }
02870 
02871 void SelectionDAGBuilder::visitICmp(const User &I) {
02872   ICmpInst::Predicate predicate = ICmpInst::BAD_ICMP_PREDICATE;
02873   if (const ICmpInst *IC = dyn_cast<ICmpInst>(&I))
02874     predicate = IC->getPredicate();
02875   else if (const ConstantExpr *IC = dyn_cast<ConstantExpr>(&I))
02876     predicate = ICmpInst::Predicate(IC->getPredicate());
02877   SDValue Op1 = getValue(I.getOperand(0));
02878   SDValue Op2 = getValue(I.getOperand(1));
02879   ISD::CondCode Opcode = getICmpCondCode(predicate);
02880 
02881   EVT DestVT = TM.getTargetLowering()->getValueType(I.getType());
02882   setValue(&I, DAG.getSetCC(getCurSDLoc(), DestVT, Op1, Op2, Opcode));
02883 }
02884 
02885 void SelectionDAGBuilder::visitFCmp(const User &I) {
02886   FCmpInst::Predicate predicate = FCmpInst::BAD_FCMP_PREDICATE;
02887   if (const FCmpInst *FC = dyn_cast<FCmpInst>(&I))
02888     predicate = FC->getPredicate();
02889   else if (const ConstantExpr *FC = dyn_cast<ConstantExpr>(&I))
02890     predicate = FCmpInst::Predicate(FC->getPredicate());
02891   SDValue Op1 = getValue(I.getOperand(0));
02892   SDValue Op2 = getValue(I.getOperand(1));
02893   ISD::CondCode Condition = getFCmpCondCode(predicate);
02894   if (TM.Options.NoNaNsFPMath)
02895     Condition = getFCmpCodeWithoutNaN(Condition);
02896   EVT DestVT = TM.getTargetLowering()->getValueType(I.getType());
02897   setValue(&I, DAG.getSetCC(getCurSDLoc(), DestVT, Op1, Op2, Condition));
02898 }
02899 
02900 void SelectionDAGBuilder::visitSelect(const User &I) {
02901   SmallVector<EVT, 4> ValueVTs;
02902   ComputeValueVTs(*TM.getTargetLowering(), I.getType(), ValueVTs);
02903   unsigned NumValues = ValueVTs.size();
02904   if (NumValues == 0) return;
02905 
02906   SmallVector<SDValue, 4> Values(NumValues);
02907   SDValue Cond     = getValue(I.getOperand(0));
02908   SDValue TrueVal  = getValue(I.getOperand(1));
02909   SDValue FalseVal = getValue(I.getOperand(2));
02910   ISD::NodeType OpCode = Cond.getValueType().isVector() ?
02911     ISD::VSELECT : ISD::SELECT;
02912 
02913   for (unsigned i = 0; i != NumValues; ++i)
02914     Values[i] = DAG.getNode(OpCode, getCurSDLoc(),
02915                             TrueVal.getNode()->getValueType(TrueVal.getResNo()+i),
02916                             Cond,
02917                             SDValue(TrueVal.getNode(),
02918                                     TrueVal.getResNo() + i),
02919                             SDValue(FalseVal.getNode(),
02920                                     FalseVal.getResNo() + i));
02921 
02922   setValue(&I, DAG.getNode(ISD::MERGE_VALUES, getCurSDLoc(),
02923                            DAG.getVTList(ValueVTs), Values));
02924 }
02925 
02926 void SelectionDAGBuilder::visitTrunc(const User &I) {
02927   // TruncInst cannot be a no-op cast because sizeof(src) > sizeof(dest).
02928   SDValue N = getValue(I.getOperand(0));
02929   EVT DestVT = TM.getTargetLowering()->getValueType(I.getType());
02930   setValue(&I, DAG.getNode(ISD::TRUNCATE, getCurSDLoc(), DestVT, N));
02931 }
02932 
02933 void SelectionDAGBuilder::visitZExt(const User &I) {
02934   // ZExt cannot be a no-op cast because sizeof(src) < sizeof(dest).
02935   // ZExt also can't be a cast to bool for same reason. So, nothing much to do
02936   SDValue N = getValue(I.getOperand(0));
02937   EVT DestVT = TM.getTargetLowering()->getValueType(I.getType());
02938   setValue(&I, DAG.getNode(ISD::ZERO_EXTEND, getCurSDLoc(), DestVT, N));
02939 }
02940 
02941 void SelectionDAGBuilder::visitSExt(const User &I) {
02942   // SExt cannot be a no-op cast because sizeof(src) < sizeof(dest).
02943   // SExt also can't be a cast to bool for same reason. So, nothing much to do
02944   SDValue N = getValue(I.getOperand(0));
02945   EVT DestVT = TM.getTargetLowering()->getValueType(I.getType());
02946   setValue(&I, DAG.getNode(ISD::SIGN_EXTEND, getCurSDLoc(), DestVT, N));
02947 }
02948 
02949 void SelectionDAGBuilder::visitFPTrunc(const User &I) {
02950   // FPTrunc is never a no-op cast, no need to check
02951   SDValue N = getValue(I.getOperand(0));
02952   const TargetLowering *TLI = TM.getTargetLowering();
02953   EVT DestVT = TLI->getValueType(I.getType());
02954   setValue(&I, DAG.getNode(ISD::FP_ROUND, getCurSDLoc(),
02955                            DestVT, N,
02956                            DAG.getTargetConstant(0, TLI->getPointerTy())));
02957 }
02958 
02959 void SelectionDAGBuilder::visitFPExt(const User &I) {
02960   // FPExt is never a no-op cast, no need to check
02961   SDValue N = getValue(I.getOperand(0));
02962   EVT DestVT = TM.getTargetLowering()->getValueType(I.getType());
02963   setValue(&I, DAG.getNode(ISD::FP_EXTEND, getCurSDLoc(), DestVT, N));
02964 }
02965 
02966 void SelectionDAGBuilder::visitFPToUI(const User &I) {
02967   // FPToUI is never a no-op cast, no need to check
02968   SDValue N = getValue(I.getOperand(0));
02969   EVT DestVT = TM.getTargetLowering()->getValueType(I.getType());
02970   setValue(&I, DAG.getNode(ISD::FP_TO_UINT, getCurSDLoc(), DestVT, N));
02971 }
02972 
02973 void SelectionDAGBuilder::visitFPToSI(const User &I) {
02974   // FPToSI is never a no-op cast, no need to check
02975   SDValue N = getValue(I.getOperand(0));
02976   EVT DestVT = TM.getTargetLowering()->getValueType(I.getType());
02977   setValue(&I, DAG.getNode(ISD::FP_TO_SINT, getCurSDLoc(), DestVT, N));
02978 }
02979 
02980 void SelectionDAGBuilder::visitUIToFP(const User &I) {
02981   // UIToFP is never a no-op cast, no need to check
02982   SDValue N = getValue(I.getOperand(0));
02983   EVT DestVT = TM.getTargetLowering()->getValueType(I.getType());
02984   setValue(&I, DAG.getNode(ISD::UINT_TO_FP, getCurSDLoc(), DestVT, N));
02985 }
02986 
02987 void SelectionDAGBuilder::visitSIToFP(const User &I) {
02988   // SIToFP is never a no-op cast, no need to check
02989   SDValue N = getValue(I.getOperand(0));
02990   EVT DestVT = TM.getTargetLowering()->getValueType(I.getType());
02991   setValue(&I, DAG.getNode(ISD::SINT_TO_FP, getCurSDLoc(), DestVT, N));
02992 }
02993 
02994 void SelectionDAGBuilder::visitPtrToInt(const User &I) {
02995   // What to do depends on the size of the integer and the size of the pointer.
02996   // We can either truncate, zero extend, or no-op, accordingly.
02997   SDValue N = getValue(I.getOperand(0));
02998   EVT DestVT = TM.getTargetLowering()->getValueType(I.getType());
02999   setValue(&I, DAG.getZExtOrTrunc(N, getCurSDLoc(), DestVT));
03000 }
03001 
03002 void SelectionDAGBuilder::visitIntToPtr(const User &I) {
03003   // What to do depends on the size of the integer and the size of the pointer.
03004   // We can either truncate, zero extend, or no-op, accordingly.
03005   SDValue N = getValue(I.getOperand(0));
03006   EVT DestVT = TM.getTargetLowering()->getValueType(I.getType());
03007   setValue(&I, DAG.getZExtOrTrunc(N, getCurSDLoc(), DestVT));
03008 }
03009 
03010 void SelectionDAGBuilder::visitBitCast(const User &I) {
03011   SDValue N = getValue(I.getOperand(0));
03012   EVT DestVT = TM.getTargetLowering()->getValueType(I.getType());
03013 
03014   // BitCast assures us that source and destination are the same size so this is
03015   // either a BITCAST or a no-op.
03016   if (DestVT != N.getValueType())
03017     setValue(&I, DAG.getNode(ISD::BITCAST, getCurSDLoc(),
03018                              DestVT, N)); // convert types.
03019   // Check if the original LLVM IR Operand was a ConstantInt, because getValue()
03020   // might fold any kind of constant expression to an integer constant and that
03021   // is not what we are looking for. Only regcognize a bitcast of a genuine
03022   // constant integer as an opaque constant.
03023   else if(ConstantInt *C = dyn_cast<ConstantInt>(I.getOperand(0)))
03024     setValue(&I, DAG.getConstant(C->getValue(), DestVT, /*isTarget=*/false,
03025                                  /*isOpaque*/true));
03026   else
03027     setValue(&I, N);            // noop cast.
03028 }
03029 
03030 void SelectionDAGBuilder::visitAddrSpaceCast(const User &I) {
03031   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
03032   const Value *SV = I.getOperand(0);
03033   SDValue N = getValue(SV);
03034   EVT DestVT = TM.getTargetLowering()->getValueType(I.getType());
03035 
03036   unsigned SrcAS = SV->getType()->getPointerAddressSpace();
03037   unsigned DestAS = I.getType()->getPointerAddressSpace();
03038 
03039   if (!TLI.isNoopAddrSpaceCast(SrcAS, DestAS))
03040     N = DAG.getAddrSpaceCast(getCurSDLoc(), DestVT, N, SrcAS, DestAS);
03041 
03042   setValue(&I, N);
03043 }
03044 
03045 void SelectionDAGBuilder::visitInsertElement(const User &I) {
03046   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
03047   SDValue InVec = getValue(I.getOperand(0));
03048   SDValue InVal = getValue(I.getOperand(1));
03049   SDValue InIdx = DAG.getSExtOrTrunc(getValue(I.getOperand(2)),
03050                                      getCurSDLoc(), TLI.getVectorIdxTy());
03051   setValue(&I, DAG.getNode(ISD::INSERT_VECTOR_ELT, getCurSDLoc(),
03052                            TM.getTargetLowering()->getValueType(I.getType()),
03053                            InVec, InVal, InIdx));
03054 }
03055 
03056 void SelectionDAGBuilder::visitExtractElement(const User &I) {
03057   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
03058   SDValue InVec = getValue(I.getOperand(0));
03059   SDValue InIdx = DAG.getSExtOrTrunc(getValue(I.getOperand(1)),
03060                                      getCurSDLoc(), TLI.getVectorIdxTy());
03061   setValue(&I, DAG.getNode(ISD::EXTRACT_VECTOR_ELT, getCurSDLoc(),
03062                            TM.getTargetLowering()->getValueType(I.getType()),
03063                            InVec, InIdx));
03064 }
03065 
03066 // Utility for visitShuffleVector - Return true if every element in Mask,
03067 // beginning from position Pos and ending in Pos+Size, falls within the
03068 // specified sequential range [L, L+Pos). or is undef.
03069 static bool isSequentialInRange(const SmallVectorImpl<int> &Mask,
03070                                 unsigned Pos, unsigned Size, int Low) {
03071   for (unsigned i = Pos, e = Pos+Size; i != e; ++i, ++Low)
03072     if (Mask[i] >= 0 && Mask[i] != Low)
03073       return false;
03074   return true;
03075 }
03076 
03077 void SelectionDAGBuilder::visitShuffleVector(const User &I) {
03078   SDValue Src1 = getValue(I.getOperand(0));
03079   SDValue Src2 = getValue(I.getOperand(1));
03080 
03081   SmallVector<int, 8> Mask;
03082   ShuffleVectorInst::getShuffleMask(cast<Constant>(I.getOperand(2)), Mask);
03083   unsigned MaskNumElts = Mask.size();
03084 
03085   const TargetLowering *TLI = TM.getTargetLowering();
03086   EVT VT = TLI->getValueType(I.getType());
03087   EVT SrcVT = Src1.getValueType();
03088   unsigned SrcNumElts = SrcVT.getVectorNumElements();
03089 
03090   if (SrcNumElts == MaskNumElts) {
03091     setValue(&I, DAG.getVectorShuffle(VT, getCurSDLoc(), Src1, Src2,
03092                                       &Mask[0]));
03093     return;
03094   }
03095 
03096   // Normalize the shuffle vector since mask and vector length don't match.
03097   if (SrcNumElts < MaskNumElts && MaskNumElts % SrcNumElts == 0) {
03098     // Mask is longer than the source vectors and is a multiple of the source
03099     // vectors.  We can use concatenate vector to make the mask and vectors
03100     // lengths match.
03101     if (SrcNumElts*2 == MaskNumElts) {
03102       // First check for Src1 in low and Src2 in high
03103       if (isSequentialInRange(Mask, 0, SrcNumElts, 0) &&
03104           isSequentialInRange(Mask, SrcNumElts, SrcNumElts, SrcNumElts)) {
03105         // The shuffle is concatenating two vectors together.
03106         setValue(&I, DAG.getNode(ISD::CONCAT_VECTORS, getCurSDLoc(),
03107                                  VT, Src1, Src2));
03108         return;
03109       }
03110       // Then check for Src2 in low and Src1 in high
03111       if (isSequentialInRange(Mask, 0, SrcNumElts, SrcNumElts) &&
03112           isSequentialInRange(Mask, SrcNumElts, SrcNumElts, 0)) {
03113         // The shuffle is concatenating two vectors together.
03114         setValue(&I, DAG.getNode(ISD::CONCAT_VECTORS, getCurSDLoc(),
03115                                  VT, Src2, Src1));
03116         return;
03117       }
03118     }
03119 
03120     // Pad both vectors with undefs to make them the same length as the mask.
03121     unsigned NumConcat = MaskNumElts / SrcNumElts;
03122     bool Src1U = Src1.getOpcode() == ISD::UNDEF;
03123     bool Src2U = Src2.getOpcode() == ISD::UNDEF;
03124     SDValue UndefVal = DAG.getUNDEF(SrcVT);
03125 
03126     SmallVector<SDValue, 8> MOps1(NumConcat, UndefVal);
03127     SmallVector<SDValue, 8> MOps2(NumConcat, UndefVal);
03128     MOps1[0] = Src1;
03129     MOps2[0] = Src2;
03130 
03131     Src1 = Src1U ? DAG.getUNDEF(VT) : DAG.getNode(ISD::CONCAT_VECTORS,
03132                                                   getCurSDLoc(), VT, MOps1);
03133     Src2 = Src2U ? DAG.getUNDEF(VT) : DAG.getNode(ISD::CONCAT_VECTORS,
03134                                                   getCurSDLoc(), VT, MOps2);
03135 
03136     // Readjust mask for new input vector length.
03137     SmallVector<int, 8> MappedOps;
03138     for (unsigned i = 0; i != MaskNumElts; ++i) {
03139       int Idx = Mask[i];
03140       if (Idx >= (int)SrcNumElts)
03141         Idx -= SrcNumElts - MaskNumElts;
03142       MappedOps.push_back(Idx);
03143     }
03144 
03145     setValue(&I, DAG.getVectorShuffle(VT, getCurSDLoc(), Src1, Src2,
03146                                       &MappedOps[0]));
03147     return;
03148   }
03149 
03150   if (SrcNumElts > MaskNumElts) {
03151     // Analyze the access pattern of the vector to see if we can extract
03152     // two subvectors and do the shuffle. The analysis is done by calculating
03153     // the range of elements the mask access on both vectors.
03154     int MinRange[2] = { static_cast<int>(SrcNumElts),
03155                         static_cast<int>(SrcNumElts)};
03156     int MaxRange[2] = {-1, -1};
03157 
03158     for (unsigned i = 0; i != MaskNumElts; ++i) {
03159       int Idx = Mask[i];
03160       unsigned Input = 0;
03161       if (Idx < 0)
03162         continue;
03163 
03164       if (Idx >= (int)SrcNumElts) {
03165         Input = 1;
03166         Idx -= SrcNumElts;
03167       }
03168       if (Idx > MaxRange[Input])
03169         MaxRange[Input] = Idx;
03170       if (Idx < MinRange[Input])
03171         MinRange[Input] = Idx;
03172     }
03173 
03174     // Check if the access is smaller than the vector size and can we find
03175     // a reasonable extract index.
03176     int RangeUse[2] = { -1, -1 };  // 0 = Unused, 1 = Extract, -1 = Can not
03177                                    // Extract.
03178     int StartIdx[2];  // StartIdx to extract from
03179     for (unsigned Input = 0; Input < 2; ++Input) {
03180       if (MinRange[Input] >= (int)SrcNumElts && MaxRange[Input] < 0) {
03181         RangeUse[Input] = 0; // Unused
03182         StartIdx[Input] = 0;
03183         continue;
03184       }
03185 
03186       // Find a good start index that is a multiple of the mask length. Then
03187       // see if the rest of the elements are in range.
03188       StartIdx[Input] = (MinRange[Input]/MaskNumElts)*MaskNumElts;
03189       if (MaxRange[Input] - StartIdx[Input] < (int)MaskNumElts &&
03190           StartIdx[Input] + MaskNumElts <= SrcNumElts)
03191         RangeUse[Input] = 1; // Extract from a multiple of the mask length.
03192     }
03193 
03194     if (RangeUse[0] == 0 && RangeUse[1] == 0) {
03195       setValue(&I, DAG.getUNDEF(VT)); // Vectors are not used.
03196       return;
03197     }
03198     if (RangeUse[0] >= 0 && RangeUse[1] >= 0) {
03199       // Extract appropriate subvector and generate a vector shuffle
03200       for (unsigned Input = 0; Input < 2; ++Input) {
03201         SDValue &Src = Input == 0 ? Src1 : Src2;
03202         if (RangeUse[Input] == 0)
03203           Src = DAG.getUNDEF(VT);
03204         else
03205           Src = DAG.getNode(ISD::EXTRACT_SUBVECTOR, getCurSDLoc(), VT,
03206                             Src, DAG.getConstant(StartIdx[Input],
03207                                                  TLI->getVectorIdxTy()));
03208       }
03209 
03210       // Calculate new mask.
03211       SmallVector<int, 8> MappedOps;
03212       for (unsigned i = 0; i != MaskNumElts; ++i) {
03213         int Idx = Mask[i];
03214         if (Idx >= 0) {
03215           if (Idx < (int)SrcNumElts)
03216             Idx -= StartIdx[0];
03217           else
03218             Idx -= SrcNumElts + StartIdx[1] - MaskNumElts;
03219         }
03220         MappedOps.push_back(Idx);
03221       }
03222 
03223       setValue(&I, DAG.getVectorShuffle(VT, getCurSDLoc(), Src1, Src2,
03224                                         &MappedOps[0]));
03225       return;
03226     }
03227   }
03228 
03229   // We can't use either concat vectors or extract subvectors so fall back to
03230   // replacing the shuffle with extract and build vector.
03231   // to insert and build vector.
03232   EVT EltVT = VT.getVectorElementType();
03233   EVT IdxVT = TLI->getVectorIdxTy();
03234   SmallVector<SDValue,8> Ops;
03235   for (unsigned i = 0; i != MaskNumElts; ++i) {
03236     int Idx = Mask[i];
03237     SDValue Res;
03238 
03239     if (Idx < 0) {
03240       Res = DAG.getUNDEF(EltVT);
03241     } else {
03242       SDValue &Src = Idx < (int)SrcNumElts ? Src1 : Src2;
03243       if (Idx >= (int)SrcNumElts) Idx -= SrcNumElts;
03244 
03245       Res = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, getCurSDLoc(),
03246                         EltVT, Src, DAG.getConstant(Idx, IdxVT));
03247     }
03248 
03249     Ops.push_back(Res);
03250   }
03251 
03252   setValue(&I, DAG.getNode(ISD::BUILD_VECTOR, getCurSDLoc(), VT, Ops));
03253 }
03254 
03255 void SelectionDAGBuilder::visitInsertValue(const InsertValueInst &I) {
03256   const Value *Op0 = I.getOperand(0);
03257   const Value *Op1 = I.getOperand(1);
03258   Type *AggTy = I.getType();
03259   Type *ValTy = Op1->getType();
03260   bool IntoUndef = isa<UndefValue>(Op0);
03261   bool FromUndef = isa<UndefValue>(Op1);
03262 
03263   unsigned LinearIndex = ComputeLinearIndex(AggTy, I.getIndices());
03264 
03265   const TargetLowering *TLI = TM.getTargetLowering();
03266   SmallVector<EVT, 4> AggValueVTs;
03267   ComputeValueVTs(*TLI, AggTy, AggValueVTs);
03268   SmallVector<EVT, 4> ValValueVTs;
03269   ComputeValueVTs(*TLI, ValTy, ValValueVTs);
03270 
03271   unsigned NumAggValues = AggValueVTs.size();
03272   unsigned NumValValues = ValValueVTs.size();
03273   SmallVector<SDValue, 4> Values(NumAggValues);
03274 
03275   SDValue Agg = getValue(Op0);
03276   unsigned i = 0;
03277   // Copy the beginning value(s) from the original aggregate.
03278   for (; i != LinearIndex; ++i)
03279     Values[i] = IntoUndef ? DAG.getUNDEF(AggValueVTs[i]) :
03280                 SDValue(Agg.getNode(), Agg.getResNo() + i);
03281   // Copy values from the inserted value(s).
03282   if (NumValValues) {
03283     SDValue Val = getValue(Op1);
03284     for (; i != LinearIndex + NumValValues; ++i)
03285       Values[i] = FromUndef ? DAG.getUNDEF(AggValueVTs[i]) :
03286                   SDValue(Val.getNode(), Val.getResNo() + i - LinearIndex);
03287   }
03288   // Copy remaining value(s) from the original aggregate.
03289   for (; i != NumAggValues; ++i)
03290     Values[i] = IntoUndef ? DAG.getUNDEF(AggValueVTs[i]) :
03291                 SDValue(Agg.getNode(), Agg.getResNo() + i);
03292 
03293   setValue(&I, DAG.getNode(ISD::MERGE_VALUES, getCurSDLoc(),
03294                            DAG.getVTList(AggValueVTs), Values));
03295 }
03296 
03297 void SelectionDAGBuilder::visitExtractValue(const ExtractValueInst &I) {
03298   const Value *Op0 = I.getOperand(0);
03299   Type *AggTy = Op0->getType();
03300   Type *ValTy = I.getType();
03301   bool OutOfUndef = isa<UndefValue>(Op0);
03302 
03303   unsigned LinearIndex = ComputeLinearIndex(AggTy, I.getIndices());
03304 
03305   const TargetLowering *TLI = TM.getTargetLowering();
03306   SmallVector<EVT, 4> ValValueVTs;
03307   ComputeValueVTs(*TLI, ValTy, ValValueVTs);
03308 
03309   unsigned NumValValues = ValValueVTs.size();
03310 
03311   // Ignore a extractvalue that produces an empty object
03312   if (!NumValValues) {
03313     setValue(&I, DAG.getUNDEF(MVT(MVT::Other)));
03314     return;
03315   }
03316 
03317   SmallVector<SDValue, 4> Values(NumValValues);
03318 
03319   SDValue Agg = getValue(Op0);
03320   // Copy out the selected value(s).
03321   for (unsigned i = LinearIndex; i != LinearIndex + NumValValues; ++i)
03322     Values[i - LinearIndex] =
03323       OutOfUndef ?
03324         DAG.getUNDEF(Agg.getNode()->getValueType(Agg.getResNo() + i)) :
03325         SDValue(Agg.getNode(), Agg.getResNo() + i);
03326 
03327   setValue(&I, DAG.getNode(ISD::MERGE_VALUES, getCurSDLoc(),
03328                            DAG.getVTList(ValValueVTs), Values));
03329 }
03330 
03331 void SelectionDAGBuilder::visitGetElementPtr(const User &I) {
03332   Value *Op0 = I.getOperand(0);
03333   // Note that the pointer operand may be a vector of pointers. Take the scalar
03334   // element which holds a pointer.
03335   Type *Ty = Op0->getType()->getScalarType();
03336   unsigned AS = Ty->getPointerAddressSpace();
03337   SDValue N = getValue(Op0);
03338 
03339   for (GetElementPtrInst::const_op_iterator OI = I.op_begin()+1, E = I.op_end();
03340        OI != E; ++OI) {
03341     const Value *Idx = *OI;
03342     if (StructType *StTy = dyn_cast<StructType>(Ty)) {
03343       unsigned Field = cast<Constant>(Idx)->getUniqueInteger().getZExtValue();
03344       if (Field) {
03345         // N = N + Offset
03346         uint64_t Offset = DL->getStructLayout(StTy)->getElementOffset(Field);
03347         N = DAG.getNode(ISD::ADD, getCurSDLoc(), N.getValueType(), N,
03348                         DAG.getConstant(Offset, N.getValueType()));
03349       }
03350 
03351       Ty = StTy->getElementType(Field);
03352     } else {
03353       Ty = cast<SequentialType>(Ty)->getElementType();
03354 
03355       // If this is a constant subscript, handle it quickly.
03356       const TargetLowering *TLI = TM.getTargetLowering();
03357       if (const ConstantInt *CI = dyn_cast<ConstantInt>(Idx)) {
03358         if (CI->isZero()) continue;
03359         uint64_t Offs =
03360             DL->getTypeAllocSize(Ty)*cast<ConstantInt>(CI)->getSExtValue();
03361         SDValue OffsVal;
03362         EVT PTy = TLI->getPointerTy(AS);
03363         unsigned PtrBits = PTy.getSizeInBits();
03364         if (PtrBits < 64)
03365           OffsVal = DAG.getNode(ISD::TRUNCATE, getCurSDLoc(), PTy,
03366                                 DAG.getConstant(Offs, MVT::i64));
03367         else
03368           OffsVal = DAG.getConstant(Offs, PTy);
03369 
03370         N = DAG.getNode(ISD::ADD, getCurSDLoc(), N.getValueType(), N,
03371                         OffsVal);
03372         continue;
03373       }
03374 
03375       // N = N + Idx * ElementSize;
03376       APInt ElementSize = APInt(TLI->getPointerSizeInBits(AS),
03377                                 DL->getTypeAllocSize(Ty));
03378       SDValue IdxN = getValue(Idx);
03379 
03380       // If the index is smaller or larger than intptr_t, truncate or extend
03381       // it.
03382       IdxN = DAG.getSExtOrTrunc(IdxN, getCurSDLoc(), N.getValueType());
03383 
03384       // If this is a multiply by a power of two, turn it into a shl
03385       // immediately.  This is a very common case.
03386       if (ElementSize != 1) {
03387         if (ElementSize.isPowerOf2()) {
03388           unsigned Amt = ElementSize.logBase2();
03389           IdxN = DAG.getNode(ISD::SHL, getCurSDLoc(),
03390                              N.getValueType(), IdxN,
03391                              DAG.getConstant(Amt, IdxN.getValueType()));
03392         } else {
03393           SDValue Scale = DAG.getConstant(ElementSize, IdxN.getValueType());
03394           IdxN = DAG.getNode(ISD::MUL, getCurSDLoc(),
03395                              N.getValueType(), IdxN, Scale);
03396         }
03397       }
03398 
03399       N = DAG.getNode(ISD::ADD, getCurSDLoc(),
03400                       N.getValueType(), N, IdxN);
03401     }
03402   }
03403 
03404   setValue(&I, N);
03405 }
03406 
03407 void SelectionDAGBuilder::visitAlloca(const AllocaInst &I) {
03408   // If this is a fixed sized alloca in the entry block of the function,
03409   // allocate it statically on the stack.
03410   if (FuncInfo.StaticAllocaMap.count(&I))
03411     return;   // getValue will auto-populate this.
03412 
03413   Type *Ty = I.getAllocatedType();
03414   const TargetLowering *TLI = TM.getTargetLowering();
03415   uint64_t TySize = TLI->getDataLayout()->getTypeAllocSize(Ty);
03416   unsigned Align =
03417     std::max((unsigned)TLI->getDataLayout()->getPrefTypeAlignment(Ty),
03418              I.getAlignment());
03419 
03420   SDValue AllocSize = getValue(I.getArraySize());
03421 
03422   EVT IntPtr = TLI->getPointerTy();
03423   if (AllocSize.getValueType() != IntPtr)
03424     AllocSize = DAG.getZExtOrTrunc(AllocSize, getCurSDLoc(), IntPtr);
03425 
03426   AllocSize = DAG.getNode(ISD::MUL, getCurSDLoc(), IntPtr,
03427                           AllocSize,
03428                           DAG.getConstant(TySize, IntPtr));
03429 
03430   // Handle alignment.  If the requested alignment is less than or equal to
03431   // the stack alignment, ignore it.  If the size is greater than or equal to
03432   // the stack alignment, we note this in the DYNAMIC_STACKALLOC node.
03433   unsigned StackAlign = TM.getFrameLowering()->getStackAlignment();
03434   if (Align <= StackAlign)
03435     Align = 0;
03436 
03437   // Round the size of the allocation up to the stack alignment size
03438   // by add SA-1 to the size.
03439   AllocSize = DAG.getNode(ISD::ADD, getCurSDLoc(),
03440                           AllocSize.getValueType(), AllocSize,
03441                           DAG.getIntPtrConstant(StackAlign-1));
03442 
03443   // Mask out the low bits for alignment purposes.
03444   AllocSize = DAG.getNode(ISD::AND, getCurSDLoc(),
03445                           AllocSize.getValueType(), AllocSize,
03446                           DAG.getIntPtrConstant(~(uint64_t)(StackAlign-1)));
03447 
03448   SDValue Ops[] = { getRoot(), AllocSize, DAG.getIntPtrConstant(Align) };
03449   SDVTList VTs = DAG.getVTList(AllocSize.getValueType(), MVT::Other);
03450   SDValue DSA = DAG.getNode(ISD::DYNAMIC_STACKALLOC, getCurSDLoc(), VTs, Ops);
03451   setValue(&I, DSA);
03452   DAG.setRoot(DSA.getValue(1));
03453 
03454   assert(FuncInfo.MF->getFrameInfo()->hasVarSizedObjects());
03455 }
03456 
03457 void SelectionDAGBuilder::visitLoad(const LoadInst &I) {
03458   if (I.isAtomic())
03459     return visitAtomicLoad(I);
03460 
03461   const Value *SV = I.getOperand(0);
03462   SDValue Ptr = getValue(SV);
03463 
03464   Type *Ty = I.getType();
03465 
03466   bool isVolatile = I.isVolatile();
03467   bool isNonTemporal = I.getMetadata("nontemporal") != nullptr;
03468   bool isInvariant = I.getMetadata("invariant.load") != nullptr;
03469   unsigned Alignment = I.getAlignment();
03470   const MDNode *TBAAInfo = I.getMetadata(LLVMContext::MD_tbaa);
03471   const MDNode *Ranges = I.getMetadata(LLVMContext::MD_range);
03472 
03473   SmallVector<EVT, 4> ValueVTs;
03474   SmallVector<uint64_t, 4> Offsets;
03475   ComputeValueVTs(*TM.getTargetLowering(), Ty, ValueVTs, &Offsets);
03476   unsigned NumValues = ValueVTs.size();
03477   if (NumValues == 0)
03478     return;
03479 
03480   SDValue Root;
03481   bool ConstantMemory = false;
03482   if (isVolatile || NumValues > MaxParallelChains)
03483     // Serialize volatile loads with other side effects.
03484     Root = getRoot();
03485   else if (AA->pointsToConstantMemory(
03486              AliasAnalysis::Location(SV, AA->getTypeStoreSize(Ty), TBAAInfo))) {
03487     // Do not serialize (non-volatile) loads of constant memory with anything.
03488     Root = DAG.getEntryNode();
03489     ConstantMemory = true;
03490   } else {
03491     // Do not serialize non-volatile loads against each other.
03492     Root = DAG.getRoot();
03493   }
03494 
03495   const TargetLowering *TLI = TM.getTargetLowering();
03496   if (isVolatile)
03497     Root = TLI->prepareVolatileOrAtomicLoad(Root, getCurSDLoc(), DAG);
03498 
03499   SmallVector<SDValue, 4> Values(NumValues);
03500   SmallVector<SDValue, 4> Chains(std::min(unsigned(MaxParallelChains),
03501                                           NumValues));
03502   EVT PtrVT = Ptr.getValueType();
03503   unsigned ChainI = 0;
03504   for (unsigned i = 0; i != NumValues; ++i, ++ChainI) {
03505     // Serializing loads here may result in excessive register pressure, and
03506     // TokenFactor places arbitrary choke points on the scheduler. SD scheduling
03507     // could recover a bit by hoisting nodes upward in the chain by recognizing
03508     // they are side-effect free or do not alias. The optimizer should really
03509     // avoid this case by converting large object/array copies to llvm.memcpy
03510     // (MaxParallelChains should always remain as failsafe).
03511     if (ChainI == MaxParallelChains) {
03512       assert(PendingLoads.empty() && "PendingLoads must be serialized first");
03513       SDValue Chain = DAG.getNode(ISD::TokenFactor, getCurSDLoc(), MVT::Other,
03514                                   makeArrayRef(Chains.data(), ChainI));
03515       Root = Chain;
03516       ChainI = 0;
03517     }
03518     SDValue A = DAG.getNode(ISD::ADD, getCurSDLoc(),
03519                             PtrVT, Ptr,
03520                             DAG.getConstant(Offsets[i], PtrVT));
03521     SDValue L = DAG.getLoad(ValueVTs[i], getCurSDLoc(), Root,
03522                             A, MachinePointerInfo(SV, Offsets[i]), isVolatile,
03523                             isNonTemporal, isInvariant, Alignment, TBAAInfo,
03524                             Ranges);
03525 
03526     Values[i] = L;
03527     Chains[ChainI] = L.getValue(1);
03528   }
03529 
03530   if (!ConstantMemory) {
03531     SDValue Chain = DAG.getNode(ISD::TokenFactor, getCurSDLoc(), MVT::Other,
03532                                 makeArrayRef(Chains.data(), ChainI));
03533     if (isVolatile)
03534       DAG.setRoot(Chain);
03535     else
03536       PendingLoads.push_back(Chain);
03537   }
03538 
03539   setValue(&I, DAG.getNode(ISD::MERGE_VALUES, getCurSDLoc(),
03540                            DAG.getVTList(ValueVTs), Values));
03541 }
03542 
03543 void SelectionDAGBuilder::visitStore(const StoreInst &I) {
03544   if (I.isAtomic())
03545     return visitAtomicStore(I);
03546 
03547   const Value *SrcV = I.getOperand(0);
03548   const Value *PtrV = I.getOperand(1);
03549 
03550   SmallVector<EVT, 4> ValueVTs;
03551   SmallVector<uint64_t, 4> Offsets;
03552   ComputeValueVTs(*TM.getTargetLowering(), SrcV->getType(), ValueVTs, &Offsets);
03553   unsigned NumValues = ValueVTs.size();
03554   if (NumValues == 0)
03555     return;
03556 
03557   // Get the lowered operands. Note that we do this after
03558   // checking if NumResults is zero, because with zero results
03559   // the operands won't have values in the map.
03560   SDValue Src = getValue(SrcV);
03561   SDValue Ptr = getValue(PtrV);
03562 
03563   SDValue Root = getRoot();
03564   SmallVector<SDValue, 4> Chains(std::min(unsigned(MaxParallelChains),
03565                                           NumValues));
03566   EVT PtrVT = Ptr.getValueType();
03567   bool isVolatile = I.isVolatile();
03568   bool isNonTemporal = I.getMetadata("nontemporal") != nullptr;
03569   unsigned Alignment = I.getAlignment();
03570   const MDNode *TBAAInfo = I.getMetadata(LLVMContext::MD_tbaa);
03571 
03572   unsigned ChainI = 0;
03573   for (unsigned i = 0; i != NumValues; ++i, ++ChainI) {
03574     // See visitLoad comments.
03575     if (ChainI == MaxParallelChains) {
03576       SDValue Chain = DAG.getNode(ISD::TokenFactor, getCurSDLoc(), MVT::Other,
03577                                   makeArrayRef(Chains.data(), ChainI));
03578       Root = Chain;
03579       ChainI = 0;
03580     }
03581     SDValue Add = DAG.getNode(ISD::ADD, getCurSDLoc(), PtrVT, Ptr,
03582                               DAG.getConstant(Offsets[i], PtrVT));
03583     SDValue St = DAG.getStore(Root, getCurSDLoc(),
03584                               SDValue(Src.getNode(), Src.getResNo() + i),
03585                               Add, MachinePointerInfo(PtrV, Offsets[i]),
03586                               isVolatile, isNonTemporal, Alignment, TBAAInfo);
03587     Chains[ChainI] = St;
03588   }
03589 
03590   SDValue StoreNode = DAG.getNode(ISD::TokenFactor, getCurSDLoc(), MVT::Other,
03591                                   makeArrayRef(Chains.data(), ChainI));
03592   DAG.setRoot(StoreNode);
03593 }
03594 
03595 static SDValue InsertFenceForAtomic(SDValue Chain, AtomicOrdering Order,
03596                                     SynchronizationScope Scope,
03597                                     bool Before, SDLoc dl,
03598                                     SelectionDAG &DAG,
03599                                     const TargetLowering &TLI) {
03600   // Fence, if necessary
03601   if (Before) {
03602     if (Order == AcquireRelease || Order == SequentiallyConsistent)
03603       Order = Release;
03604     else if (Order == Acquire || Order == Monotonic || Order == Unordered)
03605       return Chain;
03606   } else {
03607     if (Order == AcquireRelease)
03608       Order = Acquire;
03609     else if (Order == Release || Order == Monotonic || Order == Unordered)
03610       return Chain;
03611   }
03612   SDValue Ops[3];
03613   Ops[0] = Chain;
03614   Ops[1] = DAG.getConstant(Order, TLI.getPointerTy());
03615   Ops[2] = DAG.getConstant(Scope, TLI.getPointerTy());
03616   return DAG.getNode(ISD::ATOMIC_FENCE, dl, MVT::Other, Ops);
03617 }
03618 
03619 void SelectionDAGBuilder::visitAtomicCmpXchg(const AtomicCmpXchgInst &I) {
03620   SDLoc dl = getCurSDLoc();
03621   AtomicOrdering SuccessOrder = I.getSuccessOrdering();
03622   AtomicOrdering FailureOrder = I.getFailureOrdering();
03623   SynchronizationScope Scope = I.getSynchScope();
03624 
03625   SDValue InChain = getRoot();
03626 
03627   const TargetLowering *TLI = TM.getTargetLowering();
03628   if (TLI->getInsertFencesForAtomic())
03629     InChain = InsertFenceForAtomic(InChain, SuccessOrder, Scope, true, dl,
03630                                    DAG, *TLI);
03631 
03632   MVT MemVT = getValue(I.getCompareOperand()).getSimpleValueType();
03633   SDVTList VTs = DAG.getVTList(MemVT, MVT::i1, MVT::Other);
03634   SDValue L = DAG.getAtomicCmpSwap(
03635       ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS, dl, MemVT, VTs, InChain,
03636       getValue(I.getPointerOperand()), getValue(I.getCompareOperand()),
03637       getValue(I.getNewValOperand()), MachinePointerInfo(I.getPointerOperand()),
03638       0 /* Alignment */,
03639       TLI->getInsertFencesForAtomic() ? Monotonic : SuccessOrder,
03640       TLI->getInsertFencesForAtomic() ? Monotonic : FailureOrder, Scope);
03641 
03642   SDValue OutChain = L.getValue(2);
03643 
03644   if (TLI->getInsertFencesForAtomic())
03645     OutChain = InsertFenceForAtomic(OutChain, SuccessOrder, Scope, false, dl,
03646                                     DAG, *TLI);
03647 
03648   setValue(&I, L);
03649   DAG.setRoot(OutChain);
03650 }
03651 
03652 void SelectionDAGBuilder::visitAtomicRMW(const AtomicRMWInst &I) {
03653   SDLoc dl = getCurSDLoc();
03654   ISD::NodeType NT;
03655   switch (I.getOperation()) {
03656   default: llvm_unreachable("Unknown atomicrmw operation");
03657   case AtomicRMWInst::Xchg: NT = ISD::ATOMIC_SWAP; break;
03658   case AtomicRMWInst::Add:  NT = ISD::ATOMIC_LOAD_ADD; break;
03659   case AtomicRMWInst::Sub:  NT = ISD::ATOMIC_LOAD_SUB; break;
03660   case AtomicRMWInst::And:  NT = ISD::ATOMIC_LOAD_AND; break;
03661   case AtomicRMWInst::Nand: NT = ISD::ATOMIC_LOAD_NAND; break;
03662   case AtomicRMWInst::Or:   NT = ISD::ATOMIC_LOAD_OR; break;
03663   case AtomicRMWInst::Xor:  NT = ISD::ATOMIC_LOAD_XOR; break;
03664   case AtomicRMWInst::Max:  NT = ISD::ATOMIC_LOAD_MAX; break;
03665   case AtomicRMWInst::Min:  NT = ISD::ATOMIC_LOAD_MIN; break;
03666   case AtomicRMWInst::UMax: NT = ISD::ATOMIC_LOAD_UMAX; break;
03667   case AtomicRMWInst::UMin: NT = ISD::ATOMIC_LOAD_UMIN; break;
03668   }
03669   AtomicOrdering Order = I.getOrdering();
03670   SynchronizationScope Scope = I.getSynchScope();
03671 
03672   SDValue InChain = getRoot();
03673 
03674   const TargetLowering *TLI = TM.getTargetLowering();
03675   if (TLI->getInsertFencesForAtomic())
03676     InChain = InsertFenceForAtomic(InChain, Order, Scope, true, dl,
03677                                    DAG, *TLI);
03678 
03679   SDValue L =
03680     DAG.getAtomic(NT, dl,
03681                   getValue(I.getValOperand()).getSimpleValueType(),
03682                   InChain,
03683                   getValue(I.getPointerOperand()),
03684                   getValue(I.getValOperand()),
03685                   I.getPointerOperand(), 0 /* Alignment */,
03686                   TLI->getInsertFencesForAtomic() ? Monotonic : Order,
03687                   Scope);
03688 
03689   SDValue OutChain = L.getValue(1);
03690 
03691   if (TLI->getInsertFencesForAtomic())
03692     OutChain = InsertFenceForAtomic(OutChain, Order, Scope, false, dl,
03693                                     DAG, *TLI);
03694 
03695   setValue(&I, L);
03696   DAG.setRoot(OutChain);
03697 }
03698 
03699 void SelectionDAGBuilder::visitFence(const FenceInst &I) {
03700   SDLoc dl = getCurSDLoc();
03701   const TargetLowering *TLI = TM.getTargetLowering();
03702   SDValue Ops[3];
03703   Ops[0] = getRoot();
03704   Ops[1] = DAG.getConstant(I.getOrdering(), TLI->getPointerTy());
03705   Ops[2] = DAG.getConstant(I.getSynchScope(), TLI->getPointerTy());
03706   DAG.setRoot(DAG.getNode(ISD::ATOMIC_FENCE, dl, MVT::Other, Ops));
03707 }
03708 
03709 void SelectionDAGBuilder::visitAtomicLoad(const LoadInst &I) {
03710   SDLoc dl = getCurSDLoc();
03711   AtomicOrdering Order = I.getOrdering();
03712   SynchronizationScope Scope = I.getSynchScope();
03713 
03714   SDValue InChain = getRoot();
03715 
03716   const TargetLowering *TLI = TM.getTargetLowering();
03717   EVT VT = TLI->getValueType(I.getType());
03718 
03719   if (I.getAlignment() < VT.getSizeInBits() / 8)
03720     report_fatal_error("Cannot generate unaligned atomic load");
03721 
03722   MachineMemOperand *MMO =
03723       DAG.getMachineFunction().
03724       getMachineMemOperand(MachinePointerInfo(I.getPointerOperand()),
03725                            MachineMemOperand::MOVolatile |
03726                            MachineMemOperand::MOLoad,
03727                            VT.getStoreSize(),
03728                            I.getAlignment() ? I.getAlignment() :
03729                                               DAG.getEVTAlignment(VT));
03730 
03731   InChain = TLI->prepareVolatileOrAtomicLoad(InChain, dl, DAG);
03732   SDValue L =
03733       DAG.getAtomic(ISD::ATOMIC_LOAD, dl, VT, VT, InChain,
03734                     getValue(I.getPointerOperand()), MMO,
03735                     TLI->getInsertFencesForAtomic() ? Monotonic : Order,
03736                     Scope);
03737 
03738   SDValue OutChain = L.getValue(1);
03739 
03740   if (TLI->getInsertFencesForAtomic())
03741     OutChain = InsertFenceForAtomic(OutChain, Order, Scope, false, dl,
03742                                     DAG, *TLI);
03743 
03744   setValue(&I, L);
03745   DAG.setRoot(OutChain);
03746 }
03747 
03748 void SelectionDAGBuilder::visitAtomicStore(const StoreInst &I) {
03749   SDLoc dl = getCurSDLoc();
03750 
03751   AtomicOrdering Order = I.getOrdering();
03752   SynchronizationScope Scope = I.getSynchScope();
03753 
03754   SDValue InChain = getRoot();
03755 
03756   const TargetLowering *TLI = TM.getTargetLowering();
03757   EVT VT = TLI->getValueType(I.getValueOperand()->getType());
03758 
03759   if (I.getAlignment() < VT.getSizeInBits() / 8)
03760     report_fatal_error("Cannot generate unaligned atomic store");
03761 
03762   if (TLI->getInsertFencesForAtomic())
03763     InChain = InsertFenceForAtomic(InChain, Order, Scope, true, dl,
03764                                    DAG, *TLI);
03765 
03766   SDValue OutChain =
03767     DAG.getAtomic(ISD::ATOMIC_STORE, dl, VT,
03768                   InChain,
03769                   getValue(I.getPointerOperand()),
03770                   getValue(I.getValueOperand()),
03771                   I.getPointerOperand(), I.getAlignment(),
03772                   TLI->getInsertFencesForAtomic() ? Monotonic : Order,
03773                   Scope);
03774 
03775   if (TLI->getInsertFencesForAtomic())
03776     OutChain = InsertFenceForAtomic(OutChain, Order, Scope, false, dl,
03777                                     DAG, *TLI);
03778 
03779   DAG.setRoot(OutChain);
03780 }
03781 
03782 /// visitTargetIntrinsic - Lower a call of a target intrinsic to an INTRINSIC
03783 /// node.
03784 void SelectionDAGBuilder::visitTargetIntrinsic(const CallInst &I,
03785                                                unsigned Intrinsic) {
03786   bool HasChain = !I.doesNotAccessMemory();
03787   bool OnlyLoad = HasChain && I.onlyReadsMemory();
03788 
03789   // Build the operand list.
03790   SmallVector<SDValue, 8> Ops;
03791   if (HasChain) {  // If this intrinsic has side-effects, chainify it.
03792     if (OnlyLoad) {
03793       // We don't need to serialize loads against other loads.
03794       Ops.push_back(DAG.getRoot());
03795     } else {
03796       Ops.push_back(getRoot());
03797     }
03798   }
03799 
03800   // Info is set by getTgtMemInstrinsic
03801   TargetLowering::IntrinsicInfo Info;
03802   const TargetLowering *TLI = TM.getTargetLowering();
03803   bool IsTgtIntrinsic = TLI->getTgtMemIntrinsic(Info, I, Intrinsic);
03804 
03805   // Add the intrinsic ID as an integer operand if it's not a target intrinsic.
03806   if (!IsTgtIntrinsic || Info.opc == ISD::INTRINSIC_VOID ||
03807       Info.opc == ISD::INTRINSIC_W_CHAIN)
03808     Ops.push_back(DAG.getTargetConstant(Intrinsic, TLI->getPointerTy()));
03809 
03810   // Add all operands of the call to the operand list.
03811   for (unsigned i = 0, e = I.getNumArgOperands(); i != e; ++i) {
03812     SDValue Op = getValue(I.getArgOperand(i));
03813     Ops.push_back(Op);
03814   }
03815 
03816   SmallVector<EVT, 4> ValueVTs;
03817   ComputeValueVTs(*TLI, I.getType(), ValueVTs);
03818 
03819   if (HasChain)
03820     ValueVTs.push_back(MVT::Other);
03821 
03822   SDVTList VTs = DAG.getVTList(ValueVTs);
03823 
03824   // Create the node.
03825   SDValue Result;
03826   if (IsTgtIntrinsic) {
03827     // This is target intrinsic that touches memory
03828     Result = DAG.getMemIntrinsicNode(Info.opc, getCurSDLoc(),
03829                                      VTs, Ops, Info.memVT,
03830                                    MachinePointerInfo(Info.ptrVal, Info.offset),
03831                                      Info.align, Info.vol,
03832                                      Info.readMem, Info.writeMem);
03833   } else if (!HasChain) {
03834     Result = DAG.getNode(ISD::INTRINSIC_WO_CHAIN, getCurSDLoc(), VTs, Ops);
03835   } else if (!I.getType()->isVoidTy()) {
03836     Result = DAG.getNode(ISD::INTRINSIC_W_CHAIN, getCurSDLoc(), VTs, Ops);
03837   } else {
03838     Result = DAG.getNode(ISD::INTRINSIC_VOID, getCurSDLoc(), VTs, Ops);
03839   }
03840 
03841   if (HasChain) {
03842     SDValue Chain = Result.getValue(Result.getNode()->getNumValues()-1);
03843     if (OnlyLoad)
03844       PendingLoads.push_back(Chain);
03845     else
03846       DAG.setRoot(Chain);
03847   }
03848 
03849   if (!I.getType()->isVoidTy()) {
03850     if (VectorType *PTy = dyn_cast<VectorType>(I.getType())) {
03851       EVT VT = TLI->getValueType(PTy);
03852       Result = DAG.getNode(ISD::BITCAST, getCurSDLoc(), VT, Result);
03853     }
03854 
03855     setValue(&I, Result);
03856   }
03857 }
03858 
03859 /// GetSignificand - Get the significand and build it into a floating-point
03860 /// number with exponent of 1:
03861 ///
03862 ///   Op = (Op & 0x007fffff) | 0x3f800000;
03863 ///
03864 /// where Op is the hexadecimal representation of floating point value.
03865 static SDValue
03866 GetSignificand(SelectionDAG &DAG, SDValue Op, SDLoc dl) {
03867   SDValue t1 = DAG.getNode(ISD::AND, dl, MVT::i32, Op,
03868                            DAG.getConstant(0x007fffff, MVT::i32));
03869   SDValue t2 = DAG.getNode(ISD::OR, dl, MVT::i32, t1,
03870                            DAG.getConstant(0x3f800000, MVT::i32));
03871   return DAG.getNode(ISD::BITCAST, dl, MVT::f32, t2);
03872 }
03873 
03874 /// GetExponent - Get the exponent:
03875 ///
03876 ///   (float)(int)(((Op & 0x7f800000) >> 23) - 127);
03877 ///
03878 /// where Op is the hexadecimal representation of floating point value.
03879 static SDValue
03880 GetExponent(SelectionDAG &DAG, SDValue Op, const TargetLowering &TLI,
03881             SDLoc dl) {
03882   SDValue t0 = DAG.getNode(ISD::AND, dl, MVT::i32, Op,
03883                            DAG.getConstant(0x7f800000, MVT::i32));
03884   SDValue t1 = DAG.getNode(ISD::SRL, dl, MVT::i32, t0,
03885                            DAG.getConstant(23, TLI.getPointerTy()));
03886   SDValue t2 = DAG.getNode(ISD::SUB, dl, MVT::i32, t1,
03887                            DAG.getConstant(127, MVT::i32));
03888   return DAG.getNode(ISD::SINT_TO_FP, dl, MVT::f32, t2);
03889 }
03890 
03891 /// getF32Constant - Get 32-bit floating point constant.
03892 static SDValue
03893 getF32Constant(SelectionDAG &DAG, unsigned Flt) {
03894   return DAG.getConstantFP(APFloat(APFloat::IEEEsingle, APInt(32, Flt)),
03895                            MVT::f32);
03896 }
03897 
03898 /// expandExp - Lower an exp intrinsic. Handles the special sequences for
03899 /// limited-precision mode.
03900 static SDValue expandExp(SDLoc dl, SDValue Op, SelectionDAG &DAG,
03901                          const TargetLowering &TLI) {
03902   if (Op.getValueType() == MVT::f32 &&
03903       LimitFloatPrecision > 0 && LimitFloatPrecision <= 18) {
03904 
03905     // Put the exponent in the right bit position for later addition to the
03906     // final result:
03907     //
03908     //   #define LOG2OFe 1.4426950f
03909     //   IntegerPartOfX = ((int32_t)(X * LOG2OFe));
03910     SDValue t0 = DAG.getNode(ISD::FMUL, dl, MVT::f32, Op,
03911                              getF32Constant(DAG, 0x3fb8aa3b));
03912     SDValue IntegerPartOfX = DAG.getNode(ISD::FP_TO_SINT, dl, MVT::i32, t0);
03913 
03914     //   FractionalPartOfX = (X * LOG2OFe) - (float)IntegerPartOfX;
03915     SDValue t1 = DAG.getNode(ISD::SINT_TO_FP, dl, MVT::f32, IntegerPartOfX);
03916     SDValue X = DAG.getNode(ISD::FSUB, dl, MVT::f32, t0, t1);
03917 
03918     //   IntegerPartOfX <<= 23;
03919     IntegerPartOfX = DAG.getNode(ISD::SHL, dl, MVT::i32, IntegerPartOfX,
03920                                  DAG.getConstant(23, TLI.getPointerTy()));
03921 
03922     SDValue TwoToFracPartOfX;
03923     if (LimitFloatPrecision <= 6) {
03924       // For floating-point precision of 6:
03925       //
03926       //   TwoToFractionalPartOfX =
03927       //     0.997535578f +
03928       //       (0.735607626f + 0.252464424f * x) * x;
03929       //
03930       // error 0.0144103317, which is 6 bits
03931       SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X,
03932                                getF32Constant(DAG, 0x3e814304));
03933       SDValue t3 = DAG.getNode(ISD::FADD, dl, MVT::f32, t2,
03934                                getF32Constant(DAG, 0x3f3c50c8));
03935       SDValue t4 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t3, X);
03936       TwoToFracPartOfX = DAG.getNode(ISD::FADD, dl, MVT::f32, t4,
03937                                      getF32Constant(DAG, 0x3f7f5e7e));
03938     } else if (LimitFloatPrecision <= 12) {
03939       // For floating-point precision of 12:
03940       //
03941       //   TwoToFractionalPartOfX =
03942       //     0.999892986f +
03943       //       (0.696457318f +
03944       //         (0.224338339f + 0.792043434e-1f * x) * x) * x;
03945       //
03946       // 0.000107046256 error, which is 13 to 14 bits
03947       SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X,
03948                                getF32Constant(DAG, 0x3da235e3));
03949       SDValue t3 = DAG.getNode(ISD::FADD, dl, MVT::f32, t2,
03950                                getF32Constant(DAG, 0x3e65b8f3));
03951       SDValue t4 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t3, X);
03952       SDValue t5 = DAG.getNode(ISD::FADD, dl, MVT::f32, t4,
03953                                getF32Constant(DAG, 0x3f324b07));
03954       SDValue t6 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t5, X);
03955       TwoToFracPartOfX = DAG.getNode(ISD::FADD, dl, MVT::f32, t6,
03956                                      getF32Constant(DAG, 0x3f7ff8fd));
03957     } else { // LimitFloatPrecision <= 18
03958       // For floating-point precision of 18:
03959       //
03960       //   TwoToFractionalPartOfX =
03961       //     0.999999982f +
03962       //       (0.693148872f +
03963       //         (0.240227044f +
03964       //           (0.554906021e-1f +
03965       //             (0.961591928e-2f +
03966       //               (0.136028312e-2f + 0.157059148e-3f *x)*x)*x)*x)*x)*x;
03967       //
03968       // error 2.47208000*10^(-7), which is better than 18 bits
03969       SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X,
03970                                getF32Constant(DAG, 0x3924b03e));
03971       SDValue t3 = DAG.getNode(ISD::FADD, dl, MVT::f32, t2,
03972                                getF32Constant(DAG, 0x3ab24b87));
03973       SDValue t4 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t3, X);
03974       SDValue t5 = DAG.getNode(ISD::FADD, dl, MVT::f32, t4,
03975                                getF32Constant(DAG, 0x3c1d8c17));
03976       SDValue t6 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t5, X);
03977       SDValue t7 = DAG.getNode(ISD::FADD, dl, MVT::f32, t6,
03978                                getF32Constant(DAG, 0x3d634a1d));
03979       SDValue t8 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t7, X);
03980       SDValue t9 = DAG.getNode(ISD::FADD, dl, MVT::f32, t8,
03981                                getF32Constant(DAG, 0x3e75fe14));
03982       SDValue t10 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t9, X);
03983       SDValue t11 = DAG.getNode(ISD::FADD, dl, MVT::f32, t10,
03984                                 getF32Constant(DAG, 0x3f317234));
03985       SDValue t12 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t11, X);
03986       TwoToFracPartOfX = DAG.getNode(ISD::FADD, dl, MVT::f32, t12,
03987                                      getF32Constant(DAG, 0x3f800000));
03988     }
03989 
03990     // Add the exponent into the result in integer domain.
03991     SDValue t13 = DAG.getNode(ISD::BITCAST, dl, MVT::i32, TwoToFracPartOfX);
03992     return DAG.getNode(ISD::BITCAST, dl, MVT::f32,
03993                        DAG.getNode(ISD::ADD, dl, MVT::i32,
03994                                    t13, IntegerPartOfX));
03995   }
03996 
03997   // No special expansion.
03998   return DAG.getNode(ISD::FEXP, dl, Op.getValueType(), Op);
03999 }
04000 
04001 /// expandLog - Lower a log intrinsic. Handles the special sequences for
04002 /// limited-precision mode.
04003 static SDValue expandLog(SDLoc dl, SDValue Op, SelectionDAG &DAG,
04004                          const TargetLowering &TLI) {
04005   if (Op.getValueType() == MVT::f32 &&
04006       LimitFloatPrecision > 0 && LimitFloatPrecision <= 18) {
04007     SDValue Op1 = DAG.getNode(ISD::BITCAST, dl, MVT::i32, Op);
04008 
04009     // Scale the exponent by log(2) [0.69314718f].
04010     SDValue Exp = GetExponent(DAG, Op1, TLI, dl);
04011     SDValue LogOfExponent = DAG.getNode(ISD::FMUL, dl, MVT::f32, Exp,
04012                                         getF32Constant(DAG, 0x3f317218));
04013 
04014     // Get the significand and build it into a floating-point number with
04015     // exponent of 1.
04016     SDValue X = GetSignificand(DAG, Op1, dl);
04017 
04018     SDValue LogOfMantissa;
04019     if (LimitFloatPrecision <= 6) {
04020       // For floating-point precision of 6:
04021       //
04022       //   LogofMantissa =
04023       //     -1.1609546f +
04024       //       (1.4034025f - 0.23903021f * x) * x;
04025       //
04026       // error 0.0034276066, which is better than 8 bits
04027       SDValue t0 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X,
04028                                getF32Constant(DAG, 0xbe74c456));
04029       SDValue t1 = DAG.getNode(ISD::FADD, dl, MVT::f32, t0,
04030                                getF32Constant(DAG, 0x3fb3a2b1));
04031       SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t1, X);
04032       LogOfMantissa = DAG.getNode(ISD::FSUB, dl, MVT::f32, t2,
04033                                   getF32Constant(DAG, 0x3f949a29));
04034     } else if (LimitFloatPrecision <= 12) {
04035       // For floating-point precision of 12:
04036       //
04037       //   LogOfMantissa =
04038       //     -1.7417939f +
04039       //       (2.8212026f +
04040       //         (-1.4699568f +
04041       //           (0.44717955f - 0.56570851e-1f * x) * x) * x) * x;
04042       //
04043       // error 0.000061011436, which is 14 bits
04044       SDValue t0 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X,
04045                                getF32Constant(DAG, 0xbd67b6d6));
04046       SDValue t1 = DAG.getNode(ISD::FADD, dl, MVT::f32, t0,
04047                                getF32Constant(DAG, 0x3ee4f4b8));
04048       SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t1, X);
04049       SDValue t3 = DAG.getNode(ISD::FSUB, dl, MVT::f32, t2,
04050                                getF32Constant(DAG, 0x3fbc278b));
04051       SDValue t4 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t3, X);
04052       SDValue t5 = DAG.getNode(ISD::FADD, dl, MVT::f32, t4,
04053                                getF32Constant(DAG, 0x40348e95));
04054       SDValue t6 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t5, X);
04055       LogOfMantissa = DAG.getNode(ISD::FSUB, dl, MVT::f32, t6,
04056                                   getF32Constant(DAG, 0x3fdef31a));
04057     } else { // LimitFloatPrecision <= 18
04058       // For floating-point precision of 18:
04059       //
04060       //   LogOfMantissa =
04061       //     -2.1072184f +
04062       //       (4.2372794f +
04063       //         (-3.7029485f +
04064       //           (2.2781945f +
04065       //             (-0.87823314f +
04066       //               (0.19073739f - 0.17809712e-1f * x) * x) * x) * x) * x)*x;
04067       //
04068       // error 0.0000023660568, which is better than 18 bits
04069       SDValue t0 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X,
04070                                getF32Constant(DAG, 0xbc91e5ac));
04071       SDValue t1 = DAG.getNode(ISD::FADD, dl, MVT::f32, t0,
04072                                getF32Constant(DAG, 0x3e4350aa));
04073       SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t1, X);
04074       SDValue t3 = DAG.getNode(ISD::FSUB, dl, MVT::f32, t2,
04075                                getF32Constant(DAG, 0x3f60d3e3));
04076       SDValue t4 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t3, X);
04077       SDValue t5 = DAG.getNode(ISD::FADD, dl, MVT::f32, t4,
04078                                getF32Constant(DAG, 0x4011cdf0));
04079       SDValue t6 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t5, X);
04080       SDValue t7 = DAG.getNode(ISD::FSUB, dl, MVT::f32, t6,
04081                                getF32Constant(DAG, 0x406cfd1c));
04082       SDValue t8 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t7, X);
04083       SDValue t9 = DAG.getNode(ISD::FADD, dl, MVT::f32, t8,
04084                                getF32Constant(DAG, 0x408797cb));
04085       SDValue t10 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t9, X);
04086       LogOfMantissa = DAG.getNode(ISD::FSUB, dl, MVT::f32, t10,
04087                                   getF32Constant(DAG, 0x4006dcab));
04088     }
04089 
04090     return DAG.getNode(ISD::FADD, dl, MVT::f32, LogOfExponent, LogOfMantissa);
04091   }
04092 
04093   // No special expansion.
04094   return DAG.getNode(ISD::FLOG, dl, Op.getValueType(), Op);
04095 }
04096 
04097 /// expandLog2 - Lower a log2 intrinsic. Handles the special sequences for
04098 /// limited-precision mode.
04099 static SDValue expandLog2(SDLoc dl, SDValue Op, SelectionDAG &DAG,
04100                           const TargetLowering &TLI) {
04101   if (Op.getValueType() == MVT::f32 &&
04102       LimitFloatPrecision > 0 && LimitFloatPrecision <= 18) {
04103     SDValue Op1 = DAG.getNode(ISD::BITCAST, dl, MVT::i32, Op);
04104 
04105     // Get the exponent.
04106     SDValue LogOfExponent = GetExponent(DAG, Op1, TLI, dl);
04107 
04108     // Get the significand and build it into a floating-point number with
04109     // exponent of 1.
04110     SDValue X = GetSignificand(DAG, Op1, dl);
04111 
04112     // Different possible minimax approximations of significand in
04113     // floating-point for various degrees of accuracy over [1,2].
04114     SDValue Log2ofMantissa;
04115     if (LimitFloatPrecision <= 6) {
04116       // For floating-point precision of 6:
04117       //
04118       //   Log2ofMantissa = -1.6749035f + (2.0246817f - .34484768f * x) * x;
04119       //
04120       // error 0.0049451742, which is more than 7 bits
04121       SDValue t0 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X,
04122                                getF32Constant(DAG, 0xbeb08fe0));
04123       SDValue t1 = DAG.getNode(ISD::FADD, dl, MVT::f32, t0,
04124                                getF32Constant(DAG, 0x40019463));
04125       SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t1, X);
04126       Log2ofMantissa = DAG.getNode(ISD::FSUB, dl, MVT::f32, t2,
04127                                    getF32Constant(DAG, 0x3fd6633d));
04128     } else if (LimitFloatPrecision <= 12) {
04129       // For floating-point precision of 12:
04130       //
04131       //   Log2ofMantissa =
04132       //     -2.51285454f +
04133       //       (4.07009056f +
04134       //         (-2.12067489f +
04135       //           (.645142248f - 0.816157886e-1f * x) * x) * x) * x;
04136       //
04137       // error 0.0000876136000, which is better than 13 bits
04138       SDValue t0 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X,
04139                                getF32Constant(DAG, 0xbda7262e));
04140       SDValue t1 = DAG.getNode(ISD::FADD, dl, MVT::f32, t0,
04141                                getF32Constant(DAG, 0x3f25280b));
04142       SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t1, X);
04143       SDValue t3 = DAG.getNode(ISD::FSUB, dl, MVT::f32, t2,
04144                                getF32Constant(DAG, 0x4007b923));
04145       SDValue t4 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t3, X);
04146       SDValue t5 = DAG.getNode(ISD::FADD, dl, MVT::f32, t4,
04147                                getF32Constant(DAG, 0x40823e2f));
04148       SDValue t6 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t5, X);
04149       Log2ofMantissa = DAG.getNode(ISD::FSUB, dl, MVT::f32, t6,
04150                                    getF32Constant(DAG, 0x4020d29c));
04151     } else { // LimitFloatPrecision <= 18
04152       // For floating-point precision of 18:
04153       //
04154       //   Log2ofMantissa =
04155       //     -3.0400495f +
04156       //       (6.1129976f +
04157       //         (-5.3420409f +
04158       //           (3.2865683f +
04159       //             (-1.2669343f +
04160       //               (0.27515199f -
04161       //                 0.25691327e-1f * x) * x) * x) * x) * x) * x;
04162       //
04163       // error 0.0000018516, which is better than 18 bits
04164       SDValue t0 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X,
04165                                getF32Constant(DAG, 0xbcd2769e));
04166       SDValue t1 = DAG.getNode(ISD::FADD, dl, MVT::f32, t0,
04167                                getF32Constant(DAG, 0x3e8ce0b9));
04168       SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t1, X);
04169       SDValue t3 = DAG.getNode(ISD::FSUB, dl, MVT::f32, t2,
04170                                getF32Constant(DAG, 0x3fa22ae7));
04171       SDValue t4 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t3, X);
04172       SDValue t5 = DAG.getNode(ISD::FADD, dl, MVT::f32, t4,
04173                                getF32Constant(DAG, 0x40525723));
04174       SDValue t6 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t5, X);
04175       SDValue t7 = DAG.getNode(ISD::FSUB, dl, MVT::f32, t6,
04176                                getF32Constant(DAG, 0x40aaf200));
04177       SDValue t8 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t7, X);
04178       SDValue t9 = DAG.getNode(ISD::FADD, dl, MVT::f32, t8,
04179                                getF32Constant(DAG, 0x40c39dad));
04180       SDValue t10 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t9, X);
04181       Log2ofMantissa = DAG.getNode(ISD::FSUB, dl, MVT::f32, t10,
04182                                    getF32Constant(DAG, 0x4042902c));
04183     }
04184 
04185     return DAG.getNode(ISD::FADD, dl, MVT::f32, LogOfExponent, Log2ofMantissa);
04186   }
04187 
04188   // No special expansion.
04189   return DAG.getNode(ISD::FLOG2, dl, Op.getValueType(), Op);
04190 }
04191 
04192 /// expandLog10 - Lower a log10 intrinsic. Handles the special sequences for
04193 /// limited-precision mode.
04194 static SDValue expandLog10(SDLoc dl, SDValue Op, SelectionDAG &DAG,
04195                            const TargetLowering &TLI) {
04196   if (Op.getValueType() == MVT::f32 &&
04197       LimitFloatPrecision > 0 && LimitFloatPrecision <= 18) {
04198     SDValue Op1 = DAG.getNode(ISD::BITCAST, dl, MVT::i32, Op);
04199 
04200     // Scale the exponent by log10(2) [0.30102999f].
04201     SDValue Exp = GetExponent(DAG, Op1, TLI, dl);
04202     SDValue LogOfExponent = DAG.getNode(ISD::FMUL, dl, MVT::f32, Exp,
04203                                         getF32Constant(DAG, 0x3e9a209a));
04204 
04205     // Get the significand and build it into a floating-point number with
04206     // exponent of 1.
04207     SDValue X = GetSignificand(DAG, Op1, dl);
04208 
04209     SDValue Log10ofMantissa;
04210     if (LimitFloatPrecision <= 6) {
04211       // For floating-point precision of 6:
04212       //
04213       //   Log10ofMantissa =
04214       //     -0.50419619f +
04215       //       (0.60948995f - 0.10380950f * x) * x;
04216       //
04217       // error 0.0014886165, which is 6 bits
04218       SDValue t0 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X,
04219                                getF32Constant(DAG, 0xbdd49a13));
04220       SDValue t1 = DAG.getNode(ISD::FADD, dl, MVT::f32, t0,
04221                                getF32Constant(DAG, 0x3f1c0789));
04222       SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t1, X);
04223       Log10ofMantissa = DAG.getNode(ISD::FSUB, dl, MVT::f32, t2,
04224                                     getF32Constant(DAG, 0x3f011300));
04225     } else if (LimitFloatPrecision <= 12) {
04226       // For floating-point precision of 12:
04227       //
04228       //   Log10ofMantissa =
04229       //     -0.64831180f +
04230       //       (0.91751397f +
04231       //         (-0.31664806f + 0.47637168e-1f * x) * x) * x;
04232       //
04233       // error 0.00019228036, which is better than 12 bits
04234       SDValue t0 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X,
04235                                getF32Constant(DAG, 0x3d431f31));
04236       SDValue t1 = DAG.getNode(ISD::FSUB, dl, MVT::f32, t0,
04237                                getF32Constant(DAG, 0x3ea21fb2));
04238       SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t1, X);
04239       SDValue t3 = DAG.getNode(ISD::FADD, dl, MVT::f32, t2,
04240                                getF32Constant(DAG, 0x3f6ae232));
04241       SDValue t4 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t3, X);
04242       Log10ofMantissa = DAG.getNode(ISD::FSUB, dl, MVT::f32, t4,
04243                                     getF32Constant(DAG, 0x3f25f7c3));
04244     } else { // LimitFloatPrecision <= 18
04245       // For floating-point precision of 18:
04246       //
04247       //   Log10ofMantissa =
04248       //     -0.84299375f +
04249       //       (1.5327582f +
04250       //         (-1.0688956f +
04251       //           (0.49102474f +
04252       //             (-0.12539807f + 0.13508273e-1f * x) * x) * x) * x) * x;
04253       //
04254       // error 0.0000037995730, which is better than 18 bits
04255       SDValue t0 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X,
04256                                getF32Constant(DAG, 0x3c5d51ce));
04257       SDValue t1 = DAG.getNode(ISD::FSUB, dl, MVT::f32, t0,
04258                                getF32Constant(DAG, 0x3e00685a));
04259       SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t1, X);
04260       SDValue t3 = DAG.getNode(ISD::FADD, dl, MVT::f32, t2,
04261                                getF32Constant(DAG, 0x3efb6798));
04262       SDValue t4 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t3, X);
04263       SDValue t5 = DAG.getNode(ISD::FSUB, dl, MVT::f32, t4,
04264                                getF32Constant(DAG, 0x3f88d192));
04265       SDValue t6 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t5, X);
04266       SDValue t7 = DAG.getNode(ISD::FADD, dl, MVT::f32, t6,
04267                                getF32Constant(DAG, 0x3fc4316c));
04268       SDValue t8 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t7, X);
04269       Log10ofMantissa = DAG.getNode(ISD::FSUB, dl, MVT::f32, t8,
04270                                     getF32Constant(DAG, 0x3f57ce70));
04271     }
04272 
04273     return DAG.getNode(ISD::FADD, dl, MVT::f32, LogOfExponent, Log10ofMantissa);
04274   }
04275 
04276   // No special expansion.
04277   return DAG.getNode(ISD::FLOG10, dl, Op.getValueType(), Op);
04278 }
04279 
04280 /// expandExp2 - Lower an exp2 intrinsic. Handles the special sequences for
04281 /// limited-precision mode.
04282 static SDValue expandExp2(SDLoc dl, SDValue Op, SelectionDAG &DAG,
04283                           const TargetLowering &TLI) {
04284   if (Op.getValueType() == MVT::f32 &&
04285       LimitFloatPrecision > 0 && LimitFloatPrecision <= 18) {
04286     SDValue IntegerPartOfX = DAG.getNode(ISD::FP_TO_SINT, dl, MVT::i32, Op);
04287 
04288     //   FractionalPartOfX = x - (float)IntegerPartOfX;
04289     SDValue t1 = DAG.getNode(ISD::SINT_TO_FP, dl, MVT::f32, IntegerPartOfX);
04290     SDValue X = DAG.getNode(ISD::FSUB, dl, MVT::f32, Op, t1);
04291 
04292     //   IntegerPartOfX <<= 23;
04293     IntegerPartOfX = DAG.getNode(ISD::SHL, dl, MVT::i32, IntegerPartOfX,
04294                                  DAG.getConstant(23, TLI.getPointerTy()));
04295 
04296     SDValue TwoToFractionalPartOfX;
04297     if (LimitFloatPrecision <= 6) {
04298       // For floating-point precision of 6:
04299       //
04300       //   TwoToFractionalPartOfX =
04301       //     0.997535578f +
04302       //       (0.735607626f + 0.252464424f * x) * x;
04303       //
04304       // error 0.0144103317, which is 6 bits
04305       SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X,
04306                                getF32Constant(DAG, 0x3e814304));
04307       SDValue t3 = DAG.getNode(ISD::FADD, dl, MVT::f32, t2,
04308                                getF32Constant(DAG, 0x3f3c50c8));
04309       SDValue t4 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t3, X);
04310       TwoToFractionalPartOfX = DAG.getNode(ISD::FADD, dl, MVT::f32, t4,
04311                                            getF32Constant(DAG, 0x3f7f5e7e));
04312     } else if (LimitFloatPrecision <= 12) {
04313       // For floating-point precision of 12:
04314       //
04315       //   TwoToFractionalPartOfX =
04316       //     0.999892986f +
04317       //       (0.696457318f +
04318       //         (0.224338339f + 0.792043434e-1f * x) * x) * x;
04319       //
04320       // error 0.000107046256, which is 13 to 14 bits
04321       SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X,
04322                                getF32Constant(DAG, 0x3da235e3));
04323       SDValue t3 = DAG.getNode(ISD::FADD, dl, MVT::f32, t2,
04324                                getF32Constant(DAG, 0x3e65b8f3));
04325       SDValue t4 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t3, X);
04326       SDValue t5 = DAG.getNode(ISD::FADD, dl, MVT::f32, t4,
04327                                getF32Constant(DAG, 0x3f324b07));
04328       SDValue t6 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t5, X);
04329       TwoToFractionalPartOfX = DAG.getNode(ISD::FADD, dl, MVT::f32, t6,
04330                                            getF32Constant(DAG, 0x3f7ff8fd));
04331     } else { // LimitFloatPrecision <= 18
04332       // For floating-point precision of 18:
04333       //
04334       //   TwoToFractionalPartOfX =
04335       //     0.999999982f +
04336       //       (0.693148872f +
04337       //         (0.240227044f +
04338       //           (0.554906021e-1f +
04339       //             (0.961591928e-2f +
04340       //               (0.136028312e-2f + 0.157059148e-3f *x)*x)*x)*x)*x)*x;
04341       // error 2.47208000*10^(-7), which is better than 18 bits
04342       SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X,
04343                                getF32Constant(DAG, 0x3924b03e));
04344       SDValue t3 = DAG.getNode(ISD::FADD, dl, MVT::f32, t2,
04345                                getF32Constant(DAG, 0x3ab24b87));
04346       SDValue t4 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t3, X);
04347       SDValue t5 = DAG.getNode(ISD::FADD, dl, MVT::f32, t4,
04348                                getF32Constant(DAG, 0x3c1d8c17));
04349       SDValue t6 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t5, X);
04350       SDValue t7 = DAG.getNode(ISD::FADD, dl, MVT::f32, t6,
04351                                getF32Constant(DAG, 0x3d634a1d));
04352       SDValue t8 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t7, X);
04353       SDValue t9 = DAG.getNode(ISD::FADD, dl, MVT::f32, t8,
04354                                getF32Constant(DAG, 0x3e75fe14));
04355       SDValue t10 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t9, X);
04356       SDValue t11 = DAG.getNode(ISD::FADD, dl, MVT::f32, t10,
04357                                 getF32Constant(DAG, 0x3f317234));
04358       SDValue t12 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t11, X);
04359       TwoToFractionalPartOfX = DAG.getNode(ISD::FADD, dl, MVT::f32, t12,
04360                                            getF32Constant(DAG, 0x3f800000));
04361     }
04362 
04363     // Add the exponent into the result in integer domain.
04364     SDValue t13 = DAG.getNode(ISD::BITCAST, dl, MVT::i32,
04365                               TwoToFractionalPartOfX);
04366     return DAG.getNode(ISD::BITCAST, dl, MVT::f32,
04367                        DAG.getNode(ISD::ADD, dl, MVT::i32,
04368                                    t13, IntegerPartOfX));
04369   }
04370 
04371   // No special expansion.
04372   return DAG.getNode(ISD::FEXP2, dl, Op.getValueType(), Op);
04373 }
04374 
04375 /// visitPow - Lower a pow intrinsic. Handles the special sequences for
04376 /// limited-precision mode with x == 10.0f.
04377 static SDValue expandPow(SDLoc dl, SDValue LHS, SDValue RHS,
04378                          SelectionDAG &DAG, const TargetLowering &TLI) {
04379   bool IsExp10 = false;
04380   if (LHS.getValueType() == MVT::f32 && RHS.getValueType() == MVT::f32 &&
04381       LimitFloatPrecision > 0 && LimitFloatPrecision <= 18) {
04382     if (ConstantFPSDNode *LHSC = dyn_cast<ConstantFPSDNode>(LHS)) {
04383       APFloat Ten(10.0f);
04384       IsExp10 = LHSC->isExactlyValue(Ten);
04385     }
04386   }
04387 
04388   if (IsExp10) {
04389     // Put the exponent in the right bit position for later addition to the
04390     // final result:
04391     //
04392     //   #define LOG2OF10 3.3219281f
04393     //   IntegerPartOfX = (int32_t)(x * LOG2OF10);
04394     SDValue t0 = DAG.getNode(ISD::FMUL, dl, MVT::f32, RHS,
04395                              getF32Constant(DAG, 0x40549a78));
04396     SDValue IntegerPartOfX = DAG.getNode(ISD::FP_TO_SINT, dl, MVT::i32, t0);
04397 
04398     //   FractionalPartOfX = x - (float)IntegerPartOfX;
04399     SDValue t1 = DAG.getNode(ISD::SINT_TO_FP, dl, MVT::f32, IntegerPartOfX);
04400     SDValue X = DAG.getNode(ISD::FSUB, dl, MVT::f32, t0, t1);
04401 
04402     //   IntegerPartOfX <<= 23;
04403     IntegerPartOfX = DAG.getNode(ISD::SHL, dl, MVT::i32, IntegerPartOfX,
04404                                  DAG.getConstant(23, TLI.getPointerTy()));
04405 
04406     SDValue TwoToFractionalPartOfX;
04407     if (LimitFloatPrecision <= 6) {
04408       // For floating-point precision of 6:
04409       //
04410       //   twoToFractionalPartOfX =
04411       //     0.997535578f +
04412       //       (0.735607626f + 0.252464424f * x) * x;
04413       //
04414       // error 0.0144103317, which is 6 bits
04415       SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X,
04416                                getF32Constant(DAG, 0x3e814304));
04417       SDValue t3 = DAG.getNode(ISD::FADD, dl, MVT::f32, t2,
04418                                getF32Constant(DAG, 0x3f3c50c8));
04419       SDValue t4 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t3, X);
04420       TwoToFractionalPartOfX = DAG.getNode(ISD::FADD, dl, MVT::f32, t4,
04421                                            getF32Constant(DAG, 0x3f7f5e7e));
04422     } else if (LimitFloatPrecision <= 12) {
04423       // For floating-point precision of 12:
04424       //
04425       //   TwoToFractionalPartOfX =
04426       //     0.999892986f +
04427       //       (0.696457318f +
04428       //         (0.224338339f + 0.792043434e-1f * x) * x) * x;
04429       //
04430       // error 0.000107046256, which is 13 to 14 bits
04431       SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X,
04432                                getF32Constant(DAG, 0x3da235e3));
04433       SDValue t3 = DAG.getNode(ISD::FADD, dl, MVT::f32, t2,
04434                                getF32Constant(DAG, 0x3e65b8f3));
04435       SDValue t4 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t3, X);
04436       SDValue t5 = DAG.getNode(ISD::FADD, dl, MVT::f32, t4,
04437                                getF32Constant(DAG, 0x3f324b07));
04438       SDValue t6 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t5, X);
04439       TwoToFractionalPartOfX = DAG.getNode(ISD::FADD, dl, MVT::f32, t6,
04440                                            getF32Constant(DAG, 0x3f7ff8fd));
04441     } else { // LimitFloatPrecision <= 18
04442       // For floating-point precision of 18:
04443       //
04444       //   TwoToFractionalPartOfX =
04445       //     0.999999982f +
04446       //       (0.693148872f +
04447       //         (0.240227044f +
04448       //           (0.554906021e-1f +
04449       //             (0.961591928e-2f +
04450       //               (0.136028312e-2f + 0.157059148e-3f *x)*x)*x)*x)*x)*x;
04451       // error 2.47208000*10^(-7), which is better than 18 bits
04452       SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X,
04453                                getF32Constant(DAG, 0x3924b03e));
04454       SDValue t3 = DAG.getNode(ISD::FADD, dl, MVT::f32, t2,
04455                                getF32Constant(DAG, 0x3ab24b87));
04456       SDValue t4 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t3, X);
04457       SDValue t5 = DAG.getNode(ISD::FADD, dl, MVT::f32, t4,
04458                                getF32Constant(DAG, 0x3c1d8c17));
04459       SDValue t6 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t5, X);
04460       SDValue t7 = DAG.getNode(ISD::FADD, dl, MVT::f32, t6,
04461                                getF32Constant(DAG, 0x3d634a1d));
04462       SDValue t8 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t7, X);
04463       SDValue t9 = DAG.getNode(ISD::FADD, dl, MVT::f32, t8,
04464                                getF32Constant(DAG, 0x3e75fe14));
04465       SDValue t10 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t9, X);
04466       SDValue t11 = DAG.getNode(ISD::FADD, dl, MVT::f32, t10,
04467                                 getF32Constant(DAG, 0x3f317234));
04468       SDValue t12 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t11, X);
04469       TwoToFractionalPartOfX = DAG.getNode(ISD::FADD, dl, MVT::f32, t12,
04470                                            getF32Constant(DAG, 0x3f800000));
04471     }
04472 
04473     SDValue t13 = DAG.getNode(ISD::BITCAST, dl,MVT::i32,TwoToFractionalPartOfX);
04474     return DAG.getNode(ISD::BITCAST, dl, MVT::f32,
04475                        DAG.getNode(ISD::ADD, dl, MVT::i32,
04476                                    t13, IntegerPartOfX));
04477   }
04478 
04479   // No special expansion.
04480   return DAG.getNode(ISD::FPOW, dl, LHS.getValueType(), LHS, RHS);
04481 }
04482 
04483 
04484 /// ExpandPowI - Expand a llvm.powi intrinsic.
04485 static SDValue ExpandPowI(SDLoc DL, SDValue LHS, SDValue RHS,
04486                           SelectionDAG &DAG) {
04487   // If RHS is a constant, we can expand this out to a multiplication tree,
04488   // otherwise we end up lowering to a call to __powidf2 (for example).  When
04489   // optimizing for size, we only want to do this if the expansion would produce
04490   // a small number of multiplies, otherwise we do the full expansion.
04491   if (ConstantSDNode *RHSC = dyn_cast<ConstantSDNode>(RHS)) {
04492     // Get the exponent as a positive value.
04493     unsigned Val = RHSC->getSExtValue();
04494     if ((int)Val < 0) Val = -Val;
04495 
04496     // powi(x, 0) -> 1.0
04497     if (Val == 0)
04498       return DAG.getConstantFP(1.0, LHS.getValueType());
04499 
04500     const Function *F = DAG.getMachineFunction().getFunction();
04501     if (!F->getAttributes().hasAttribute(AttributeSet::FunctionIndex,
04502                                          Attribute::OptimizeForSize) ||
04503         // If optimizing for size, don't insert too many multiplies.  This
04504         // inserts up to 5 multiplies.
04505         CountPopulation_32(Val)+Log2_32(Val) < 7) {
04506       // We use the simple binary decomposition method to generate the multiply
04507       // sequence.  There are more optimal ways to do this (for example,
04508       // powi(x,15) generates one more multiply than it should), but this has
04509       // the benefit of being both really simple and much better than a libcall.
04510       SDValue Res;  // Logically starts equal to 1.0
04511       SDValue CurSquare = LHS;
04512       while (Val) {
04513         if (Val & 1) {
04514           if (Res.getNode())
04515             Res = DAG.getNode(ISD::FMUL, DL,Res.getValueType(), Res, CurSquare);
04516           else
04517             Res = CurSquare;  // 1.0*CurSquare.
04518         }
04519 
04520         CurSquare = DAG.getNode(ISD::FMUL, DL, CurSquare.getValueType(),
04521                                 CurSquare, CurSquare);
04522         Val >>= 1;
04523       }
04524 
04525       // If the original was negative, invert the result, producing 1/(x*x*x).
04526       if (RHSC->getSExtValue() < 0)
04527         Res = DAG.getNode(ISD::FDIV, DL, LHS.getValueType(),
04528                           DAG.getConstantFP(1.0, LHS.getValueType()), Res);
04529       return Res;
04530     }
04531   }
04532 
04533   // Otherwise, expand to a libcall.
04534   return DAG.getNode(ISD::FPOWI, DL, LHS.getValueType(), LHS, RHS);
04535 }
04536 
04537 // getTruncatedArgReg - Find underlying register used for an truncated
04538 // argument.
04539 static unsigned getTruncatedArgReg(const SDValue &N) {
04540   if (N.getOpcode() != ISD::TRUNCATE)
04541     return 0;
04542 
04543   const SDValue &Ext = N.getOperand(0);
04544   if (Ext.getOpcode() == ISD::AssertZext ||
04545       Ext.getOpcode() == ISD::AssertSext) {
04546     const SDValue &CFR = Ext.getOperand(0);
04547     if (CFR.getOpcode() == ISD::CopyFromReg)
04548       return cast<RegisterSDNode>(CFR.getOperand(1))->getReg();
04549     if (CFR.getOpcode() == ISD::TRUNCATE)
04550       return getTruncatedArgReg(CFR);
04551   }
04552   return 0;
04553 }
04554 
04555 /// EmitFuncArgumentDbgValue - If the DbgValueInst is a dbg_value of a function
04556 /// argument, create the corresponding DBG_VALUE machine instruction for it now.
04557 /// At the end of instruction selection, they will be inserted to the entry BB.
04558 bool
04559 SelectionDAGBuilder::EmitFuncArgumentDbgValue(const Value *V, MDNode *Variable,
04560                                               int64_t Offset, bool IsIndirect,
04561                                               const SDValue &N) {
04562   const Argument *Arg = dyn_cast<Argument>(V);
04563   if (!Arg)
04564     return false;
04565 
04566   MachineFunction &MF = DAG.getMachineFunction();
04567   const TargetInstrInfo *TII = DAG.getTarget().getInstrInfo();
04568 
04569   // Ignore inlined function arguments here.
04570   DIVariable DV(Variable);
04571   if (DV.isInlinedFnArgument(MF.getFunction()))
04572     return false;
04573 
04574   Optional<MachineOperand> Op;
04575   // Some arguments' frame index is recorded during argument lowering.
04576   if (int FI = FuncInfo.getArgumentFrameIndex(Arg))
04577     Op = MachineOperand::CreateFI(FI);
04578 
04579   if (!Op && N.getNode()) {
04580     unsigned Reg;
04581     if (N.getOpcode() == ISD::CopyFromReg)
04582       Reg = cast<RegisterSDNode>(N.getOperand(1))->getReg();
04583     else
04584       Reg = getTruncatedArgReg(N);
04585     if (Reg && TargetRegisterInfo::isVirtualRegister(Reg)) {
04586       MachineRegisterInfo &RegInfo = MF.getRegInfo();
04587       unsigned PR = RegInfo.getLiveInPhysReg(Reg);
04588       if (PR)
04589         Reg = PR;
04590     }
04591     if (Reg)
04592       Op = MachineOperand::CreateReg(Reg, false);
04593   }
04594 
04595   if (!Op) {
04596     // Check if ValueMap has reg number.
04597     DenseMap<const Value *, unsigned>::iterator VMI = FuncInfo.ValueMap.find(V);
04598     if (VMI != FuncInfo.ValueMap.end())
04599       Op = MachineOperand::CreateReg(VMI->second, false);
04600   }
04601 
04602   if (!Op && N.getNode())
04603     // Check if frame index is available.
04604     if (LoadSDNode *LNode = dyn_cast<LoadSDNode>(N.getNode()))
04605       if (FrameIndexSDNode *FINode =
04606           dyn_cast<FrameIndexSDNode>(LNode->getBasePtr().getNode()))
04607         Op = MachineOperand::CreateFI(FINode->getIndex());
04608 
04609   if (!Op)
04610     return false;
04611 
04612   if (Op->isReg())
04613     FuncInfo.ArgDbgValues.push_back(BuildMI(MF, getCurDebugLoc(),
04614                                             TII->get(TargetOpcode::DBG_VALUE),
04615                                             IsIndirect,
04616                                             Op->getReg(), Offset, Variable));
04617   else
04618     FuncInfo.ArgDbgValues.push_back(
04619       BuildMI(MF, getCurDebugLoc(), TII->get(TargetOpcode::DBG_VALUE))
04620           .addOperand(*Op).addImm(Offset).addMetadata(Variable));
04621 
04622   return true;
04623 }
04624 
04625 // VisualStudio defines setjmp as _setjmp
04626 #if defined(_MSC_VER) && defined(setjmp) && \
04627                          !defined(setjmp_undefined_for_msvc)
04628 #  pragma push_macro("setjmp")
04629 #  undef setjmp
04630 #  define setjmp_undefined_for_msvc
04631 #endif
04632 
04633 /// visitIntrinsicCall - Lower the call to the specified intrinsic function.  If
04634 /// we want to emit this as a call to a named external function, return the name
04635 /// otherwise lower it and return null.
04636 const char *
04637 SelectionDAGBuilder::visitIntrinsicCall(const CallInst &I, unsigned Intrinsic) {
04638   const TargetLowering *TLI = TM.getTargetLowering();
04639   SDLoc sdl = getCurSDLoc();
04640   DebugLoc dl = getCurDebugLoc();
04641   SDValue Res;
04642 
04643   switch (Intrinsic) {
04644   default:
04645     // By default, turn this into a target intrinsic node.
04646     visitTargetIntrinsic(I, Intrinsic);
04647     return nullptr;
04648   case Intrinsic::vastart:  visitVAStart(I); return nullptr;
04649   case Intrinsic::vaend:    visitVAEnd(I); return nullptr;
04650   case Intrinsic::vacopy:   visitVACopy(I); return nullptr;
04651   case Intrinsic::returnaddress:
04652     setValue(&I, DAG.getNode(ISD::RETURNADDR, sdl, TLI->getPointerTy(),
04653                              getValue(I.getArgOperand(0))));
04654     return nullptr;
04655   case Intrinsic::frameaddress:
04656     setValue(&I, DAG.getNode(ISD::FRAMEADDR, sdl, TLI->getPointerTy(),
04657                              getValue(I.getArgOperand(0))));
04658     return nullptr;
04659   case Intrinsic::read_register: {
04660     Value *Reg = I.getArgOperand(0);
04661     SDValue RegName = DAG.getMDNode(cast<MDNode>(Reg));
04662     EVT VT = TM.getTargetLowering()->getValueType(I.getType());
04663     setValue(&I, DAG.getNode(ISD::READ_REGISTER, sdl, VT, RegName));
04664     return nullptr;
04665   }
04666   case Intrinsic::write_register: {
04667     Value *Reg = I.getArgOperand(0);
04668     Value *RegValue = I.getArgOperand(1);
04669     SDValue Chain = getValue(RegValue).getOperand(0);
04670     SDValue RegName = DAG.getMDNode(cast<MDNode>(Reg));
04671     DAG.setRoot(DAG.getNode(ISD::WRITE_REGISTER, sdl, MVT::Other, Chain,
04672                             RegName, getValue(RegValue)));
04673     return nullptr;
04674   }
04675   case Intrinsic::setjmp:
04676     return &"_setjmp"[!TLI->usesUnderscoreSetJmp()];
04677   case Intrinsic::longjmp:
04678     return &"_longjmp"[!TLI->usesUnderscoreLongJmp()];
04679   case Intrinsic::memcpy: {
04680     // Assert for address < 256 since we support only user defined address
04681     // spaces.
04682     assert(cast<PointerType>(I.getArgOperand(0)->getType())->getAddressSpace()
04683            < 256 &&
04684            cast<PointerType>(I.getArgOperand(1)->getType())->getAddressSpace()
04685            < 256 &&
04686            "Unknown address space");
04687     SDValue Op1 = getValue(I.getArgOperand(0));
04688     SDValue Op2 = getValue(I.getArgOperand(1));
04689     SDValue Op3 = getValue(I.getArgOperand(2));
04690     unsigned Align = cast<ConstantInt>(I.getArgOperand(3))->getZExtValue();
04691     if (!Align)
04692       Align = 1; // @llvm.memcpy defines 0 and 1 to both mean no alignment.
04693     bool isVol = cast<ConstantInt>(I.getArgOperand(4))->getZExtValue();
04694     DAG.setRoot(DAG.getMemcpy(getRoot(), sdl, Op1, Op2, Op3, Align, isVol, false,
04695                               MachinePointerInfo(I.getArgOperand(0)),
04696                               MachinePointerInfo(I.getArgOperand(1))));
04697     return nullptr;
04698   }
04699   case Intrinsic::memset: {
04700     // Assert for address < 256 since we support only user defined address
04701     // spaces.
04702     assert(cast<PointerType>(I.getArgOperand(0)->getType())->getAddressSpace()
04703            < 256 &&
04704            "Unknown address space");
04705     SDValue Op1 = getValue(I.getArgOperand(0));
04706     SDValue Op2 = getValue(I.getArgOperand(1));
04707     SDValue Op3 = getValue(I.getArgOperand(2));
04708     unsigned Align = cast<ConstantInt>(I.getArgOperand(3))->getZExtValue();
04709     if (!Align)
04710       Align = 1; // @llvm.memset defines 0 and 1 to both mean no alignment.
04711     bool isVol = cast<ConstantInt>(I.getArgOperand(4))->getZExtValue();
04712     DAG.setRoot(DAG.getMemset(getRoot(), sdl, Op1, Op2, Op3, Align, isVol,
04713                               MachinePointerInfo(I.getArgOperand(0))));
04714     return nullptr;
04715   }
04716   case Intrinsic::memmove: {
04717     // Assert for address < 256 since we support only user defined address
04718     // spaces.
04719     assert(cast<PointerType>(I.getArgOperand(0)->getType())->getAddressSpace()
04720            < 256 &&
04721            cast<PointerType>(I.getArgOperand(1)->getType())->getAddressSpace()
04722            < 256 &&
04723            "Unknown address space");
04724     SDValue Op1 = getValue(I.getArgOperand(0));
04725     SDValue Op2 = getValue(I.getArgOperand(1));
04726     SDValue Op3 = getValue(I.getArgOperand(2));
04727     unsigned Align = cast<ConstantInt>(I.getArgOperand(3))->getZExtValue();
04728     if (!Align)
04729       Align = 1; // @llvm.memmove defines 0 and 1 to both mean no alignment.
04730     bool isVol = cast<ConstantInt>(I.getArgOperand(4))->getZExtValue();
04731     DAG.setRoot(DAG.getMemmove(getRoot(), sdl, Op1, Op2, Op3, Align, isVol,
04732                                MachinePointerInfo(I.getArgOperand(0)),
04733                                MachinePointerInfo(I.getArgOperand(1))));
04734     return nullptr;
04735   }
04736   case Intrinsic::dbg_declare: {
04737     const DbgDeclareInst &DI = cast<DbgDeclareInst>(I);
04738     MDNode *Variable = DI.getVariable();
04739     const Value *Address = DI.getAddress();
04740     DIVariable DIVar(Variable);
04741     assert((!DIVar || DIVar.isVariable()) &&
04742       "Variable in DbgDeclareInst should be either null or a DIVariable.");
04743     if (!Address || !DIVar) {
04744       DEBUG(dbgs() << "Dropping debug info for " << DI << "\n");
04745       return nullptr;
04746     }
04747 
04748     // Check if address has undef value.
04749     if (isa<UndefValue>(Address) ||
04750         (Address->use_empty() && !isa<Argument>(Address))) {
04751       DEBUG(dbgs() << "Dropping debug info for " << DI << "\n");
04752       return nullptr;
04753     }
04754 
04755     SDValue &N = NodeMap[Address];
04756     if (!N.getNode() && isa<Argument>(Address))
04757       // Check unused arguments map.
04758       N = UnusedArgNodeMap[Address];
04759     SDDbgValue *SDV;
04760     if (N.getNode()) {
04761       if (const BitCastInst *BCI = dyn_cast<BitCastInst>(Address))
04762         Address = BCI->getOperand(0);
04763       // Parameters are handled specially.
04764       bool isParameter =
04765         (DIVariable(Variable).getTag() == dwarf::DW_TAG_arg_variable ||
04766          isa<Argument>(Address));
04767 
04768       const AllocaInst *AI = dyn_cast<AllocaInst>(Address);
04769 
04770       if (isParameter && !AI) {
04771         FrameIndexSDNode *FINode = dyn_cast<FrameIndexSDNode>(N.getNode());
04772         if (FINode)
04773           // Byval parameter.  We have a frame index at this point.
04774           SDV = DAG.getFrameIndexDbgValue(Variable, FINode->getIndex(),
04775             0, dl, SDNodeOrder);
04776         else {
04777           // Address is an argument, so try to emit its dbg value using
04778           // virtual register info from the FuncInfo.ValueMap.
04779           EmitFuncArgumentDbgValue(Address, Variable, 0, false, N);
04780           return nullptr;
04781         }
04782       } else if (AI)
04783         SDV = DAG.getDbgValue(Variable, N.getNode(), N.getResNo(),
04784                               true, 0, dl, SDNodeOrder);
04785       else {
04786         // Can't do anything with other non-AI cases yet.
04787         DEBUG(dbgs() << "Dropping debug info for " << DI << "\n");
04788         DEBUG(dbgs() << "non-AllocaInst issue for Address: \n\t");
04789         DEBUG(Address->dump());
04790         return nullptr;
04791       }
04792       DAG.AddDbgValue(SDV, N.getNode(), isParameter);
04793     } else {
04794       // If Address is an argument then try to emit its dbg value using
04795       // virtual register info from the FuncInfo.ValueMap.
04796       if (!EmitFuncArgumentDbgValue(Address, Variable, 0, false, N)) {
04797         // If variable is pinned by a alloca in dominating bb then
04798         // use StaticAllocaMap.
04799         if (const AllocaInst *AI = dyn_cast<AllocaInst>(Address)) {
04800           if (AI->getParent() != DI.getParent()) {
04801             DenseMap<const AllocaInst*, int>::iterator SI =
04802               FuncInfo.StaticAllocaMap.find(AI);
04803             if (SI != FuncInfo.StaticAllocaMap.end()) {
04804               SDV = DAG.getFrameIndexDbgValue(Variable, SI->second,
04805                                               0, dl, SDNodeOrder);
04806               DAG.AddDbgValue(SDV, nullptr, false);
04807               return nullptr;
04808             }
04809           }
04810         }
04811         DEBUG(dbgs() << "Dropping debug info for " << DI << "\n");
04812       }
04813     }
04814     return nullptr;
04815   }
04816   case Intrinsic::dbg_value: {
04817     const DbgValueInst &DI = cast<DbgValueInst>(I);
04818     DIVariable DIVar(DI.getVariable());
04819     assert((!DIVar || DIVar.isVariable()) &&
04820       "Variable in DbgValueInst should be either null or a DIVariable.");
04821     if (!DIVar)
04822       return nullptr;
04823 
04824     MDNode *Variable = DI.getVariable();
04825     uint64_t Offset = DI.getOffset();
04826     const Value *V = DI.getValue();
04827     if (!V)
04828       return nullptr;
04829 
04830     SDDbgValue *SDV;
04831     if (isa<ConstantInt>(V) || isa<ConstantFP>(V) || isa<UndefValue>(V)) {
04832       SDV = DAG.getConstantDbgValue(Variable, V, Offset, dl, SDNodeOrder);
04833       DAG.AddDbgValue(SDV, nullptr, false);
04834     } else {
04835       // Do not use getValue() in here; we don't want to generate code at
04836       // this point if it hasn't been done yet.
04837       SDValue N = NodeMap[V];
04838       if (!N.getNode() && isa<Argument>(V))
04839         // Check unused arguments map.
04840         N = UnusedArgNodeMap[V];
04841       if (N.getNode()) {
04842         // A dbg.value for an alloca is always indirect.
04843         bool IsIndirect = isa<AllocaInst>(V) || Offset != 0;
04844         if (!EmitFuncArgumentDbgValue(V, Variable, Offset, IsIndirect, N)) {
04845           SDV = DAG.getDbgValue(Variable, N.getNode(),
04846                                 N.getResNo(), IsIndirect,
04847         Offset, dl, SDNodeOrder);
04848           DAG.AddDbgValue(SDV, N.getNode(), false);
04849         }
04850       } else if (!V->use_empty() ) {
04851         // Do not call getValue(V) yet, as we don't want to generate code.
04852         // Remember it for later.
04853         DanglingDebugInfo DDI(&DI, dl, SDNodeOrder);
04854         DanglingDebugInfoMap[V] = DDI;
04855       } else {
04856         // We may expand this to cover more cases.  One case where we have no
04857         // data available is an unreferenced parameter.
04858         DEBUG(dbgs() << "Dropping debug info for " << DI << "\n");
04859       }
04860     }
04861 
04862     // Build a debug info table entry.
04863     if (const BitCastInst *BCI = dyn_cast<BitCastInst>(V))
04864       V = BCI->getOperand(0);
04865     const AllocaInst *AI = dyn_cast<AllocaInst>(V);
04866     // Don't handle byval struct arguments or VLAs, for example.
04867     if (!AI) {
04868       DEBUG(dbgs() << "Dropping debug location info for:\n  " << DI << "\n");
04869       DEBUG(dbgs() << "  Last seen at:\n    " << *V << "\n");
04870       return nullptr;
04871     }
04872     DenseMap<const AllocaInst*, int>::iterator SI =
04873       FuncInfo.StaticAllocaMap.find(AI);
04874     if (SI == FuncInfo.StaticAllocaMap.end())
04875       return nullptr; // VLAs.
04876     return nullptr;
04877   }
04878 
04879   case Intrinsic::eh_typeid_for: {
04880     // Find the type id for the given typeinfo.
04881     GlobalVariable *GV = ExtractTypeInfo(I.getArgOperand(0));
04882     unsigned TypeID = DAG.getMachineFunction().getMMI().getTypeIDFor(GV);
04883     Res = DAG.getConstant(TypeID, MVT::i32);
04884     setValue(&I, Res);
04885     return nullptr;
04886   }
04887 
04888   case Intrinsic::eh_return_i32:
04889   case Intrinsic::eh_return_i64:
04890     DAG.getMachineFunction().getMMI().setCallsEHReturn(true);
04891     DAG.setRoot(DAG.getNode(ISD::EH_RETURN, sdl,
04892                             MVT::Other,
04893                             getControlRoot(),
04894                             getValue(I.getArgOperand(0)),
04895                             getValue(I.getArgOperand(1))));
04896     return nullptr;
04897   case Intrinsic::eh_unwind_init:
04898     DAG.getMachineFunction().getMMI().setCallsUnwindInit(true);
04899     return nullptr;
04900   case Intrinsic::eh_dwarf_cfa: {
04901     SDValue CfaArg = DAG.getSExtOrTrunc(getValue(I.getArgOperand(0)), sdl,
04902                                         TLI->getPointerTy());
04903     SDValue Offset = DAG.getNode(ISD::ADD, sdl,
04904                                  CfaArg.getValueType(),
04905                                  DAG.getNode(ISD::FRAME_TO_ARGS_OFFSET, sdl,
04906                                              CfaArg.getValueType()),
04907                                  CfaArg);
04908     SDValue FA = DAG.getNode(ISD::FRAMEADDR, sdl,
04909                              TLI->getPointerTy(),
04910                              DAG.getConstant(0, TLI->getPointerTy()));
04911     setValue(&I, DAG.getNode(ISD::ADD, sdl, FA.getValueType(),
04912                              FA, Offset));
04913     return nullptr;
04914   }
04915   case Intrinsic::eh_sjlj_callsite: {
04916     MachineModuleInfo &MMI = DAG.getMachineFunction().getMMI();
04917     ConstantInt *CI = dyn_cast<ConstantInt>(I.getArgOperand(0));
04918     assert(CI && "Non-constant call site value in eh.sjlj.callsite!");
04919     assert(MMI.getCurrentCallSite() == 0 && "Overlapping call sites!");
04920 
04921     MMI.setCurrentCallSite(CI->getZExtValue());
04922     return nullptr;
04923   }
04924   case Intrinsic::eh_sjlj_functioncontext: {
04925     // Get and store the index of the function context.
04926     MachineFrameInfo *MFI = DAG.getMachineFunction().getFrameInfo();
04927     AllocaInst *FnCtx =
04928       cast<AllocaInst>(I.getArgOperand(0)->stripPointerCasts());
04929     int FI = FuncInfo.StaticAllocaMap[FnCtx];
04930     MFI->setFunctionContextIndex(FI);
04931     return nullptr;
04932   }
04933   case Intrinsic::eh_sjlj_setjmp: {
04934     SDValue Ops[2];
04935     Ops[0] = getRoot();
04936     Ops[1] = getValue(I.getArgOperand(0));
04937     SDValue Op = DAG.getNode(ISD::EH_SJLJ_SETJMP, sdl,
04938                              DAG.getVTList(MVT::i32, MVT::Other), Ops);
04939     setValue(&I, Op.getValue(0));
04940     DAG.setRoot(Op.getValue(1));
04941     return nullptr;
04942   }
04943   case Intrinsic::eh_sjlj_longjmp: {
04944     DAG.setRoot(DAG.getNode(ISD::EH_SJLJ_LONGJMP, sdl, MVT::Other,
04945                             getRoot(), getValue(I.getArgOperand(0))));
04946     return nullptr;
04947   }
04948 
04949   case Intrinsic::x86_mmx_pslli_w:
04950   case Intrinsic::x86_mmx_pslli_d:
04951   case Intrinsic::x86_mmx_pslli_q:
04952   case Intrinsic::x86_mmx_psrli_w:
04953   case Intrinsic::x86_mmx_psrli_d:
04954   case Intrinsic::x86_mmx_psrli_q:
04955   case Intrinsic::x86_mmx_psrai_w:
04956   case Intrinsic::x86_mmx_psrai_d: {
04957     SDValue ShAmt = getValue(I.getArgOperand(1));
04958     if (isa<ConstantSDNode>(ShAmt)) {
04959       visitTargetIntrinsic(I, Intrinsic);
04960       return nullptr;
04961     }
04962     unsigned NewIntrinsic = 0;
04963     EVT ShAmtVT = MVT::v2i32;
04964     switch (Intrinsic) {
04965     case Intrinsic::x86_mmx_pslli_w:
04966       NewIntrinsic = Intrinsic::x86_mmx_psll_w;
04967       break;
04968     case Intrinsic::x86_mmx_pslli_d:
04969       NewIntrinsic = Intrinsic::x86_mmx_psll_d;
04970       break;
04971     case Intrinsic::x86_mmx_pslli_q:
04972       NewIntrinsic = Intrinsic::x86_mmx_psll_q;
04973       break;
04974     case Intrinsic::x86_mmx_psrli_w:
04975       NewIntrinsic = Intrinsic::x86_mmx_psrl_w;
04976       break;
04977     case Intrinsic::x86_mmx_psrli_d:
04978       NewIntrinsic = Intrinsic::x86_mmx_psrl_d;
04979       break;
04980     case Intrinsic::x86_mmx_psrli_q:
04981       NewIntrinsic = Intrinsic::x86_mmx_psrl_q;
04982       break;
04983     case Intrinsic::x86_mmx_psrai_w:
04984       NewIntrinsic = Intrinsic::x86_mmx_psra_w;
04985       break;
04986     case Intrinsic::x86_mmx_psrai_d:
04987       NewIntrinsic = Intrinsic::x86_mmx_psra_d;
04988       break;
04989     default: llvm_unreachable("Impossible intrinsic");  // Can't reach here.
04990     }
04991 
04992     // The vector shift intrinsics with scalars uses 32b shift amounts but
04993     // the sse2/mmx shift instructions reads 64 bits. Set the upper 32 bits
04994     // to be zero.
04995     // We must do this early because v2i32 is not a legal type.
04996     SDValue ShOps[2];
04997     ShOps[0] = ShAmt;
04998     ShOps[1] = DAG.getConstant(0, MVT::i32);
04999     ShAmt =  DAG.getNode(ISD::BUILD_VECTOR, sdl, ShAmtVT, ShOps);
05000     EVT DestVT = TLI->getValueType(I.getType());
05001     ShAmt = DAG.getNode(ISD::BITCAST, sdl, DestVT, ShAmt);
05002     Res = DAG.getNode(ISD::INTRINSIC_WO_CHAIN, sdl, DestVT,
05003                        DAG.getConstant(NewIntrinsic, MVT::i32),
05004                        getValue(I.getArgOperand(0)), ShAmt);
05005     setValue(&I, Res);
05006     return nullptr;
05007   }
05008   case Intrinsic::x86_avx_vinsertf128_pd_256:
05009   case Intrinsic::x86_avx_vinsertf128_ps_256:
05010   case Intrinsic::x86_avx_vinsertf128_si_256:
05011   case Intrinsic::x86_avx2_vinserti128: {
05012     EVT DestVT = TLI->getValueType(I.getType());
05013     EVT ElVT = TLI->getValueType(I.getArgOperand(1)->getType());
05014     uint64_t Idx = (cast<ConstantInt>(I.getArgOperand(2))->getZExtValue() & 1) *
05015                    ElVT.getVectorNumElements();
05016     Res = DAG.getNode(ISD::INSERT_SUBVECTOR, sdl, DestVT,
05017                       getValue(I.getArgOperand(0)),
05018                       getValue(I.getArgOperand(1)),
05019                       DAG.getConstant(Idx, TLI->getVectorIdxTy()));
05020     setValue(&I, Res);
05021     return nullptr;
05022   }
05023   case Intrinsic::x86_avx_vextractf128_pd_256:
05024   case Intrinsic::x86_avx_vextractf128_ps_256:
05025   case Intrinsic::x86_avx_vextractf128_si_256:
05026   case Intrinsic::x86_avx2_vextracti128: {
05027     EVT DestVT = TLI->getValueType(I.getType());
05028     uint64_t Idx = (cast<ConstantInt>(I.getArgOperand(1))->getZExtValue() & 1) *
05029                    DestVT.getVectorNumElements();
05030     Res = DAG.getNode(ISD::EXTRACT_SUBVECTOR, sdl, DestVT,
05031                       getValue(I.getArgOperand(0)),
05032                       DAG.getConstant(Idx, TLI->getVectorIdxTy()));
05033     setValue(&I, Res);
05034     return nullptr;
05035   }
05036   case Intrinsic::convertff:
05037   case Intrinsic::convertfsi:
05038   case Intrinsic::convertfui:
05039   case Intrinsic::convertsif:
05040   case Intrinsic::convertuif:
05041   case Intrinsic::convertss:
05042   case Intrinsic::convertsu:
05043   case Intrinsic::convertus:
05044   case Intrinsic::convertuu: {
05045     ISD::CvtCode Code = ISD::CVT_INVALID;
05046     switch (Intrinsic) {
05047     default: llvm_unreachable("Impossible intrinsic");  // Can't reach here.
05048     case Intrinsic::convertff:  Code = ISD::CVT_FF; break;
05049     case Intrinsic::convertfsi: Code = ISD::CVT_FS; break;
05050     case Intrinsic::convertfui: Code = ISD::CVT_FU; break;
05051     case Intrinsic::convertsif: Code = ISD::CVT_SF; break;
05052     case Intrinsic::convertuif: Code = ISD::CVT_UF; break;
05053     case Intrinsic::convertss:  Code = ISD::CVT_SS; break;
05054     case Intrinsic::convertsu:  Code = ISD::CVT_SU; break;
05055     case Intrinsic::convertus:  Code = ISD::CVT_US; break;
05056     case Intrinsic::convertuu:  Code = ISD::CVT_UU; break;
05057     }
05058     EVT DestVT = TLI->getValueType(I.getType());
05059     const Value *Op1 = I.getArgOperand(0);
05060     Res = DAG.getConvertRndSat(DestVT, sdl, getValue(Op1),
05061                                DAG.getValueType(DestVT),
05062                                DAG.getValueType(getValue(Op1).getValueType()),
05063                                getValue(I.getArgOperand(1)),
05064                                getValue(I.getArgOperand(2)),
05065                                Code);
05066     setValue(&I, Res);
05067     return nullptr;
05068   }
05069   case Intrinsic::powi:
05070     setValue(&I, ExpandPowI(sdl, getValue(I.getArgOperand(0)),
05071                             getValue(I.getArgOperand(1)), DAG));
05072     return nullptr;
05073   case Intrinsic::log:
05074     setValue(&I, expandLog(sdl, getValue(I.getArgOperand(0)), DAG, *TLI));
05075     return nullptr;
05076   case Intrinsic::log2:
05077     setValue(&I, expandLog2(sdl, getValue(I.getArgOperand(0)), DAG, *TLI));
05078     return nullptr;
05079   case Intrinsic::log10:
05080     setValue(&I, expandLog10(sdl, getValue(I.getArgOperand(0)), DAG, *TLI));
05081     return nullptr;
05082   case Intrinsic::exp:
05083     setValue(&I, expandExp(sdl, getValue(I.getArgOperand(0)), DAG, *TLI));
05084     return nullptr;
05085   case Intrinsic::exp2:
05086     setValue(&I, expandExp2(sdl, getValue(I.getArgOperand(0)), DAG, *TLI));
05087     return nullptr;
05088   case Intrinsic::pow:
05089     setValue(&I, expandPow(sdl, getValue(I.getArgOperand(0)),
05090                            getValue(I.getArgOperand(1)), DAG, *TLI));
05091     return nullptr;
05092   case Intrinsic::sqrt:
05093   case Intrinsic::fabs:
05094   case Intrinsic::sin:
05095   case Intrinsic::cos:
05096   case Intrinsic::floor:
05097   case Intrinsic::ceil:
05098   case Intrinsic::trunc:
05099   case Intrinsic::rint:
05100   case Intrinsic::nearbyint:
05101   case Intrinsic::round: {
05102     unsigned Opcode;
05103     switch (Intrinsic) {
05104     default: llvm_unreachable("Impossible intrinsic");  // Can't reach here.
05105     case Intrinsic::sqrt:      Opcode = ISD::FSQRT;      break;
05106     case Intrinsic::fabs:      Opcode = ISD::FABS;       break;
05107     case Intrinsic::sin:       Opcode = ISD::FSIN;       break;
05108     case Intrinsic::cos:       Opcode = ISD::FCOS;       break;
05109     case Intrinsic::floor:     Opcode = ISD::FFLOOR;     break;
05110     case Intrinsic::ceil:      Opcode = ISD::FCEIL;      break;
05111     case Intrinsic::trunc:     Opcode = ISD::FTRUNC;     break;
05112     case Intrinsic::rint:      Opcode = ISD::FRINT;      break;
05113     case Intrinsic::nearbyint: Opcode = ISD::FNEARBYINT; break;
05114     case Intrinsic::round:     Opcode = ISD::FROUND;     break;
05115     }
05116 
05117     setValue(&I, DAG.getNode(Opcode, sdl,
05118                              getValue(I.getArgOperand(0)).getValueType(),
05119                              getValue(I.getArgOperand(0))));
05120     return nullptr;
05121   }
05122   case Intrinsic::copysign:
05123     setValue(&I, DAG.getNode(ISD::FCOPYSIGN, sdl,
05124                              getValue(I.getArgOperand(0)).getValueType(),
05125                              getValue(I.getArgOperand(0)),
05126                              getValue(I.getArgOperand(1))));
05127     return nullptr;
05128   case Intrinsic::fma:
05129     setValue(&I, DAG.getNode(ISD::FMA, sdl,
05130                              getValue(I.getArgOperand(0)).getValueType(),
05131                              getValue(I.getArgOperand(0)),
05132                              getValue(I.getArgOperand(1)),
05133                              getValue(I.getArgOperand(2))));
05134     return nullptr;
05135   case Intrinsic::fmuladd: {
05136     EVT VT = TLI->getValueType(I.getType());
05137     if (TM.Options.AllowFPOpFusion != FPOpFusion::Strict &&
05138         TLI->isFMAFasterThanFMulAndFAdd(VT)) {
05139       setValue(&I, DAG.getNode(ISD::FMA, sdl,
05140                                getValue(I.getArgOperand(0)).getValueType(),
05141                                getValue(I.getArgOperand(0)),
05142                                getValue(I.getArgOperand(1)),
05143                                getValue(I.getArgOperand(2))));
05144     } else {
05145       SDValue Mul = DAG.getNode(ISD::FMUL, sdl,
05146                                 getValue(I.getArgOperand(0)).getValueType(),
05147                                 getValue(I.getArgOperand(0)),
05148                                 getValue(I.getArgOperand(1)));
05149       SDValue Add = DAG.getNode(ISD::FADD, sdl,
05150                                 getValue(I.getArgOperand(0)).getValueType(),
05151                                 Mul,
05152                                 getValue(I.getArgOperand(2)));
05153       setValue(&I, Add);
05154     }
05155     return nullptr;
05156   }
05157   case Intrinsic::convert_to_fp16:
05158     setValue(&I, DAG.getNode(ISD::BITCAST, sdl, MVT::i16,
05159                              DAG.getNode(ISD::FP_ROUND, sdl, MVT::f16,
05160                                          getValue(I.getArgOperand(0)),
05161                                          DAG.getTargetConstant(0, MVT::i32))));
05162     return nullptr;
05163   case Intrinsic::convert_from_fp16:
05164     setValue(&I,
05165              DAG.getNode(ISD::FP_EXTEND, sdl, TLI->getValueType(I.getType()),
05166                          DAG.getNode(ISD::BITCAST, sdl, MVT::f16,
05167                                      getValue(I.getArgOperand(0)))));
05168     return nullptr;
05169   case Intrinsic::pcmarker: {
05170     SDValue Tmp = getValue(I.getArgOperand(0));
05171     DAG.setRoot(DAG.getNode(ISD::PCMARKER, sdl, MVT::Other, getRoot(), Tmp));
05172     return nullptr;
05173   }
05174   case Intrinsic::readcyclecounter: {
05175     SDValue Op = getRoot();
05176     Res = DAG.getNode(ISD::READCYCLECOUNTER, sdl,
05177                       DAG.getVTList(MVT::i64, MVT::Other), Op);
05178     setValue(&I, Res);
05179     DAG.setRoot(Res.getValue(1));
05180     return nullptr;
05181   }
05182   case Intrinsic::bswap:
05183     setValue(&I, DAG.getNode(ISD::BSWAP, sdl,
05184                              getValue(I.getArgOperand(0)).getValueType(),
05185                              getValue(I.getArgOperand(0))));
05186     return nullptr;
05187   case Intrinsic::cttz: {
05188     SDValue Arg = getValue(I.getArgOperand(0));
05189     ConstantInt *CI = cast<ConstantInt>(I.getArgOperand(1));
05190     EVT Ty = Arg.getValueType();
05191     setValue(&I, DAG.getNode(CI->isZero() ? ISD::CTTZ : ISD::CTTZ_ZERO_UNDEF,
05192                              sdl, Ty, Arg));
05193     return nullptr;
05194   }
05195   case Intrinsic::ctlz: {
05196     SDValue Arg = getValue(I.getArgOperand(0));
05197     ConstantInt *CI = cast<ConstantInt>(I.getArgOperand(1));
05198     EVT Ty = Arg.getValueType();
05199     setValue(&I, DAG.getNode(CI->isZero() ? ISD::CTLZ : ISD::CTLZ_ZERO_UNDEF,
05200                              sdl, Ty, Arg));
05201     return nullptr;
05202   }
05203   case Intrinsic::ctpop: {
05204     SDValue Arg = getValue(I.getArgOperand(0));
05205     EVT Ty = Arg.getValueType();
05206     setValue(&I, DAG.getNode(ISD::CTPOP, sdl, Ty, Arg));
05207     return nullptr;
05208   }
05209   case Intrinsic::stacksave: {
05210     SDValue Op = getRoot();
05211     Res = DAG.getNode(ISD::STACKSAVE, sdl,
05212                       DAG.getVTList(TLI->getPointerTy(), MVT::Other), Op);
05213     setValue(&I, Res);
05214     DAG.setRoot(Res.getValue(1));
05215     return nullptr;
05216   }
05217   case Intrinsic::stackrestore: {
05218     Res = getValue(I.getArgOperand(0));
05219     DAG.setRoot(DAG.getNode(ISD::STACKRESTORE, sdl, MVT::Other, getRoot(), Res));
05220     return nullptr;
05221   }
05222   case Intrinsic::stackprotector: {
05223     // Emit code into the DAG to store the stack guard onto the stack.
05224     MachineFunction &MF = DAG.getMachineFunction();
05225     MachineFrameInfo *MFI = MF.getFrameInfo();
05226     EVT PtrTy = TLI->getPointerTy();
05227 
05228     SDValue Src = getValue(I.getArgOperand(0));   // The guard's value.
05229     AllocaInst *Slot = cast<AllocaInst>(I.getArgOperand(1));
05230 
05231     int FI = FuncInfo.StaticAllocaMap[Slot];
05232     MFI->setStackProtectorIndex(FI);
05233 
05234     SDValue FIN = DAG.getFrameIndex(FI, PtrTy);
05235 
05236     // Store the stack protector onto the stack.
05237     Res = DAG.getStore(getRoot(), sdl, Src, FIN,
05238                        MachinePointerInfo::getFixedStack(FI),
05239                        true, false, 0);
05240     setValue(&I, Res);
05241     DAG.setRoot(Res);
05242     return nullptr;
05243   }
05244   case Intrinsic::objectsize: {
05245     // If we don't know by now, we're never going to know.
05246     ConstantInt *CI = dyn_cast<ConstantInt>(I.getArgOperand(1));
05247 
05248     assert(CI && "Non-constant type in __builtin_object_size?");
05249 
05250     SDValue Arg = getValue(I.getCalledValue());
05251     EVT Ty = Arg.getValueType();
05252 
05253     if (CI->isZero())
05254       Res = DAG.getConstant(-1ULL, Ty);
05255     else
05256       Res = DAG.getConstant(0, Ty);
05257 
05258     setValue(&I, Res);
05259     return nullptr;
05260   }
05261   case Intrinsic::annotation:
05262   case Intrinsic::ptr_annotation:
05263     // Drop the intrinsic, but forward the value
05264     setValue(&I, getValue(I.getOperand(0)));
05265     return nullptr;
05266   case Intrinsic::var_annotation:
05267     // Discard annotate attributes
05268     return nullptr;
05269 
05270   case Intrinsic::init_trampoline: {
05271     const Function *F = cast<Function>(I.getArgOperand(1)->stripPointerCasts());
05272 
05273     SDValue Ops[6];
05274     Ops[0] = getRoot();
05275     Ops[1] = getValue(I.getArgOperand(0));
05276     Ops[2] = getValue(I.getArgOperand(1));
05277     Ops[3] = getValue(I.getArgOperand(2));
05278     Ops[4] = DAG.getSrcValue(I.getArgOperand(0));
05279     Ops[5] = DAG.getSrcValue(F);
05280 
05281     Res = DAG.getNode(ISD::INIT_TRAMPOLINE, sdl, MVT::Other, Ops);
05282 
05283     DAG.setRoot(Res);
05284     return nullptr;
05285   }
05286   case Intrinsic::adjust_trampoline: {
05287     setValue(&I, DAG.getNode(ISD::ADJUST_TRAMPOLINE, sdl,
05288                              TLI->getPointerTy(),
05289                              getValue(I.getArgOperand(0))));
05290     return nullptr;
05291   }
05292   case Intrinsic::gcroot:
05293     if (GFI) {
05294       const Value *Alloca = I.getArgOperand(0)->stripPointerCasts();
05295       const Constant *TypeMap = cast<Constant>(I.getArgOperand(1));
05296 
05297       FrameIndexSDNode *FI = cast<FrameIndexSDNode>(getValue(Alloca).getNode());
05298       GFI->addStackRoot(FI->getIndex(), TypeMap);
05299     }
05300     return nullptr;
05301   case Intrinsic::gcread:
05302   case Intrinsic::gcwrite:
05303     llvm_unreachable("GC failed to lower gcread/gcwrite intrinsics!");
05304   case Intrinsic::flt_rounds:
05305     setValue(&I, DAG.getNode(ISD::FLT_ROUNDS_, sdl, MVT::i32));
05306     return nullptr;
05307 
05308   case Intrinsic::expect: {
05309     // Just replace __builtin_expect(exp, c) with EXP.
05310     setValue(&I, getValue(I.getArgOperand(0)));
05311     return nullptr;
05312   }
05313 
05314   case Intrinsic::debugtrap:
05315   case Intrinsic::trap: {
05316     StringRef TrapFuncName = TM.Options.getTrapFunctionName();
05317     if (TrapFuncName.empty()) {
05318       ISD::NodeType Op = (Intrinsic == Intrinsic::trap) ?
05319         ISD::TRAP : ISD::DEBUGTRAP;
05320       DAG.setRoot(DAG.getNode(Op, sdl,MVT::Other, getRoot()));
05321       return nullptr;
05322     }
05323     TargetLowering::ArgListTy Args;
05324 
05325     TargetLowering::CallLoweringInfo CLI(DAG);
05326     CLI.setDebugLoc(sdl).setChain(getRoot())
05327       .setCallee(CallingConv::C, I.getType(),
05328                  DAG.getExternalSymbol(TrapFuncName.data(), TLI->getPointerTy()),
05329                  std::move(Args), 0);
05330 
05331     std::pair<SDValue, SDValue> Result = TLI->LowerCallTo(CLI);
05332     DAG.setRoot(Result.second);
05333     return nullptr;
05334   }
05335 
05336   case Intrinsic::uadd_with_overflow:
05337   case Intrinsic::sadd_with_overflow:
05338   case Intrinsic::usub_with_overflow:
05339   case Intrinsic::ssub_with_overflow:
05340   case Intrinsic::umul_with_overflow:
05341   case Intrinsic::smul_with_overflow: {
05342     ISD::NodeType Op;
05343     switch (Intrinsic) {
05344     default: llvm_unreachable("Impossible intrinsic");  // Can't reach here.
05345     case Intrinsic::uadd_with_overflow: Op = ISD::UADDO; break;
05346     case Intrinsic::sadd_with_overflow: Op = ISD::SADDO; break;
05347     case Intrinsic::usub_with_overflow: Op = ISD::USUBO; break;
05348     case Intrinsic::ssub_with_overflow: Op = ISD::SSUBO; break;
05349     case Intrinsic::umul_with_overflow: Op = ISD::UMULO; break;
05350     case Intrinsic::smul_with_overflow: Op = ISD::SMULO; break;
05351     }
05352     SDValue Op1 = getValue(I.getArgOperand(0));
05353     SDValue Op2 = getValue(I.getArgOperand(1));
05354 
05355     SDVTList VTs = DAG.getVTList(Op1.getValueType(), MVT::i1);
05356     setValue(&I, DAG.getNode(Op, sdl, VTs, Op1, Op2));
05357     return nullptr;
05358   }
05359   case Intrinsic::prefetch: {
05360     SDValue Ops[5];
05361     unsigned rw = cast<ConstantInt>(I.getArgOperand(1))->getZExtValue();
05362     Ops[0] = getRoot();
05363     Ops[1] = getValue(I.getArgOperand(0));
05364     Ops[2] = getValue(I.getArgOperand(1));
05365     Ops[3] = getValue(I.getArgOperand(2));
05366     Ops[4] = getValue(I.getArgOperand(3));
05367     DAG.setRoot(DAG.getMemIntrinsicNode(ISD::PREFETCH, sdl,
05368                                         DAG.getVTList(MVT::Other), Ops,
05369                                         EVT::getIntegerVT(*Context, 8),
05370                                         MachinePointerInfo(I.getArgOperand(0)),
05371                                         0, /* align */
05372                                         false, /* volatile */
05373                                         rw==0, /* read */
05374                                         rw==1)); /* write */
05375     return nullptr;
05376   }
05377   case Intrinsic::lifetime_start:
05378   case Intrinsic::lifetime_end: {
05379     bool IsStart = (Intrinsic == Intrinsic::lifetime_start);
05380     // Stack coloring is not enabled in O0, discard region information.
05381     if (TM.getOptLevel() == CodeGenOpt::None)
05382       return nullptr;
05383 
05384     SmallVector<Value *, 4> Allocas;
05385     GetUnderlyingObjects(I.getArgOperand(1), Allocas, DL);
05386 
05387     for (SmallVectorImpl<Value*>::iterator Object = Allocas.begin(),
05388            E = Allocas.end(); Object != E; ++Object) {
05389       AllocaInst *LifetimeObject = dyn_cast_or_null<AllocaInst>(*Object);
05390 
05391       // Could not find an Alloca.
05392       if (!LifetimeObject)
05393         continue;
05394 
05395       int FI = FuncInfo.StaticAllocaMap[LifetimeObject];
05396 
05397       SDValue Ops[2];
05398       Ops[0] = getRoot();
05399       Ops[1] = DAG.getFrameIndex(FI, TLI->getPointerTy(), true);
05400       unsigned Opcode = (IsStart ? ISD::LIFETIME_START : ISD::LIFETIME_END);
05401 
05402       Res = DAG.getNode(Opcode, sdl, MVT::Other, Ops);
05403       DAG.setRoot(Res);
05404     }
05405     return nullptr;
05406   }
05407   case Intrinsic::invariant_start:
05408     // Discard region information.
05409     setValue(&I, DAG.getUNDEF(TLI->getPointerTy()));
05410     return nullptr;
05411   case Intrinsic::invariant_end:
05412     // Discard region information.
05413     return nullptr;
05414   case Intrinsic::stackprotectorcheck: {
05415     // Do not actually emit anything for this basic block. Instead we initialize
05416     // the stack protector descriptor and export the guard variable so we can
05417     // access it in FinishBasicBlock.
05418     const BasicBlock *BB = I.getParent();
05419     SPDescriptor.initialize(BB, FuncInfo.MBBMap[BB], I);
05420     ExportFromCurrentBlock(SPDescriptor.getGuard());
05421 
05422     // Flush our exports since we are going to process a terminator.
05423     (void)getControlRoot();
05424     return nullptr;
05425   }
05426   case Intrinsic::clear_cache:
05427     return TLI->getClearCacheBuiltinName();
05428   case Intrinsic::donothing:
05429     // ignore
05430     return nullptr;
05431   case Intrinsic::experimental_stackmap: {
05432     visitStackmap(I);
05433     return nullptr;
05434   }
05435   case Intrinsic::experimental_patchpoint_void:
05436   case Intrinsic::experimental_patchpoint_i64: {
05437     visitPatchpoint(I);
05438     return nullptr;
05439   }
05440   }
05441 }
05442 
05443 void SelectionDAGBuilder::LowerCallTo(ImmutableCallSite CS, SDValue Callee,
05444                                       bool isTailCall,
05445                                       MachineBasicBlock *LandingPad) {
05446   const TargetLowering *TLI = TM.getTargetLowering();
05447   PointerType *PT = cast<PointerType>(CS.getCalledValue()->getType());
05448   FunctionType *FTy = cast<FunctionType>(PT->getElementType());
05449   Type *RetTy = FTy->getReturnType();
05450   MachineModuleInfo &MMI = DAG.getMachineFunction().getMMI();
05451   MCSymbol *BeginLabel = nullptr;
05452 
05453   TargetLowering::ArgListTy Args;
05454   TargetLowering::ArgListEntry Entry;
05455   Args.reserve(CS.arg_size());
05456 
05457   for (ImmutableCallSite::arg_iterator i = CS.arg_begin(), e = CS.arg_end();
05458        i != e; ++i) {
05459     const Value *V = *i;
05460 
05461     // Skip empty types
05462     if (V->getType()->isEmptyTy())
05463       continue;
05464 
05465     SDValue ArgNode = getValue(V);
05466     Entry.Node = ArgNode; Entry.Ty = V->getType();
05467 
05468     // Skip the first return-type Attribute to get to params.
05469     Entry.setAttributes(&CS, i - CS.arg_begin() + 1);
05470     Args.push_back(Entry);
05471   }
05472 
05473   if (LandingPad) {
05474     // Insert a label before the invoke call to mark the try range.  This can be
05475     // used to detect deletion of the invoke via the MachineModuleInfo.
05476     BeginLabel = MMI.getContext().CreateTempSymbol();
05477 
05478     // For SjLj, keep track of which landing pads go with which invokes
05479     // so as to maintain the ordering of pads in the LSDA.
05480     unsigned CallSiteIndex = MMI.getCurrentCallSite();
05481     if (CallSiteIndex) {
05482       MMI.setCallSiteBeginLabel(BeginLabel, CallSiteIndex);
05483       LPadToCallSiteMap[LandingPad].push_back(CallSiteIndex);
05484 
05485       // Now that the call site is handled, stop tracking it.
05486       MMI.setCurrentCallSite(0);
05487     }
05488 
05489     // Both PendingLoads and PendingExports must be flushed here;
05490     // this call might not return.
05491     (void)getRoot();
05492     DAG.setRoot(DAG.getEHLabel(getCurSDLoc(), getControlRoot(), BeginLabel));
05493   }
05494 
05495   // Check if target-independent constraints permit a tail call here.
05496   // Target-dependent constraints are checked within TLI->LowerCallTo.
05497   if (isTailCall && !isInTailCallPosition(CS, DAG.getTarget()))
05498     isTailCall = false;
05499 
05500   TargetLowering::CallLoweringInfo CLI(DAG);
05501   CLI.setDebugLoc(getCurSDLoc()).setChain(getRoot())
05502     .setCallee(RetTy, FTy, Callee, std::move(Args), CS).setTailCall(isTailCall);
05503 
05504   std::pair<SDValue,SDValue> Result = TLI->LowerCallTo(CLI);
05505   assert((isTailCall || Result.second.getNode()) &&
05506          "Non-null chain expected with non-tail call!");
05507   assert((Result.second.getNode() || !Result.first.getNode()) &&
05508          "Null value expected with tail call!");
05509   if (Result.first.getNode())
05510     setValue(CS.getInstruction(), Result.first);
05511 
05512   if (!Result.second.getNode()) {
05513     // As a special case, a null chain means that a tail call has been emitted
05514     // and the DAG root is already updated.
05515     HasTailCall = true;
05516 
05517     // Since there's no actual continuation from this block, nothing can be
05518     // relying on us setting vregs for them.
05519     PendingExports.clear();
05520   } else {
05521     DAG.setRoot(Result.second);
05522   }
05523 
05524   if (LandingPad) {
05525     // Insert a label at the end of the invoke call to mark the try range.  This
05526     // can be used to detect deletion of the invoke via the MachineModuleInfo.
05527     MCSymbol *EndLabel = MMI.getContext().CreateTempSymbol();
05528     DAG.setRoot(DAG.getEHLabel(getCurSDLoc(), getRoot(), EndLabel));
05529 
05530     // Inform MachineModuleInfo of range.
05531     MMI.addInvoke(LandingPad, BeginLabel, EndLabel);
05532   }
05533 }
05534 
05535 /// IsOnlyUsedInZeroEqualityComparison - Return true if it only matters that the
05536 /// value is equal or not-equal to zero.
05537 static bool IsOnlyUsedInZeroEqualityComparison(const Value *V) {
05538   for (const User *U : V->users()) {
05539     if (const ICmpInst *IC = dyn_cast<ICmpInst>(U))
05540       if (IC->isEquality())
05541         if (const Constant *C = dyn_cast<Constant>(IC->getOperand(1)))
05542           if (C->isNullValue())
05543             continue;
05544     // Unknown instruction.
05545     return false;
05546   }
05547   return true;
05548 }
05549 
05550 static SDValue getMemCmpLoad(const Value *PtrVal, MVT LoadVT,
05551                              Type *LoadTy,
05552                              SelectionDAGBuilder &Builder) {
05553 
05554   // Check to see if this load can be trivially constant folded, e.g. if the
05555   // input is from a string literal.
05556   if (const Constant *LoadInput = dyn_cast<Constant>(PtrVal)) {
05557     // Cast pointer to the type we really want to load.
05558     LoadInput = ConstantExpr::getBitCast(const_cast<Constant *>(LoadInput),
05559                                          PointerType::getUnqual(LoadTy));
05560 
05561     if (const Constant *LoadCst =
05562           ConstantFoldLoadFromConstPtr(const_cast<Constant *>(LoadInput),
05563                                        Builder.DL))
05564       return Builder.getValue(LoadCst);
05565   }
05566 
05567   // Otherwise, we have to emit the load.  If the pointer is to unfoldable but
05568   // still constant memory, the input chain can be the entry node.
05569   SDValue Root;
05570   bool ConstantMemory = false;
05571 
05572   // Do not serialize (non-volatile) loads of constant memory with anything.
05573   if (Builder.AA->pointsToConstantMemory(PtrVal)) {
05574     Root = Builder.DAG.getEntryNode();
05575     ConstantMemory = true;
05576   } else {
05577     // Do not serialize non-volatile loads against each other.
05578     Root = Builder.DAG.getRoot();
05579   }
05580 
05581   SDValue Ptr = Builder.getValue(PtrVal);
05582   SDValue LoadVal = Builder.DAG.getLoad(LoadVT, Builder.getCurSDLoc(), Root,
05583                                         Ptr, MachinePointerInfo(PtrVal),
05584                                         false /*volatile*/,
05585                                         false /*nontemporal*/,
05586                                         false /*isinvariant*/, 1 /* align=1 */);
05587 
05588   if (!ConstantMemory)
05589     Builder.PendingLoads.push_back(LoadVal.getValue(1));
05590   return LoadVal;
05591 }
05592 
05593 /// processIntegerCallValue - Record the value for an instruction that
05594 /// produces an integer result, converting the type where necessary.
05595 void SelectionDAGBuilder::processIntegerCallValue(const Instruction &I,
05596                                                   SDValue Value,
05597                                                   bool IsSigned) {
05598   EVT VT = TM.getTargetLowering()->getValueType(I.getType(), true);
05599   if (IsSigned)
05600     Value = DAG.getSExtOrTrunc(Value, getCurSDLoc(), VT);
05601   else
05602     Value = DAG.getZExtOrTrunc(Value, getCurSDLoc(), VT);
05603   setValue(&I, Value);
05604 }
05605 
05606 /// visitMemCmpCall - See if we can lower a call to memcmp in an optimized form.
05607 /// If so, return true and lower it, otherwise return false and it will be
05608 /// lowered like a normal call.
05609 bool SelectionDAGBuilder::visitMemCmpCall(const CallInst &I) {
05610   // Verify that the prototype makes sense.  int memcmp(void*,void*,size_t)
05611   if (I.getNumArgOperands() != 3)
05612     return false;
05613 
05614   const Value *LHS = I.getArgOperand(0), *RHS = I.getArgOperand(1);
05615   if (!LHS->getType()->isPointerTy() || !RHS->getType()->isPointerTy() ||
05616       !I.getArgOperand(2)->getType()->isIntegerTy() ||
05617       !I.getType()->isIntegerTy())
05618     return false;
05619 
05620   const Value *Size = I.getArgOperand(2);
05621   const ConstantInt *CSize = dyn_cast<ConstantInt>(Size);
05622   if (CSize && CSize->getZExtValue() == 0) {
05623     EVT CallVT = TM.getTargetLowering()->getValueType(I.getType(), true);
05624     setValue(&I, DAG.getConstant(0, CallVT));
05625     return true;
05626   }
05627 
05628   const TargetSelectionDAGInfo &TSI = DAG.getSelectionDAGInfo();
05629   std::pair<SDValue, SDValue> Res =
05630     TSI.EmitTargetCodeForMemcmp(DAG, getCurSDLoc(), DAG.getRoot(),
05631                                 getValue(LHS), getValue(RHS), getValue(Size),
05632                                 MachinePointerInfo(LHS),
05633                                 MachinePointerInfo(RHS));
05634   if (Res.first.getNode()) {
05635     processIntegerCallValue(I, Res.first, true);
05636     PendingLoads.push_back(Res.second);
05637     return true;
05638   }
05639 
05640   // memcmp(S1,S2,2) != 0 -> (*(short*)LHS != *(short*)RHS)  != 0
05641   // memcmp(S1,S2,4) != 0 -> (*(int*)LHS != *(int*)RHS)  != 0
05642   if (CSize && IsOnlyUsedInZeroEqualityComparison(&I)) {
05643     bool ActuallyDoIt = true;
05644     MVT LoadVT;
05645     Type *LoadTy;
05646     switch (CSize->getZExtValue()) {
05647     default:
05648       LoadVT = MVT::Other;
05649       LoadTy = nullptr;
05650       ActuallyDoIt = false;
05651       break;
05652     case 2:
05653       LoadVT = MVT::i16;
05654       LoadTy = Type::getInt16Ty(CSize->getContext());
05655       break;
05656     case 4:
05657       LoadVT = MVT::i32;
05658       LoadTy = Type::getInt32Ty(CSize->getContext());
05659       break;
05660     case 8:
05661       LoadVT = MVT::i64;
05662       LoadTy = Type::getInt64Ty(CSize->getContext());
05663       break;
05664         /*
05665     case 16:
05666       LoadVT = MVT::v4i32;
05667       LoadTy = Type::getInt32Ty(CSize->getContext());
05668       LoadTy = VectorType::get(LoadTy, 4);
05669       break;
05670          */
05671     }
05672 
05673     // This turns into unaligned loads.  We only do this if the target natively
05674     // supports the MVT we'll be loading or if it is small enough (<= 4) that
05675     // we'll only produce a small number of byte loads.
05676 
05677     // Require that we can find a legal MVT, and only do this if the target
05678     // supports unaligned loads of that type.  Expanding into byte loads would
05679     // bloat the code.
05680     const TargetLowering *TLI = TM.getTargetLowering();
05681     if (ActuallyDoIt && CSize->getZExtValue() > 4) {
05682       unsigned DstAS = LHS->getType()->getPointerAddressSpace();
05683       unsigned SrcAS = RHS->getType()->getPointerAddressSpace();
05684       // TODO: Handle 5 byte compare as 4-byte + 1 byte.
05685       // TODO: Handle 8 byte compare on x86-32 as two 32-bit loads.
05686       if (!TLI->isTypeLegal(LoadVT) ||
05687           !TLI->allowsUnalignedMemoryAccesses(LoadVT, SrcAS) ||
05688           !TLI->allowsUnalignedMemoryAccesses(LoadVT, DstAS))
05689         ActuallyDoIt = false;
05690     }
05691 
05692     if (ActuallyDoIt) {
05693       SDValue LHSVal = getMemCmpLoad(LHS, LoadVT, LoadTy, *this);
05694       SDValue RHSVal = getMemCmpLoad(RHS, LoadVT, LoadTy, *this);
05695 
05696       SDValue Res = DAG.getSetCC(getCurSDLoc(), MVT::i1, LHSVal, RHSVal,
05697                                  ISD::SETNE);
05698       processIntegerCallValue(I, Res, false);
05699       return true;
05700     }
05701   }
05702 
05703 
05704   return false;
05705 }
05706 
05707 /// visitMemChrCall -- See if we can lower a memchr call into an optimized
05708 /// form.  If so, return true and lower it, otherwise return false and it
05709 /// will be lowered like a normal call.
05710 bool SelectionDAGBuilder::visitMemChrCall(const CallInst &I) {
05711   // Verify that the prototype makes sense.  void *memchr(void *, int, size_t)
05712   if (I.getNumArgOperands() != 3)
05713     return false;
05714 
05715   const Value *Src = I.getArgOperand(0);
05716   const Value *Char = I.getArgOperand(1);
05717   const Value *Length = I.getArgOperand(2);
05718   if (!Src->getType()->isPointerTy() ||
05719       !Char->getType()->isIntegerTy() ||
05720       !Length->getType()->isIntegerTy() ||
05721       !I.getType()->isPointerTy())
05722     return false;
05723 
05724   const TargetSelectionDAGInfo &TSI = DAG.getSelectionDAGInfo();
05725   std::pair<SDValue, SDValue> Res =
05726     TSI.EmitTargetCodeForMemchr(DAG, getCurSDLoc(), DAG.getRoot(),
05727                                 getValue(Src), getValue(Char), getValue(Length),
05728                                 MachinePointerInfo(Src));
05729   if (Res.first.getNode()) {
05730     setValue(&I, Res.first);
05731     PendingLoads.push_back(Res.second);
05732     return true;
05733   }
05734 
05735   return false;
05736 }
05737 
05738 /// visitStrCpyCall -- See if we can lower a strcpy or stpcpy call into an
05739 /// optimized form.  If so, return true and lower it, otherwise return false
05740 /// and it will be lowered like a normal call.
05741 bool SelectionDAGBuilder::visitStrCpyCall(const CallInst &I, bool isStpcpy) {
05742   // Verify that the prototype makes sense.  char *strcpy(char *, char *)
05743   if (I.getNumArgOperands() != 2)
05744     return false;
05745 
05746   const Value *Arg0 = I.getArgOperand(0), *Arg1 = I.getArgOperand(1);
05747   if (!Arg0->getType()->isPointerTy() ||
05748       !Arg1->getType()->isPointerTy() ||
05749       !I.getType()->isPointerTy())
05750     return false;
05751 
05752   const TargetSelectionDAGInfo &TSI = DAG.getSelectionDAGInfo();
05753   std::pair<SDValue, SDValue> Res =
05754     TSI.EmitTargetCodeForStrcpy(DAG, getCurSDLoc(), getRoot(),
05755                                 getValue(Arg0), getValue(Arg1),
05756                                 MachinePointerInfo(Arg0),
05757                                 MachinePointerInfo(Arg1), isStpcpy);
05758   if (Res.first.getNode()) {
05759     setValue(&I, Res.first);
05760     DAG.setRoot(Res.second);
05761     return true;
05762   }
05763 
05764   return false;
05765 }
05766 
05767 /// visitStrCmpCall - See if we can lower a call to strcmp in an optimized form.
05768 /// If so, return true and lower it, otherwise return false and it will be
05769 /// lowered like a normal call.
05770 bool SelectionDAGBuilder::visitStrCmpCall(const CallInst &I) {
05771   // Verify that the prototype makes sense.  int strcmp(void*,void*)
05772   if (I.getNumArgOperands() != 2)
05773     return false;
05774 
05775   const Value *Arg0 = I.getArgOperand(0), *Arg1 = I.getArgOperand(1);
05776   if (!Arg0->getType()->isPointerTy() ||
05777       !Arg1->getType()->isPointerTy() ||
05778       !I.getType()->isIntegerTy())
05779     return false;
05780 
05781   const TargetSelectionDAGInfo &TSI = DAG.getSelectionDAGInfo();
05782   std::pair<SDValue, SDValue> Res =
05783     TSI.EmitTargetCodeForStrcmp(DAG, getCurSDLoc(), DAG.getRoot(),
05784                                 getValue(Arg0), getValue(Arg1),
05785                                 MachinePointerInfo(Arg0),
05786                                 MachinePointerInfo(Arg1));
05787   if (Res.first.getNode()) {
05788     processIntegerCallValue(I, Res.first, true);
05789     PendingLoads.push_back(Res.second);
05790     return true;
05791   }
05792 
05793   return false;
05794 }
05795 
05796 /// visitStrLenCall -- See if we can lower a strlen call into an optimized
05797 /// form.  If so, return true and lower it, otherwise return false and it
05798 /// will be lowered like a normal call.
05799 bool SelectionDAGBuilder::visitStrLenCall(const CallInst &I) {
05800   // Verify that the prototype makes sense.  size_t strlen(char *)
05801   if (I.getNumArgOperands() != 1)
05802     return false;
05803 
05804   const Value *Arg0 = I.getArgOperand(0);
05805   if (!Arg0->getType()->isPointerTy() || !I.getType()->isIntegerTy())
05806     return false;
05807 
05808   const TargetSelectionDAGInfo &TSI = DAG.getSelectionDAGInfo();
05809   std::pair<SDValue, SDValue> Res =
05810     TSI.EmitTargetCodeForStrlen(DAG, getCurSDLoc(), DAG.getRoot(),
05811                                 getValue(Arg0), MachinePointerInfo(Arg0));
05812   if (Res.first.getNode()) {
05813     processIntegerCallValue(I, Res.first, false);
05814     PendingLoads.push_back(Res.second);
05815     return true;
05816   }
05817 
05818   return false;
05819 }
05820 
05821 /// visitStrNLenCall -- See if we can lower a strnlen call into an optimized
05822 /// form.  If so, return true and lower it, otherwise return false and it
05823 /// will be lowered like a normal call.
05824 bool SelectionDAGBuilder::visitStrNLenCall(const CallInst &I) {
05825   // Verify that the prototype makes sense.  size_t strnlen(char *, size_t)
05826   if (I.getNumArgOperands() != 2)
05827     return false;
05828 
05829   const Value *Arg0 = I.getArgOperand(0), *Arg1 = I.getArgOperand(1);
05830   if (!Arg0->getType()->isPointerTy() ||
05831       !Arg1->getType()->isIntegerTy() ||
05832       !I.getType()->isIntegerTy())
05833     return false;
05834 
05835   const TargetSelectionDAGInfo &TSI = DAG.getSelectionDAGInfo();
05836   std::pair<SDValue, SDValue> Res =
05837     TSI.EmitTargetCodeForStrnlen(DAG, getCurSDLoc(), DAG.getRoot(),
05838                                  getValue(Arg0), getValue(Arg1),
05839                                  MachinePointerInfo(Arg0));
05840   if (Res.first.getNode()) {
05841     processIntegerCallValue(I, Res.first, false);
05842     PendingLoads.push_back(Res.second);
05843     return true;
05844   }
05845 
05846   return false;
05847 }
05848 
05849 /// visitUnaryFloatCall - If a call instruction is a unary floating-point
05850 /// operation (as expected), translate it to an SDNode with the specified opcode
05851 /// and return true.
05852 bool SelectionDAGBuilder::visitUnaryFloatCall(const CallInst &I,
05853                                               unsigned Opcode) {
05854   // Sanity check that it really is a unary floating-point call.
05855   if (I.getNumArgOperands() != 1 ||
05856       !I.getArgOperand(0)->getType()->isFloatingPointTy() ||
05857       I.getType() != I.getArgOperand(0)->getType() ||
05858       !I.onlyReadsMemory())
05859     return false;
05860 
05861   SDValue Tmp = getValue(I.getArgOperand(0));
05862   setValue(&I, DAG.getNode(Opcode, getCurSDLoc(), Tmp.getValueType(), Tmp));
05863   return true;
05864 }
05865 
05866 void SelectionDAGBuilder::visitCall(const CallInst &I) {
05867   // Handle inline assembly differently.
05868   if (isa<InlineAsm>(I.getCalledValue())) {
05869     visitInlineAsm(&I);
05870     return;
05871   }
05872 
05873   MachineModuleInfo &MMI = DAG.getMachineFunction().getMMI();
05874   ComputeUsesVAFloatArgument(I, &MMI);
05875 
05876   const char *RenameFn = nullptr;
05877   if (Function *F = I.getCalledFunction()) {
05878     if (F->isDeclaration()) {
05879       if (const TargetIntrinsicInfo *II = TM.getIntrinsicInfo()) {
05880         if (unsigned IID = II->getIntrinsicID(F)) {
05881           RenameFn = visitIntrinsicCall(I, IID);
05882           if (!RenameFn)
05883             return;
05884         }
05885       }
05886       if (unsigned IID = F->getIntrinsicID()) {
05887         RenameFn = visitIntrinsicCall(I, IID);
05888         if (!RenameFn)
05889           return;
05890       }
05891     }
05892 
05893     // Check for well-known libc/libm calls.  If the function is internal, it
05894     // can't be a library call.
05895     LibFunc::Func Func;
05896     if (!F->hasLocalLinkage() && F->hasName() &&
05897         LibInfo->getLibFunc(F->getName(), Func) &&
05898         LibInfo->hasOptimizedCodeGen(Func)) {
05899       switch (Func) {
05900       default: break;
05901       case LibFunc::copysign:
05902       case LibFunc::copysignf:
05903       case LibFunc::copysignl:
05904         if (I.getNumArgOperands() == 2 &&   // Basic sanity checks.
05905             I.getArgOperand(0)->getType()->isFloatingPointTy() &&
05906             I.getType() == I.getArgOperand(0)->getType() &&
05907             I.getType() == I.getArgOperand(1)->getType() &&
05908             I.onlyReadsMemory()) {
05909           SDValue LHS = getValue(I.getArgOperand(0));
05910           SDValue RHS = getValue(I.getArgOperand(1));
05911           setValue(&I, DAG.getNode(ISD::FCOPYSIGN, getCurSDLoc(),
05912                                    LHS.getValueType(), LHS, RHS));
05913           return;
05914         }
05915         break;
05916       case LibFunc::fabs:
05917       case LibFunc::fabsf:
05918       case LibFunc::fabsl:
05919         if (visitUnaryFloatCall(I, ISD::FABS))
05920           return;
05921         break;
05922       case LibFunc::sin:
05923       case LibFunc::sinf:
05924       case LibFunc::sinl:
05925         if (visitUnaryFloatCall(I, ISD::FSIN))
05926           return;
05927         break;
05928       case LibFunc::cos:
05929       case LibFunc::cosf:
05930       case LibFunc::cosl:
05931         if (visitUnaryFloatCall(I, ISD::FCOS))
05932           return;
05933         break;
05934       case LibFunc::sqrt:
05935       case LibFunc::sqrtf:
05936       case LibFunc::sqrtl:
05937       case LibFunc::sqrt_finite:
05938       case LibFunc::sqrtf_finite:
05939       case LibFunc::sqrtl_finite:
05940         if (visitUnaryFloatCall(I, ISD::FSQRT))
05941           return;
05942         break;
05943       case LibFunc::floor:
05944       case LibFunc::floorf:
05945       case LibFunc::floorl:
05946         if (visitUnaryFloatCall(I, ISD::FFLOOR))
05947           return;
05948         break;
05949       case LibFunc::nearbyint:
05950       case LibFunc::nearbyintf:
05951       case LibFunc::nearbyintl:
05952         if (visitUnaryFloatCall(I, ISD::FNEARBYINT))
05953           return;
05954         break;
05955       case LibFunc::ceil:
05956       case LibFunc::ceilf:
05957       case LibFunc::ceill:
05958         if (visitUnaryFloatCall(I, ISD::FCEIL))
05959           return;
05960         break;
05961       case LibFunc::rint:
05962       case LibFunc::rintf:
05963       case LibFunc::rintl:
05964         if (visitUnaryFloatCall(I, ISD::FRINT))
05965           return;
05966         break;
05967       case LibFunc::round:
05968       case LibFunc::roundf:
05969       case LibFunc::roundl:
05970         if (visitUnaryFloatCall(I, ISD::FROUND))
05971           return;
05972         break;
05973       case LibFunc::trunc:
05974       case LibFunc::truncf:
05975       case LibFunc::truncl:
05976         if (visitUnaryFloatCall(I, ISD::FTRUNC))
05977           return;
05978         break;
05979       case LibFunc::log2:
05980       case LibFunc::log2f:
05981       case LibFunc::log2l:
05982         if (visitUnaryFloatCall(I, ISD::FLOG2))
05983           return;
05984         break;
05985       case LibFunc::exp2:
05986       case LibFunc::exp2f:
05987       case LibFunc::exp2l:
05988         if (visitUnaryFloatCall(I, ISD::FEXP2))
05989           return;
05990         break;
05991       case LibFunc::memcmp:
05992         if (visitMemCmpCall(I))
05993           return;
05994         break;
05995       case LibFunc::memchr:
05996         if (visitMemChrCall(I))
05997           return;
05998         break;
05999       case LibFunc::strcpy:
06000         if (visitStrCpyCall(I, false))
06001           return;
06002         break;
06003       case LibFunc::stpcpy:
06004         if (visitStrCpyCall(I, true))
06005           return;
06006         break;
06007       case LibFunc::strcmp:
06008         if (visitStrCmpCall(I))
06009           return;
06010         break;
06011       case LibFunc::strlen:
06012         if (visitStrLenCall(I))
06013           return;
06014         break;
06015       case LibFunc::strnlen:
06016         if (visitStrNLenCall(I))
06017           return;
06018         break;
06019       }
06020     }
06021   }
06022 
06023   SDValue Callee;
06024   if (!RenameFn)
06025     Callee = getValue(I.getCalledValue());
06026   else
06027     Callee = DAG.getExternalSymbol(RenameFn,
06028                                    TM.getTargetLowering()->getPointerTy());
06029 
06030   // Check if we can potentially perform a tail call. More detailed checking is
06031   // be done within LowerCallTo, after more information about the call is known.
06032   LowerCallTo(&I, Callee, I.isTailCall());
06033 }
06034 
06035 namespace {
06036 
06037 /// AsmOperandInfo - This contains information for each constraint that we are
06038 /// lowering.
06039 class SDISelAsmOperandInfo : public TargetLowering::AsmOperandInfo {
06040 public:
06041   /// CallOperand - If this is the result output operand or a clobber
06042   /// this is null, otherwise it is the incoming operand to the CallInst.
06043   /// This gets modified as the asm is processed.
06044   SDValue CallOperand;
06045 
06046   /// AssignedRegs - If this is a register or register class operand, this
06047   /// contains the set of register corresponding to the operand.
06048   RegsForValue AssignedRegs;
06049 
06050   explicit SDISelAsmOperandInfo(const TargetLowering::AsmOperandInfo &info)
06051     : TargetLowering::AsmOperandInfo(info), CallOperand(nullptr,0) {
06052   }
06053 
06054   /// getCallOperandValEVT - Return the EVT of the Value* that this operand
06055   /// corresponds to.  If there is no Value* for this operand, it returns
06056   /// MVT::Other.
06057   EVT getCallOperandValEVT(LLVMContext &Context,
06058                            const TargetLowering &TLI,
06059                            const DataLayout *DL) const {
06060     if (!CallOperandVal) return MVT::Other;
06061 
06062     if (isa<BasicBlock>(CallOperandVal))
06063       return TLI.getPointerTy();
06064 
06065     llvm::Type *OpTy = CallOperandVal->getType();
06066 
06067     // FIXME: code duplicated from TargetLowering::ParseConstraints().
06068     // If this is an indirect operand, the operand is a pointer to the
06069     // accessed type.
06070     if (isIndirect) {
06071       llvm::PointerType *PtrTy = dyn_cast<PointerType>(OpTy);
06072       if (!PtrTy)
06073         report_fatal_error("Indirect operand for inline asm not a pointer!");
06074       OpTy = PtrTy->getElementType();
06075     }
06076 
06077     // Look for vector wrapped in a struct. e.g. { <16 x i8> }.
06078     if (StructType *STy = dyn_cast<StructType>(OpTy))
06079       if (STy->getNumElements() == 1)
06080         OpTy = STy->getElementType(0);
06081 
06082     // If OpTy is not a single value, it may be a struct/union that we
06083     // can tile with integers.
06084     if (!OpTy->isSingleValueType() && OpTy->isSized()) {
06085       unsigned BitSize = DL->getTypeSizeInBits(OpTy);
06086       switch (BitSize) {
06087       default: break;
06088       case 1:
06089       case 8:
06090       case 16:
06091       case 32:
06092       case 64:
06093       case 128:
06094         OpTy = IntegerType::get(Context, BitSize);
06095         break;
06096       }
06097     }
06098 
06099     return TLI.getValueType(OpTy, true);
06100   }
06101 };
06102 
06103 typedef SmallVector<SDISelAsmOperandInfo,16> SDISelAsmOperandInfoVector;
06104 
06105 } // end anonymous namespace
06106 
06107 /// GetRegistersForValue - Assign registers (virtual or physical) for the
06108 /// specified operand.  We prefer to assign virtual registers, to allow the
06109 /// register allocator to handle the assignment process.  However, if the asm
06110 /// uses features that we can't model on machineinstrs, we have SDISel do the
06111 /// allocation.  This produces generally horrible, but correct, code.
06112 ///
06113 ///   OpInfo describes the operand.
06114 ///
06115 static void GetRegistersForValue(SelectionDAG &DAG,
06116                                  const TargetLowering &TLI,
06117                                  SDLoc DL,
06118                                  SDISelAsmOperandInfo &OpInfo) {
06119   LLVMContext &Context = *DAG.getContext();
06120 
06121   MachineFunction &MF = DAG.getMachineFunction();
06122   SmallVector<unsigned, 4> Regs;
06123 
06124   // If this is a constraint for a single physreg, or a constraint for a
06125   // register class, find it.
06126   std::pair<unsigned, const TargetRegisterClass*> PhysReg =
06127     TLI.getRegForInlineAsmConstraint(OpInfo.ConstraintCode,
06128                                      OpInfo.ConstraintVT);
06129 
06130   unsigned NumRegs = 1;
06131   if (OpInfo.ConstraintVT != MVT::Other) {
06132     // If this is a FP input in an integer register (or visa versa) insert a bit
06133     // cast of the input value.  More generally, handle any case where the input
06134     // value disagrees with the register class we plan to stick this in.
06135     if (OpInfo.Type == InlineAsm::isInput &&
06136         PhysReg.second && !PhysReg.second->hasType(OpInfo.ConstraintVT)) {
06137       // Try to convert to the first EVT that the reg class contains.  If the
06138       // types are identical size, use a bitcast to convert (e.g. two differing
06139       // vector types).
06140       MVT RegVT = *PhysReg.second->vt_begin();
06141       if (RegVT.getSizeInBits() == OpInfo.CallOperand.getValueSizeInBits()) {
06142         OpInfo.CallOperand = DAG.getNode(ISD::BITCAST, DL,
06143                                          RegVT, OpInfo.CallOperand);
06144         OpInfo.ConstraintVT = RegVT;
06145       } else if (RegVT.isInteger() && OpInfo.ConstraintVT.isFloatingPoint()) {
06146         // If the input is a FP value and we want it in FP registers, do a
06147         // bitcast to the corresponding integer type.  This turns an f64 value
06148         // into i64, which can be passed with two i32 values on a 32-bit
06149         // machine.
06150         RegVT = MVT::getIntegerVT(OpInfo.ConstraintVT.getSizeInBits());
06151         OpInfo.CallOperand = DAG.getNode(ISD::BITCAST, DL,
06152                                          RegVT, OpInfo.CallOperand);
06153         OpInfo.ConstraintVT = RegVT;
06154       }
06155     }
06156 
06157     NumRegs = TLI.getNumRegisters(Context, OpInfo.ConstraintVT);
06158   }
06159 
06160   MVT RegVT;
06161   EVT ValueVT = OpInfo.ConstraintVT;
06162 
06163   // If this is a constraint for a specific physical register, like {r17},
06164   // assign it now.
06165   if (unsigned AssignedReg = PhysReg.first) {
06166     const TargetRegisterClass *RC = PhysReg.second;
06167     if (OpInfo.ConstraintVT == MVT::Other)
06168       ValueVT = *RC->vt_begin();
06169 
06170     // Get the actual register value type.  This is important, because the user
06171     // may have asked for (e.g.) the AX register in i32 type.  We need to
06172     // remember that AX is actually i16 to get the right extension.
06173     RegVT = *RC->vt_begin();
06174 
06175     // This is a explicit reference to a physical register.
06176     Regs.push_back(AssignedReg);
06177 
06178     // If this is an expanded reference, add the rest of the regs to Regs.
06179     if (NumRegs != 1) {
06180       TargetRegisterClass::iterator I = RC->begin();
06181       for (; *I != AssignedReg; ++I)
06182         assert(I != RC->end() && "Didn't find reg!");
06183 
06184       // Already added the first reg.
06185       --NumRegs; ++I;
06186       for (; NumRegs; --NumRegs, ++I) {
06187         assert(I != RC->end() && "Ran out of registers to allocate!");
06188         Regs.push_back(*I);
06189       }
06190     }
06191 
06192     OpInfo.AssignedRegs = RegsForValue(Regs, RegVT, ValueVT);
06193     return;
06194   }
06195 
06196   // Otherwise, if this was a reference to an LLVM register class, create vregs
06197   // for this reference.
06198   if (const TargetRegisterClass *RC = PhysReg.second) {
06199     RegVT = *RC->vt_begin();
06200     if (OpInfo.ConstraintVT == MVT::Other)
06201       ValueVT = RegVT;
06202 
06203     // Create the appropriate number of virtual registers.
06204     MachineRegisterInfo &RegInfo = MF.getRegInfo();
06205     for (; NumRegs; --NumRegs)
06206       Regs.push_back(RegInfo.createVirtualRegister(RC));
06207 
06208     OpInfo.AssignedRegs = RegsForValue(Regs, RegVT, ValueVT);
06209     return;
06210   }
06211 
06212   // Otherwise, we couldn't allocate enough registers for this.
06213 }
06214 
06215 /// visitInlineAsm - Handle a call to an InlineAsm object.
06216 ///
06217 void SelectionDAGBuilder::visitInlineAsm(ImmutableCallSite CS) {
06218   const InlineAsm *IA = cast<InlineAsm>(CS.getCalledValue());
06219 
06220   /// ConstraintOperands - Information about all of the constraints.
06221   SDISelAsmOperandInfoVector ConstraintOperands;
06222 
06223   const TargetLowering *TLI = TM.getTargetLowering();
06224   TargetLowering::AsmOperandInfoVector
06225     TargetConstraints = TLI->ParseConstraints(CS);
06226 
06227   bool hasMemory = false;
06228 
06229   unsigned ArgNo = 0;   // ArgNo - The argument of the CallInst.
06230   unsigned ResNo = 0;   // ResNo - The result number of the next output.
06231   for (unsigned i = 0, e = TargetConstraints.size(); i != e; ++i) {
06232     ConstraintOperands.push_back(SDISelAsmOperandInfo(TargetConstraints[i]));
06233     SDISelAsmOperandInfo &OpInfo = ConstraintOperands.back();
06234 
06235     MVT OpVT = MVT::Other;
06236 
06237     // Compute the value type for each operand.
06238     switch (OpInfo.Type) {
06239     case InlineAsm::isOutput:
06240       // Indirect outputs just consume an argument.
06241       if (OpInfo.isIndirect) {
06242         OpInfo.CallOperandVal = const_cast<Value *>(CS.getArgument(ArgNo++));
06243         break;
06244       }
06245 
06246       // The return value of the call is this value.  As such, there is no
06247       // corresponding argument.