LLVM API Documentation

SelectionDAGBuilder.cpp
Go to the documentation of this file.
00001 //===-- SelectionDAGBuilder.cpp - Selection-DAG building ------------------===//
00002 //
00003 //                     The LLVM Compiler Infrastructure
00004 //
00005 // This file is distributed under the University of Illinois Open Source
00006 // License. See LICENSE.TXT for details.
00007 //
00008 //===----------------------------------------------------------------------===//
00009 //
00010 // This implements routines for translating from LLVM IR into SelectionDAG IR.
00011 //
00012 //===----------------------------------------------------------------------===//
00013 
00014 #include "SelectionDAGBuilder.h"
00015 #include "SDNodeDbgValue.h"
00016 #include "llvm/ADT/BitVector.h"
00017 #include "llvm/ADT/Optional.h"
00018 #include "llvm/ADT/SmallSet.h"
00019 #include "llvm/Analysis/AliasAnalysis.h"
00020 #include "llvm/Analysis/BranchProbabilityInfo.h"
00021 #include "llvm/Analysis/ConstantFolding.h"
00022 #include "llvm/Analysis/ValueTracking.h"
00023 #include "llvm/CodeGen/Analysis.h"
00024 #include "llvm/CodeGen/FastISel.h"
00025 #include "llvm/CodeGen/FunctionLoweringInfo.h"
00026 #include "llvm/CodeGen/GCMetadata.h"
00027 #include "llvm/CodeGen/GCStrategy.h"
00028 #include "llvm/CodeGen/MachineFrameInfo.h"
00029 #include "llvm/CodeGen/MachineFunction.h"
00030 #include "llvm/CodeGen/MachineInstrBuilder.h"
00031 #include "llvm/CodeGen/MachineJumpTableInfo.h"
00032 #include "llvm/CodeGen/MachineModuleInfo.h"
00033 #include "llvm/CodeGen/MachineRegisterInfo.h"
00034 #include "llvm/CodeGen/SelectionDAG.h"
00035 #include "llvm/CodeGen/StackMaps.h"
00036 #include "llvm/IR/CallingConv.h"
00037 #include "llvm/IR/Constants.h"
00038 #include "llvm/IR/DataLayout.h"
00039 #include "llvm/IR/DebugInfo.h"
00040 #include "llvm/IR/DerivedTypes.h"
00041 #include "llvm/IR/Function.h"
00042 #include "llvm/IR/GlobalVariable.h"
00043 #include "llvm/IR/InlineAsm.h"
00044 #include "llvm/IR/Instructions.h"
00045 #include "llvm/IR/IntrinsicInst.h"
00046 #include "llvm/IR/Intrinsics.h"
00047 #include "llvm/IR/LLVMContext.h"
00048 #include "llvm/IR/Module.h"
00049 #include "llvm/Support/CommandLine.h"
00050 #include "llvm/Support/Debug.h"
00051 #include "llvm/Support/ErrorHandling.h"
00052 #include "llvm/Support/MathExtras.h"
00053 #include "llvm/Support/raw_ostream.h"
00054 #include "llvm/Target/TargetFrameLowering.h"
00055 #include "llvm/Target/TargetInstrInfo.h"
00056 #include "llvm/Target/TargetIntrinsicInfo.h"
00057 #include "llvm/Target/TargetLibraryInfo.h"
00058 #include "llvm/Target/TargetLowering.h"
00059 #include "llvm/Target/TargetOptions.h"
00060 #include "llvm/Target/TargetSelectionDAGInfo.h"
00061 #include <algorithm>
00062 using namespace llvm;
00063 
00064 #define DEBUG_TYPE "isel"
00065 
00066 /// LimitFloatPrecision - Generate low-precision inline sequences for
00067 /// some float libcalls (6, 8 or 12 bits).
00068 static unsigned LimitFloatPrecision;
00069 
00070 static cl::opt<unsigned, true>
00071 LimitFPPrecision("limit-float-precision",
00072                  cl::desc("Generate low-precision inline sequences "
00073                           "for some float libcalls"),
00074                  cl::location(LimitFloatPrecision),
00075                  cl::init(0));
00076 
00077 // Limit the width of DAG chains. This is important in general to prevent
00078 // prevent DAG-based analysis from blowing up. For example, alias analysis and
00079 // load clustering may not complete in reasonable time. It is difficult to
00080 // recognize and avoid this situation within each individual analysis, and
00081 // future analyses are likely to have the same behavior. Limiting DAG width is
00082 // the safe approach, and will be especially important with global DAGs.
00083 //
00084 // MaxParallelChains default is arbitrarily high to avoid affecting
00085 // optimization, but could be lowered to improve compile time. Any ld-ld-st-st
00086 // sequence over this should have been converted to llvm.memcpy by the
00087 // frontend. It easy to induce this behavior with .ll code such as:
00088 // %buffer = alloca [4096 x i8]
00089 // %data = load [4096 x i8]* %argPtr
00090 // store [4096 x i8] %data, [4096 x i8]* %buffer
00091 static const unsigned MaxParallelChains = 64;
00092 
00093 static SDValue getCopyFromPartsVector(SelectionDAG &DAG, SDLoc DL,
00094                                       const SDValue *Parts, unsigned NumParts,
00095                                       MVT PartVT, EVT ValueVT, const Value *V);
00096 
00097 /// getCopyFromParts - Create a value that contains the specified legal parts
00098 /// combined into the value they represent.  If the parts combine to a type
00099 /// larger then ValueVT then AssertOp can be used to specify whether the extra
00100 /// bits are known to be zero (ISD::AssertZext) or sign extended from ValueVT
00101 /// (ISD::AssertSext).
00102 static SDValue getCopyFromParts(SelectionDAG &DAG, SDLoc DL,
00103                                 const SDValue *Parts,
00104                                 unsigned NumParts, MVT PartVT, EVT ValueVT,
00105                                 const Value *V,
00106                                 ISD::NodeType AssertOp = ISD::DELETED_NODE) {
00107   if (ValueVT.isVector())
00108     return getCopyFromPartsVector(DAG, DL, Parts, NumParts,
00109                                   PartVT, ValueVT, V);
00110 
00111   assert(NumParts > 0 && "No parts to assemble!");
00112   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
00113   SDValue Val = Parts[0];
00114 
00115   if (NumParts > 1) {
00116     // Assemble the value from multiple parts.
00117     if (ValueVT.isInteger()) {
00118       unsigned PartBits = PartVT.getSizeInBits();
00119       unsigned ValueBits = ValueVT.getSizeInBits();
00120 
00121       // Assemble the power of 2 part.
00122       unsigned RoundParts = NumParts & (NumParts - 1) ?
00123         1 << Log2_32(NumParts) : NumParts;
00124       unsigned RoundBits = PartBits * RoundParts;
00125       EVT RoundVT = RoundBits == ValueBits ?
00126         ValueVT : EVT::getIntegerVT(*DAG.getContext(), RoundBits);
00127       SDValue Lo, Hi;
00128 
00129       EVT HalfVT = EVT::getIntegerVT(*DAG.getContext(), RoundBits/2);
00130 
00131       if (RoundParts > 2) {
00132         Lo = getCopyFromParts(DAG, DL, Parts, RoundParts / 2,
00133                               PartVT, HalfVT, V);
00134         Hi = getCopyFromParts(DAG, DL, Parts + RoundParts / 2,
00135                               RoundParts / 2, PartVT, HalfVT, V);
00136       } else {
00137         Lo = DAG.getNode(ISD::BITCAST, DL, HalfVT, Parts[0]);
00138         Hi = DAG.getNode(ISD::BITCAST, DL, HalfVT, Parts[1]);
00139       }
00140 
00141       if (TLI.isBigEndian())
00142         std::swap(Lo, Hi);
00143 
00144       Val = DAG.getNode(ISD::BUILD_PAIR, DL, RoundVT, Lo, Hi);
00145 
00146       if (RoundParts < NumParts) {
00147         // Assemble the trailing non-power-of-2 part.
00148         unsigned OddParts = NumParts - RoundParts;
00149         EVT OddVT = EVT::getIntegerVT(*DAG.getContext(), OddParts * PartBits);
00150         Hi = getCopyFromParts(DAG, DL,
00151                               Parts + RoundParts, OddParts, PartVT, OddVT, V);
00152 
00153         // Combine the round and odd parts.
00154         Lo = Val;
00155         if (TLI.isBigEndian())
00156           std::swap(Lo, Hi);
00157         EVT TotalVT = EVT::getIntegerVT(*DAG.getContext(), NumParts * PartBits);
00158         Hi = DAG.getNode(ISD::ANY_EXTEND, DL, TotalVT, Hi);
00159         Hi = DAG.getNode(ISD::SHL, DL, TotalVT, Hi,
00160                          DAG.getConstant(Lo.getValueType().getSizeInBits(),
00161                                          TLI.getPointerTy()));
00162         Lo = DAG.getNode(ISD::ZERO_EXTEND, DL, TotalVT, Lo);
00163         Val = DAG.getNode(ISD::OR, DL, TotalVT, Lo, Hi);
00164       }
00165     } else if (PartVT.isFloatingPoint()) {
00166       // FP split into multiple FP parts (for ppcf128)
00167       assert(ValueVT == EVT(MVT::ppcf128) && PartVT == MVT::f64 &&
00168              "Unexpected split");
00169       SDValue Lo, Hi;
00170       Lo = DAG.getNode(ISD::BITCAST, DL, EVT(MVT::f64), Parts[0]);
00171       Hi = DAG.getNode(ISD::BITCAST, DL, EVT(MVT::f64), Parts[1]);
00172       if (TLI.isBigEndian())
00173         std::swap(Lo, Hi);
00174       Val = DAG.getNode(ISD::BUILD_PAIR, DL, ValueVT, Lo, Hi);
00175     } else {
00176       // FP split into integer parts (soft fp)
00177       assert(ValueVT.isFloatingPoint() && PartVT.isInteger() &&
00178              !PartVT.isVector() && "Unexpected split");
00179       EVT IntVT = EVT::getIntegerVT(*DAG.getContext(), ValueVT.getSizeInBits());
00180       Val = getCopyFromParts(DAG, DL, Parts, NumParts, PartVT, IntVT, V);
00181     }
00182   }
00183 
00184   // There is now one part, held in Val.  Correct it to match ValueVT.
00185   EVT PartEVT = Val.getValueType();
00186 
00187   if (PartEVT == ValueVT)
00188     return Val;
00189 
00190   if (PartEVT.isInteger() && ValueVT.isInteger()) {
00191     if (ValueVT.bitsLT(PartEVT)) {
00192       // For a truncate, see if we have any information to
00193       // indicate whether the truncated bits will always be
00194       // zero or sign-extension.
00195       if (AssertOp != ISD::DELETED_NODE)
00196         Val = DAG.getNode(AssertOp, DL, PartEVT, Val,
00197                           DAG.getValueType(ValueVT));
00198       return DAG.getNode(ISD::TRUNCATE, DL, ValueVT, Val);
00199     }
00200     return DAG.getNode(ISD::ANY_EXTEND, DL, ValueVT, Val);
00201   }
00202 
00203   if (PartEVT.isFloatingPoint() && ValueVT.isFloatingPoint()) {
00204     // FP_ROUND's are always exact here.
00205     if (ValueVT.bitsLT(Val.getValueType()))
00206       return DAG.getNode(ISD::FP_ROUND, DL, ValueVT, Val,
00207                          DAG.getTargetConstant(1, TLI.getPointerTy()));
00208 
00209     return DAG.getNode(ISD::FP_EXTEND, DL, ValueVT, Val);
00210   }
00211 
00212   if (PartEVT.getSizeInBits() == ValueVT.getSizeInBits())
00213     return DAG.getNode(ISD::BITCAST, DL, ValueVT, Val);
00214 
00215   llvm_unreachable("Unknown mismatch!");
00216 }
00217 
00218 static void diagnosePossiblyInvalidConstraint(LLVMContext &Ctx, const Value *V,
00219                                               const Twine &ErrMsg) {
00220   const Instruction *I = dyn_cast_or_null<Instruction>(V);
00221   if (!V)
00222     return Ctx.emitError(ErrMsg);
00223 
00224   const char *AsmError = ", possible invalid constraint for vector type";
00225   if (const CallInst *CI = dyn_cast<CallInst>(I))
00226     if (isa<InlineAsm>(CI->getCalledValue()))
00227       return Ctx.emitError(I, ErrMsg + AsmError);
00228 
00229   return Ctx.emitError(I, ErrMsg);
00230 }
00231 
00232 /// getCopyFromPartsVector - Create a value that contains the specified legal
00233 /// parts combined into the value they represent.  If the parts combine to a
00234 /// type larger then ValueVT then AssertOp can be used to specify whether the
00235 /// extra bits are known to be zero (ISD::AssertZext) or sign extended from
00236 /// ValueVT (ISD::AssertSext).
00237 static SDValue getCopyFromPartsVector(SelectionDAG &DAG, SDLoc DL,
00238                                       const SDValue *Parts, unsigned NumParts,
00239                                       MVT PartVT, EVT ValueVT, const Value *V) {
00240   assert(ValueVT.isVector() && "Not a vector value");
00241   assert(NumParts > 0 && "No parts to assemble!");
00242   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
00243   SDValue Val = Parts[0];
00244 
00245   // Handle a multi-element vector.
00246   if (NumParts > 1) {
00247     EVT IntermediateVT;
00248     MVT RegisterVT;
00249     unsigned NumIntermediates;
00250     unsigned NumRegs =
00251     TLI.getVectorTypeBreakdown(*DAG.getContext(), ValueVT, IntermediateVT,
00252                                NumIntermediates, RegisterVT);
00253     assert(NumRegs == NumParts && "Part count doesn't match vector breakdown!");
00254     NumParts = NumRegs; // Silence a compiler warning.
00255     assert(RegisterVT == PartVT && "Part type doesn't match vector breakdown!");
00256     assert(RegisterVT == Parts[0].getSimpleValueType() &&
00257            "Part type doesn't match part!");
00258 
00259     // Assemble the parts into intermediate operands.
00260     SmallVector<SDValue, 8> Ops(NumIntermediates);
00261     if (NumIntermediates == NumParts) {
00262       // If the register was not expanded, truncate or copy the value,
00263       // as appropriate.
00264       for (unsigned i = 0; i != NumParts; ++i)
00265         Ops[i] = getCopyFromParts(DAG, DL, &Parts[i], 1,
00266                                   PartVT, IntermediateVT, V);
00267     } else if (NumParts > 0) {
00268       // If the intermediate type was expanded, build the intermediate
00269       // operands from the parts.
00270       assert(NumParts % NumIntermediates == 0 &&
00271              "Must expand into a divisible number of parts!");
00272       unsigned Factor = NumParts / NumIntermediates;
00273       for (unsigned i = 0; i != NumIntermediates; ++i)
00274         Ops[i] = getCopyFromParts(DAG, DL, &Parts[i * Factor], Factor,
00275                                   PartVT, IntermediateVT, V);
00276     }
00277 
00278     // Build a vector with BUILD_VECTOR or CONCAT_VECTORS from the
00279     // intermediate operands.
00280     Val = DAG.getNode(IntermediateVT.isVector() ?
00281                       ISD::CONCAT_VECTORS : ISD::BUILD_VECTOR, DL,
00282                       ValueVT, &Ops[0], NumIntermediates);
00283   }
00284 
00285   // There is now one part, held in Val.  Correct it to match ValueVT.
00286   EVT PartEVT = Val.getValueType();
00287 
00288   if (PartEVT == ValueVT)
00289     return Val;
00290 
00291   if (PartEVT.isVector()) {
00292     // If the element type of the source/dest vectors are the same, but the
00293     // parts vector has more elements than the value vector, then we have a
00294     // vector widening case (e.g. <2 x float> -> <4 x float>).  Extract the
00295     // elements we want.
00296     if (PartEVT.getVectorElementType() == ValueVT.getVectorElementType()) {
00297       assert(PartEVT.getVectorNumElements() > ValueVT.getVectorNumElements() &&
00298              "Cannot narrow, it would be a lossy transformation");
00299       return DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, ValueVT, Val,
00300                          DAG.getConstant(0, TLI.getVectorIdxTy()));
00301     }
00302 
00303     // Vector/Vector bitcast.
00304     if (ValueVT.getSizeInBits() == PartEVT.getSizeInBits())
00305       return DAG.getNode(ISD::BITCAST, DL, ValueVT, Val);
00306 
00307     assert(PartEVT.getVectorNumElements() == ValueVT.getVectorNumElements() &&
00308       "Cannot handle this kind of promotion");
00309     // Promoted vector extract
00310     bool Smaller = ValueVT.bitsLE(PartEVT);
00311     return DAG.getNode((Smaller ? ISD::TRUNCATE : ISD::ANY_EXTEND),
00312                        DL, ValueVT, Val);
00313 
00314   }
00315 
00316   // Trivial bitcast if the types are the same size and the destination
00317   // vector type is legal.
00318   if (PartEVT.getSizeInBits() == ValueVT.getSizeInBits() &&
00319       TLI.isTypeLegal(ValueVT))
00320     return DAG.getNode(ISD::BITCAST, DL, ValueVT, Val);
00321 
00322   // Handle cases such as i8 -> <1 x i1>
00323   if (ValueVT.getVectorNumElements() != 1) {
00324     diagnosePossiblyInvalidConstraint(*DAG.getContext(), V,
00325                                       "non-trivial scalar-to-vector conversion");
00326     return DAG.getUNDEF(ValueVT);
00327   }
00328 
00329   if (ValueVT.getVectorNumElements() == 1 &&
00330       ValueVT.getVectorElementType() != PartEVT) {
00331     bool Smaller = ValueVT.bitsLE(PartEVT);
00332     Val = DAG.getNode((Smaller ? ISD::TRUNCATE : ISD::ANY_EXTEND),
00333                        DL, ValueVT.getScalarType(), Val);
00334   }
00335 
00336   return DAG.getNode(ISD::BUILD_VECTOR, DL, ValueVT, Val);
00337 }
00338 
00339 static void getCopyToPartsVector(SelectionDAG &DAG, SDLoc dl,
00340                                  SDValue Val, SDValue *Parts, unsigned NumParts,
00341                                  MVT PartVT, const Value *V);
00342 
00343 /// getCopyToParts - Create a series of nodes that contain the specified value
00344 /// split into legal parts.  If the parts contain more bits than Val, then, for
00345 /// integers, ExtendKind can be used to specify how to generate the extra bits.
00346 static void getCopyToParts(SelectionDAG &DAG, SDLoc DL,
00347                            SDValue Val, SDValue *Parts, unsigned NumParts,
00348                            MVT PartVT, const Value *V,
00349                            ISD::NodeType ExtendKind = ISD::ANY_EXTEND) {
00350   EVT ValueVT = Val.getValueType();
00351 
00352   // Handle the vector case separately.
00353   if (ValueVT.isVector())
00354     return getCopyToPartsVector(DAG, DL, Val, Parts, NumParts, PartVT, V);
00355 
00356   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
00357   unsigned PartBits = PartVT.getSizeInBits();
00358   unsigned OrigNumParts = NumParts;
00359   assert(TLI.isTypeLegal(PartVT) && "Copying to an illegal type!");
00360 
00361   if (NumParts == 0)
00362     return;
00363 
00364   assert(!ValueVT.isVector() && "Vector case handled elsewhere");
00365   EVT PartEVT = PartVT;
00366   if (PartEVT == ValueVT) {
00367     assert(NumParts == 1 && "No-op copy with multiple parts!");
00368     Parts[0] = Val;
00369     return;
00370   }
00371 
00372   if (NumParts * PartBits > ValueVT.getSizeInBits()) {
00373     // If the parts cover more bits than the value has, promote the value.
00374     if (PartVT.isFloatingPoint() && ValueVT.isFloatingPoint()) {
00375       assert(NumParts == 1 && "Do not know what to promote to!");
00376       Val = DAG.getNode(ISD::FP_EXTEND, DL, PartVT, Val);
00377     } else {
00378       assert((PartVT.isInteger() || PartVT == MVT::x86mmx) &&
00379              ValueVT.isInteger() &&
00380              "Unknown mismatch!");
00381       ValueVT = EVT::getIntegerVT(*DAG.getContext(), NumParts * PartBits);
00382       Val = DAG.getNode(ExtendKind, DL, ValueVT, Val);
00383       if (PartVT == MVT::x86mmx)
00384         Val = DAG.getNode(ISD::BITCAST, DL, PartVT, Val);
00385     }
00386   } else if (PartBits == ValueVT.getSizeInBits()) {
00387     // Different types of the same size.
00388     assert(NumParts == 1 && PartEVT != ValueVT);
00389     Val = DAG.getNode(ISD::BITCAST, DL, PartVT, Val);
00390   } else if (NumParts * PartBits < ValueVT.getSizeInBits()) {
00391     // If the parts cover less bits than value has, truncate the value.
00392     assert((PartVT.isInteger() || PartVT == MVT::x86mmx) &&
00393            ValueVT.isInteger() &&
00394            "Unknown mismatch!");
00395     ValueVT = EVT::getIntegerVT(*DAG.getContext(), NumParts * PartBits);
00396     Val = DAG.getNode(ISD::TRUNCATE, DL, ValueVT, Val);
00397     if (PartVT == MVT::x86mmx)
00398       Val = DAG.getNode(ISD::BITCAST, DL, PartVT, Val);
00399   }
00400 
00401   // The value may have changed - recompute ValueVT.
00402   ValueVT = Val.getValueType();
00403   assert(NumParts * PartBits == ValueVT.getSizeInBits() &&
00404          "Failed to tile the value with PartVT!");
00405 
00406   if (NumParts == 1) {
00407     if (PartEVT != ValueVT)
00408       diagnosePossiblyInvalidConstraint(*DAG.getContext(), V,
00409                                         "scalar-to-vector conversion failed");
00410 
00411     Parts[0] = Val;
00412     return;
00413   }
00414 
00415   // Expand the value into multiple parts.
00416   if (NumParts & (NumParts - 1)) {
00417     // The number of parts is not a power of 2.  Split off and copy the tail.
00418     assert(PartVT.isInteger() && ValueVT.isInteger() &&
00419            "Do not know what to expand to!");
00420     unsigned RoundParts = 1 << Log2_32(NumParts);
00421     unsigned RoundBits = RoundParts * PartBits;
00422     unsigned OddParts = NumParts - RoundParts;
00423     SDValue OddVal = DAG.getNode(ISD::SRL, DL, ValueVT, Val,
00424                                  DAG.getIntPtrConstant(RoundBits));
00425     getCopyToParts(DAG, DL, OddVal, Parts + RoundParts, OddParts, PartVT, V);
00426 
00427     if (TLI.isBigEndian())
00428       // The odd parts were reversed by getCopyToParts - unreverse them.
00429       std::reverse(Parts + RoundParts, Parts + NumParts);
00430 
00431     NumParts = RoundParts;
00432     ValueVT = EVT::getIntegerVT(*DAG.getContext(), NumParts * PartBits);
00433     Val = DAG.getNode(ISD::TRUNCATE, DL, ValueVT, Val);
00434   }
00435 
00436   // The number of parts is a power of 2.  Repeatedly bisect the value using
00437   // EXTRACT_ELEMENT.
00438   Parts[0] = DAG.getNode(ISD::BITCAST, DL,
00439                          EVT::getIntegerVT(*DAG.getContext(),
00440                                            ValueVT.getSizeInBits()),
00441                          Val);
00442 
00443   for (unsigned StepSize = NumParts; StepSize > 1; StepSize /= 2) {
00444     for (unsigned i = 0; i < NumParts; i += StepSize) {
00445       unsigned ThisBits = StepSize * PartBits / 2;
00446       EVT ThisVT = EVT::getIntegerVT(*DAG.getContext(), ThisBits);
00447       SDValue &Part0 = Parts[i];
00448       SDValue &Part1 = Parts[i+StepSize/2];
00449 
00450       Part1 = DAG.getNode(ISD::EXTRACT_ELEMENT, DL,
00451                           ThisVT, Part0, DAG.getIntPtrConstant(1));
00452       Part0 = DAG.getNode(ISD::EXTRACT_ELEMENT, DL,
00453                           ThisVT, Part0, DAG.getIntPtrConstant(0));
00454 
00455       if (ThisBits == PartBits && ThisVT != PartVT) {
00456         Part0 = DAG.getNode(ISD::BITCAST, DL, PartVT, Part0);
00457         Part1 = DAG.getNode(ISD::BITCAST, DL, PartVT, Part1);
00458       }
00459     }
00460   }
00461 
00462   if (TLI.isBigEndian())
00463     std::reverse(Parts, Parts + OrigNumParts);
00464 }
00465 
00466 
00467 /// getCopyToPartsVector - Create a series of nodes that contain the specified
00468 /// value split into legal parts.
00469 static void getCopyToPartsVector(SelectionDAG &DAG, SDLoc DL,
00470                                  SDValue Val, SDValue *Parts, unsigned NumParts,
00471                                  MVT PartVT, const Value *V) {
00472   EVT ValueVT = Val.getValueType();
00473   assert(ValueVT.isVector() && "Not a vector");
00474   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
00475 
00476   if (NumParts == 1) {
00477     EVT PartEVT = PartVT;
00478     if (PartEVT == ValueVT) {
00479       // Nothing to do.
00480     } else if (PartVT.getSizeInBits() == ValueVT.getSizeInBits()) {
00481       // Bitconvert vector->vector case.
00482       Val = DAG.getNode(ISD::BITCAST, DL, PartVT, Val);
00483     } else if (PartVT.isVector() &&
00484                PartEVT.getVectorElementType() == ValueVT.getVectorElementType() &&
00485                PartEVT.getVectorNumElements() > ValueVT.getVectorNumElements()) {
00486       EVT ElementVT = PartVT.getVectorElementType();
00487       // Vector widening case, e.g. <2 x float> -> <4 x float>.  Shuffle in
00488       // undef elements.
00489       SmallVector<SDValue, 16> Ops;
00490       for (unsigned i = 0, e = ValueVT.getVectorNumElements(); i != e; ++i)
00491         Ops.push_back(DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL,
00492                                   ElementVT, Val, DAG.getConstant(i,
00493                                                   TLI.getVectorIdxTy())));
00494 
00495       for (unsigned i = ValueVT.getVectorNumElements(),
00496            e = PartVT.getVectorNumElements(); i != e; ++i)
00497         Ops.push_back(DAG.getUNDEF(ElementVT));
00498 
00499       Val = DAG.getNode(ISD::BUILD_VECTOR, DL, PartVT, &Ops[0], Ops.size());
00500 
00501       // FIXME: Use CONCAT for 2x -> 4x.
00502 
00503       //SDValue UndefElts = DAG.getUNDEF(VectorTy);
00504       //Val = DAG.getNode(ISD::CONCAT_VECTORS, DL, PartVT, Val, UndefElts);
00505     } else if (PartVT.isVector() &&
00506                PartEVT.getVectorElementType().bitsGE(
00507                  ValueVT.getVectorElementType()) &&
00508                PartEVT.getVectorNumElements() == ValueVT.getVectorNumElements()) {
00509 
00510       // Promoted vector extract
00511       bool Smaller = PartEVT.bitsLE(ValueVT);
00512       Val = DAG.getNode((Smaller ? ISD::TRUNCATE : ISD::ANY_EXTEND),
00513                         DL, PartVT, Val);
00514     } else{
00515       // Vector -> scalar conversion.
00516       assert(ValueVT.getVectorNumElements() == 1 &&
00517              "Only trivial vector-to-scalar conversions should get here!");
00518       Val = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL,
00519                         PartVT, Val, DAG.getConstant(0, TLI.getVectorIdxTy()));
00520 
00521       bool Smaller = ValueVT.bitsLE(PartVT);
00522       Val = DAG.getNode((Smaller ? ISD::TRUNCATE : ISD::ANY_EXTEND),
00523                          DL, PartVT, Val);
00524     }
00525 
00526     Parts[0] = Val;
00527     return;
00528   }
00529 
00530   // Handle a multi-element vector.
00531   EVT IntermediateVT;
00532   MVT RegisterVT;
00533   unsigned NumIntermediates;
00534   unsigned NumRegs = TLI.getVectorTypeBreakdown(*DAG.getContext(), ValueVT,
00535                                                 IntermediateVT,
00536                                                 NumIntermediates, RegisterVT);
00537   unsigned NumElements = ValueVT.getVectorNumElements();
00538 
00539   assert(NumRegs == NumParts && "Part count doesn't match vector breakdown!");
00540   NumParts = NumRegs; // Silence a compiler warning.
00541   assert(RegisterVT == PartVT && "Part type doesn't match vector breakdown!");
00542 
00543   // Split the vector into intermediate operands.
00544   SmallVector<SDValue, 8> Ops(NumIntermediates);
00545   for (unsigned i = 0; i != NumIntermediates; ++i) {
00546     if (IntermediateVT.isVector())
00547       Ops[i] = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL,
00548                            IntermediateVT, Val,
00549                    DAG.getConstant(i * (NumElements / NumIntermediates),
00550                                    TLI.getVectorIdxTy()));
00551     else
00552       Ops[i] = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL,
00553                            IntermediateVT, Val,
00554                            DAG.getConstant(i, TLI.getVectorIdxTy()));
00555   }
00556 
00557   // Split the intermediate operands into legal parts.
00558   if (NumParts == NumIntermediates) {
00559     // If the register was not expanded, promote or copy the value,
00560     // as appropriate.
00561     for (unsigned i = 0; i != NumParts; ++i)
00562       getCopyToParts(DAG, DL, Ops[i], &Parts[i], 1, PartVT, V);
00563   } else if (NumParts > 0) {
00564     // If the intermediate type was expanded, split each the value into
00565     // legal parts.
00566     assert(NumParts % NumIntermediates == 0 &&
00567            "Must expand into a divisible number of parts!");
00568     unsigned Factor = NumParts / NumIntermediates;
00569     for (unsigned i = 0; i != NumIntermediates; ++i)
00570       getCopyToParts(DAG, DL, Ops[i], &Parts[i*Factor], Factor, PartVT, V);
00571   }
00572 }
00573 
00574 namespace {
00575   /// RegsForValue - This struct represents the registers (physical or virtual)
00576   /// that a particular set of values is assigned, and the type information
00577   /// about the value. The most common situation is to represent one value at a
00578   /// time, but struct or array values are handled element-wise as multiple
00579   /// values.  The splitting of aggregates is performed recursively, so that we
00580   /// never have aggregate-typed registers. The values at this point do not
00581   /// necessarily have legal types, so each value may require one or more
00582   /// registers of some legal type.
00583   ///
00584   struct RegsForValue {
00585     /// ValueVTs - The value types of the values, which may not be legal, and
00586     /// may need be promoted or synthesized from one or more registers.
00587     ///
00588     SmallVector<EVT, 4> ValueVTs;
00589 
00590     /// RegVTs - The value types of the registers. This is the same size as
00591     /// ValueVTs and it records, for each value, what the type of the assigned
00592     /// register or registers are. (Individual values are never synthesized
00593     /// from more than one type of register.)
00594     ///
00595     /// With virtual registers, the contents of RegVTs is redundant with TLI's
00596     /// getRegisterType member function, however when with physical registers
00597     /// it is necessary to have a separate record of the types.
00598     ///
00599     SmallVector<MVT, 4> RegVTs;
00600 
00601     /// Regs - This list holds the registers assigned to the values.
00602     /// Each legal or promoted value requires one register, and each
00603     /// expanded value requires multiple registers.
00604     ///
00605     SmallVector<unsigned, 4> Regs;
00606 
00607     RegsForValue() {}
00608 
00609     RegsForValue(const SmallVector<unsigned, 4> &regs,
00610                  MVT regvt, EVT valuevt)
00611       : ValueVTs(1, valuevt), RegVTs(1, regvt), Regs(regs) {}
00612 
00613     RegsForValue(LLVMContext &Context, const TargetLowering &tli,
00614                  unsigned Reg, Type *Ty) {
00615       ComputeValueVTs(tli, Ty, ValueVTs);
00616 
00617       for (unsigned Value = 0, e = ValueVTs.size(); Value != e; ++Value) {
00618         EVT ValueVT = ValueVTs[Value];
00619         unsigned NumRegs = tli.getNumRegisters(Context, ValueVT);
00620         MVT RegisterVT = tli.getRegisterType(Context, ValueVT);
00621         for (unsigned i = 0; i != NumRegs; ++i)
00622           Regs.push_back(Reg + i);
00623         RegVTs.push_back(RegisterVT);
00624         Reg += NumRegs;
00625       }
00626     }
00627 
00628     /// append - Add the specified values to this one.
00629     void append(const RegsForValue &RHS) {
00630       ValueVTs.append(RHS.ValueVTs.begin(), RHS.ValueVTs.end());
00631       RegVTs.append(RHS.RegVTs.begin(), RHS.RegVTs.end());
00632       Regs.append(RHS.Regs.begin(), RHS.Regs.end());
00633     }
00634 
00635     /// getCopyFromRegs - Emit a series of CopyFromReg nodes that copies from
00636     /// this value and returns the result as a ValueVTs value.  This uses
00637     /// Chain/Flag as the input and updates them for the output Chain/Flag.
00638     /// If the Flag pointer is NULL, no flag is used.
00639     SDValue getCopyFromRegs(SelectionDAG &DAG, FunctionLoweringInfo &FuncInfo,
00640                             SDLoc dl,
00641                             SDValue &Chain, SDValue *Flag,
00642                             const Value *V = nullptr) const;
00643 
00644     /// getCopyToRegs - Emit a series of CopyToReg nodes that copies the
00645     /// specified value into the registers specified by this object.  This uses
00646     /// Chain/Flag as the input and updates them for the output Chain/Flag.
00647     /// If the Flag pointer is NULL, no flag is used.
00648     void getCopyToRegs(SDValue Val, SelectionDAG &DAG, SDLoc dl,
00649                        SDValue &Chain, SDValue *Flag, const Value *V) const;
00650 
00651     /// AddInlineAsmOperands - Add this value to the specified inlineasm node
00652     /// operand list.  This adds the code marker, matching input operand index
00653     /// (if applicable), and includes the number of values added into it.
00654     void AddInlineAsmOperands(unsigned Kind,
00655                               bool HasMatching, unsigned MatchingIdx,
00656                               SelectionDAG &DAG,
00657                               std::vector<SDValue> &Ops) const;
00658   };
00659 }
00660 
00661 /// getCopyFromRegs - Emit a series of CopyFromReg nodes that copies from
00662 /// this value and returns the result as a ValueVT value.  This uses
00663 /// Chain/Flag as the input and updates them for the output Chain/Flag.
00664 /// If the Flag pointer is NULL, no flag is used.
00665 SDValue RegsForValue::getCopyFromRegs(SelectionDAG &DAG,
00666                                       FunctionLoweringInfo &FuncInfo,
00667                                       SDLoc dl,
00668                                       SDValue &Chain, SDValue *Flag,
00669                                       const Value *V) const {
00670   // A Value with type {} or [0 x %t] needs no registers.
00671   if (ValueVTs.empty())
00672     return SDValue();
00673 
00674   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
00675 
00676   // Assemble the legal parts into the final values.
00677   SmallVector<SDValue, 4> Values(ValueVTs.size());
00678   SmallVector<SDValue, 8> Parts;
00679   for (unsigned Value = 0, Part = 0, e = ValueVTs.size(); Value != e; ++Value) {
00680     // Copy the legal parts from the registers.
00681     EVT ValueVT = ValueVTs[Value];
00682     unsigned NumRegs = TLI.getNumRegisters(*DAG.getContext(), ValueVT);
00683     MVT RegisterVT = RegVTs[Value];
00684 
00685     Parts.resize(NumRegs);
00686     for (unsigned i = 0; i != NumRegs; ++i) {
00687       SDValue P;
00688       if (!Flag) {
00689         P = DAG.getCopyFromReg(Chain, dl, Regs[Part+i], RegisterVT);
00690       } else {
00691         P = DAG.getCopyFromReg(Chain, dl, Regs[Part+i], RegisterVT, *Flag);
00692         *Flag = P.getValue(2);
00693       }
00694 
00695       Chain = P.getValue(1);
00696       Parts[i] = P;
00697 
00698       // If the source register was virtual and if we know something about it,
00699       // add an assert node.
00700       if (!TargetRegisterInfo::isVirtualRegister(Regs[Part+i]) ||
00701           !RegisterVT.isInteger() || RegisterVT.isVector())
00702         continue;
00703 
00704       const FunctionLoweringInfo::LiveOutInfo *LOI =
00705         FuncInfo.GetLiveOutRegInfo(Regs[Part+i]);
00706       if (!LOI)
00707         continue;
00708 
00709       unsigned RegSize = RegisterVT.getSizeInBits();
00710       unsigned NumSignBits = LOI->NumSignBits;
00711       unsigned NumZeroBits = LOI->KnownZero.countLeadingOnes();
00712 
00713       if (NumZeroBits == RegSize) {
00714         // The current value is a zero.
00715         // Explicitly express that as it would be easier for
00716         // optimizations to kick in.
00717         Parts[i] = DAG.getConstant(0, RegisterVT);
00718         continue;
00719       }
00720 
00721       // FIXME: We capture more information than the dag can represent.  For
00722       // now, just use the tightest assertzext/assertsext possible.
00723       bool isSExt = true;
00724       EVT FromVT(MVT::Other);
00725       if (NumSignBits == RegSize)
00726         isSExt = true, FromVT = MVT::i1;   // ASSERT SEXT 1
00727       else if (NumZeroBits >= RegSize-1)
00728         isSExt = false, FromVT = MVT::i1;  // ASSERT ZEXT 1
00729       else if (NumSignBits > RegSize-8)
00730         isSExt = true, FromVT = MVT::i8;   // ASSERT SEXT 8
00731       else if (NumZeroBits >= RegSize-8)
00732         isSExt = false, FromVT = MVT::i8;  // ASSERT ZEXT 8
00733       else if (NumSignBits > RegSize-16)
00734         isSExt = true, FromVT = MVT::i16;  // ASSERT SEXT 16
00735       else if (NumZeroBits >= RegSize-16)
00736         isSExt = false, FromVT = MVT::i16; // ASSERT ZEXT 16
00737       else if (NumSignBits > RegSize-32)
00738         isSExt = true, FromVT = MVT::i32;  // ASSERT SEXT 32
00739       else if (NumZeroBits >= RegSize-32)
00740         isSExt = false, FromVT = MVT::i32; // ASSERT ZEXT 32
00741       else
00742         continue;
00743 
00744       // Add an assertion node.
00745       assert(FromVT != MVT::Other);
00746       Parts[i] = DAG.getNode(isSExt ? ISD::AssertSext : ISD::AssertZext, dl,
00747                              RegisterVT, P, DAG.getValueType(FromVT));
00748     }
00749 
00750     Values[Value] = getCopyFromParts(DAG, dl, Parts.begin(),
00751                                      NumRegs, RegisterVT, ValueVT, V);
00752     Part += NumRegs;
00753     Parts.clear();
00754   }
00755 
00756   return DAG.getNode(ISD::MERGE_VALUES, dl,
00757                      DAG.getVTList(ValueVTs),
00758                      &Values[0], ValueVTs.size());
00759 }
00760 
00761 /// getCopyToRegs - Emit a series of CopyToReg nodes that copies the
00762 /// specified value into the registers specified by this object.  This uses
00763 /// Chain/Flag as the input and updates them for the output Chain/Flag.
00764 /// If the Flag pointer is NULL, no flag is used.
00765 void RegsForValue::getCopyToRegs(SDValue Val, SelectionDAG &DAG, SDLoc dl,
00766                                  SDValue &Chain, SDValue *Flag,
00767                                  const Value *V) const {
00768   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
00769 
00770   // Get the list of the values's legal parts.
00771   unsigned NumRegs = Regs.size();
00772   SmallVector<SDValue, 8> Parts(NumRegs);
00773   for (unsigned Value = 0, Part = 0, e = ValueVTs.size(); Value != e; ++Value) {
00774     EVT ValueVT = ValueVTs[Value];
00775     unsigned NumParts = TLI.getNumRegisters(*DAG.getContext(), ValueVT);
00776     MVT RegisterVT = RegVTs[Value];
00777     ISD::NodeType ExtendKind =
00778       TLI.isZExtFree(Val, RegisterVT)? ISD::ZERO_EXTEND: ISD::ANY_EXTEND;
00779 
00780     getCopyToParts(DAG, dl, Val.getValue(Val.getResNo() + Value),
00781                    &Parts[Part], NumParts, RegisterVT, V, ExtendKind);
00782     Part += NumParts;
00783   }
00784 
00785   // Copy the parts into the registers.
00786   SmallVector<SDValue, 8> Chains(NumRegs);
00787   for (unsigned i = 0; i != NumRegs; ++i) {
00788     SDValue Part;
00789     if (!Flag) {
00790       Part = DAG.getCopyToReg(Chain, dl, Regs[i], Parts[i]);
00791     } else {
00792       Part = DAG.getCopyToReg(Chain, dl, Regs[i], Parts[i], *Flag);
00793       *Flag = Part.getValue(1);
00794     }
00795 
00796     Chains[i] = Part.getValue(0);
00797   }
00798 
00799   if (NumRegs == 1 || Flag)
00800     // If NumRegs > 1 && Flag is used then the use of the last CopyToReg is
00801     // flagged to it. That is the CopyToReg nodes and the user are considered
00802     // a single scheduling unit. If we create a TokenFactor and return it as
00803     // chain, then the TokenFactor is both a predecessor (operand) of the
00804     // user as well as a successor (the TF operands are flagged to the user).
00805     // c1, f1 = CopyToReg
00806     // c2, f2 = CopyToReg
00807     // c3     = TokenFactor c1, c2
00808     // ...
00809     //        = op c3, ..., f2
00810     Chain = Chains[NumRegs-1];
00811   else
00812     Chain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, &Chains[0], NumRegs);
00813 }
00814 
00815 /// AddInlineAsmOperands - Add this value to the specified inlineasm node
00816 /// operand list.  This adds the code marker and includes the number of
00817 /// values added into it.
00818 void RegsForValue::AddInlineAsmOperands(unsigned Code, bool HasMatching,
00819                                         unsigned MatchingIdx,
00820                                         SelectionDAG &DAG,
00821                                         std::vector<SDValue> &Ops) const {
00822   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
00823 
00824   unsigned Flag = InlineAsm::getFlagWord(Code, Regs.size());
00825   if (HasMatching)
00826     Flag = InlineAsm::getFlagWordForMatchingOp(Flag, MatchingIdx);
00827   else if (!Regs.empty() &&
00828            TargetRegisterInfo::isVirtualRegister(Regs.front())) {
00829     // Put the register class of the virtual registers in the flag word.  That
00830     // way, later passes can recompute register class constraints for inline
00831     // assembly as well as normal instructions.
00832     // Don't do this for tied operands that can use the regclass information
00833     // from the def.
00834     const MachineRegisterInfo &MRI = DAG.getMachineFunction().getRegInfo();
00835     const TargetRegisterClass *RC = MRI.getRegClass(Regs.front());
00836     Flag = InlineAsm::getFlagWordForRegClass(Flag, RC->getID());
00837   }
00838 
00839   SDValue Res = DAG.getTargetConstant(Flag, MVT::i32);
00840   Ops.push_back(Res);
00841 
00842   unsigned SP = TLI.getStackPointerRegisterToSaveRestore();
00843   for (unsigned Value = 0, Reg = 0, e = ValueVTs.size(); Value != e; ++Value) {
00844     unsigned NumRegs = TLI.getNumRegisters(*DAG.getContext(), ValueVTs[Value]);
00845     MVT RegisterVT = RegVTs[Value];
00846     for (unsigned i = 0; i != NumRegs; ++i) {
00847       assert(Reg < Regs.size() && "Mismatch in # registers expected");
00848       unsigned TheReg = Regs[Reg++];
00849       Ops.push_back(DAG.getRegister(TheReg, RegisterVT));
00850 
00851       if (TheReg == SP && Code == InlineAsm::Kind_Clobber) {
00852         // If we clobbered the stack pointer, MFI should know about it.
00853         assert(DAG.getMachineFunction().getFrameInfo()->
00854             hasInlineAsmWithSPAdjust());
00855       }
00856     }
00857   }
00858 }
00859 
00860 void SelectionDAGBuilder::init(GCFunctionInfo *gfi, AliasAnalysis &aa,
00861                                const TargetLibraryInfo *li) {
00862   AA = &aa;
00863   GFI = gfi;
00864   LibInfo = li;
00865   DL = DAG.getTarget().getDataLayout();
00866   Context = DAG.getContext();
00867   LPadToCallSiteMap.clear();
00868 }
00869 
00870 /// clear - Clear out the current SelectionDAG and the associated
00871 /// state and prepare this SelectionDAGBuilder object to be used
00872 /// for a new block. This doesn't clear out information about
00873 /// additional blocks that are needed to complete switch lowering
00874 /// or PHI node updating; that information is cleared out as it is
00875 /// consumed.
00876 void SelectionDAGBuilder::clear() {
00877   NodeMap.clear();
00878   UnusedArgNodeMap.clear();
00879   PendingLoads.clear();
00880   PendingExports.clear();
00881   CurInst = nullptr;
00882   HasTailCall = false;
00883   SDNodeOrder = LowestSDNodeOrder;
00884 }
00885 
00886 /// clearDanglingDebugInfo - Clear the dangling debug information
00887 /// map. This function is separated from the clear so that debug
00888 /// information that is dangling in a basic block can be properly
00889 /// resolved in a different basic block. This allows the
00890 /// SelectionDAG to resolve dangling debug information attached
00891 /// to PHI nodes.
00892 void SelectionDAGBuilder::clearDanglingDebugInfo() {
00893   DanglingDebugInfoMap.clear();
00894 }
00895 
00896 /// getRoot - Return the current virtual root of the Selection DAG,
00897 /// flushing any PendingLoad items. This must be done before emitting
00898 /// a store or any other node that may need to be ordered after any
00899 /// prior load instructions.
00900 ///
00901 SDValue SelectionDAGBuilder::getRoot() {
00902   if (PendingLoads.empty())
00903     return DAG.getRoot();
00904 
00905   if (PendingLoads.size() == 1) {
00906     SDValue Root = PendingLoads[0];
00907     DAG.setRoot(Root);
00908     PendingLoads.clear();
00909     return Root;
00910   }
00911 
00912   // Otherwise, we have to make a token factor node.
00913   SDValue Root = DAG.getNode(ISD::TokenFactor, getCurSDLoc(), MVT::Other,
00914                                &PendingLoads[0], PendingLoads.size());
00915   PendingLoads.clear();
00916   DAG.setRoot(Root);
00917   return Root;
00918 }
00919 
00920 /// getControlRoot - Similar to getRoot, but instead of flushing all the
00921 /// PendingLoad items, flush all the PendingExports items. It is necessary
00922 /// to do this before emitting a terminator instruction.
00923 ///
00924 SDValue SelectionDAGBuilder::getControlRoot() {
00925   SDValue Root = DAG.getRoot();
00926 
00927   if (PendingExports.empty())
00928     return Root;
00929 
00930   // Turn all of the CopyToReg chains into one factored node.
00931   if (Root.getOpcode() != ISD::EntryToken) {
00932     unsigned i = 0, e = PendingExports.size();
00933     for (; i != e; ++i) {
00934       assert(PendingExports[i].getNode()->getNumOperands() > 1);
00935       if (PendingExports[i].getNode()->getOperand(0) == Root)
00936         break;  // Don't add the root if we already indirectly depend on it.
00937     }
00938 
00939     if (i == e)
00940       PendingExports.push_back(Root);
00941   }
00942 
00943   Root = DAG.getNode(ISD::TokenFactor, getCurSDLoc(), MVT::Other,
00944                      &PendingExports[0],
00945                      PendingExports.size());
00946   PendingExports.clear();
00947   DAG.setRoot(Root);
00948   return Root;
00949 }
00950 
00951 void SelectionDAGBuilder::visit(const Instruction &I) {
00952   // Set up outgoing PHI node register values before emitting the terminator.
00953   if (isa<TerminatorInst>(&I))
00954     HandlePHINodesInSuccessorBlocks(I.getParent());
00955 
00956   ++SDNodeOrder;
00957 
00958   CurInst = &I;
00959 
00960   visit(I.getOpcode(), I);
00961 
00962   if (!isa<TerminatorInst>(&I) && !HasTailCall)
00963     CopyToExportRegsIfNeeded(&I);
00964 
00965   CurInst = nullptr;
00966 }
00967 
00968 void SelectionDAGBuilder::visitPHI(const PHINode &) {
00969   llvm_unreachable("SelectionDAGBuilder shouldn't visit PHI nodes!");
00970 }
00971 
00972 void SelectionDAGBuilder::visit(unsigned Opcode, const User &I) {
00973   // Note: this doesn't use InstVisitor, because it has to work with
00974   // ConstantExpr's in addition to instructions.
00975   switch (Opcode) {
00976   default: llvm_unreachable("Unknown instruction type encountered!");
00977     // Build the switch statement using the Instruction.def file.
00978 #define HANDLE_INST(NUM, OPCODE, CLASS) \
00979     case Instruction::OPCODE: visit##OPCODE((const CLASS&)I); break;
00980 #include "llvm/IR/Instruction.def"
00981   }
00982 }
00983 
00984 // resolveDanglingDebugInfo - if we saw an earlier dbg_value referring to V,
00985 // generate the debug data structures now that we've seen its definition.
00986 void SelectionDAGBuilder::resolveDanglingDebugInfo(const Value *V,
00987                                                    SDValue Val) {
00988   DanglingDebugInfo &DDI = DanglingDebugInfoMap[V];
00989   if (DDI.getDI()) {
00990     const DbgValueInst *DI = DDI.getDI();
00991     DebugLoc dl = DDI.getdl();
00992     unsigned DbgSDNodeOrder = DDI.getSDNodeOrder();
00993     MDNode *Variable = DI->getVariable();
00994     uint64_t Offset = DI->getOffset();
00995     SDDbgValue *SDV;
00996     if (Val.getNode()) {
00997       if (!EmitFuncArgumentDbgValue(V, Variable, Offset, Val)) {
00998         SDV = DAG.getDbgValue(Variable, Val.getNode(),
00999                               Val.getResNo(), Offset, dl, DbgSDNodeOrder);
01000         DAG.AddDbgValue(SDV, Val.getNode(), false);
01001       }
01002     } else
01003       DEBUG(dbgs() << "Dropping debug info for " << *DI << "\n");
01004     DanglingDebugInfoMap[V] = DanglingDebugInfo();
01005   }
01006 }
01007 
01008 /// getValue - Return an SDValue for the given Value.
01009 SDValue SelectionDAGBuilder::getValue(const Value *V) {
01010   // If we already have an SDValue for this value, use it. It's important
01011   // to do this first, so that we don't create a CopyFromReg if we already
01012   // have a regular SDValue.
01013   SDValue &N = NodeMap[V];
01014   if (N.getNode()) return N;
01015 
01016   // If there's a virtual register allocated and initialized for this
01017   // value, use it.
01018   DenseMap<const Value *, unsigned>::iterator It = FuncInfo.ValueMap.find(V);
01019   if (It != FuncInfo.ValueMap.end()) {
01020     unsigned InReg = It->second;
01021     RegsForValue RFV(*DAG.getContext(), *TM.getTargetLowering(),
01022                      InReg, V->getType());
01023     SDValue Chain = DAG.getEntryNode();
01024     N = RFV.getCopyFromRegs(DAG, FuncInfo, getCurSDLoc(), Chain, nullptr, V);
01025     resolveDanglingDebugInfo(V, N);
01026     return N;
01027   }
01028 
01029   // Otherwise create a new SDValue and remember it.
01030   SDValue Val = getValueImpl(V);
01031   NodeMap[V] = Val;
01032   resolveDanglingDebugInfo(V, Val);
01033   return Val;
01034 }
01035 
01036 /// getNonRegisterValue - Return an SDValue for the given Value, but
01037 /// don't look in FuncInfo.ValueMap for a virtual register.
01038 SDValue SelectionDAGBuilder::getNonRegisterValue(const Value *V) {
01039   // If we already have an SDValue for this value, use it.
01040   SDValue &N = NodeMap[V];
01041   if (N.getNode()) return N;
01042 
01043   // Otherwise create a new SDValue and remember it.
01044   SDValue Val = getValueImpl(V);
01045   NodeMap[V] = Val;
01046   resolveDanglingDebugInfo(V, Val);
01047   return Val;
01048 }
01049 
01050 /// getValueImpl - Helper function for getValue and getNonRegisterValue.
01051 /// Create an SDValue for the given value.
01052 SDValue SelectionDAGBuilder::getValueImpl(const Value *V) {
01053   const TargetLowering *TLI = TM.getTargetLowering();
01054 
01055   if (const Constant *C = dyn_cast<Constant>(V)) {
01056     EVT VT = TLI->getValueType(V->getType(), true);
01057 
01058     if (const ConstantInt *CI = dyn_cast<ConstantInt>(C))
01059       return DAG.getConstant(*CI, VT);
01060 
01061     if (const GlobalValue *GV = dyn_cast<GlobalValue>(C))
01062       return DAG.getGlobalAddress(GV, getCurSDLoc(), VT);
01063 
01064     if (isa<ConstantPointerNull>(C)) {
01065       unsigned AS = V->getType()->getPointerAddressSpace();
01066       return DAG.getConstant(0, TLI->getPointerTy(AS));
01067     }
01068 
01069     if (const ConstantFP *CFP = dyn_cast<ConstantFP>(C))
01070       return DAG.getConstantFP(*CFP, VT);
01071 
01072     if (isa<UndefValue>(C) && !V->getType()->isAggregateType())
01073       return DAG.getUNDEF(VT);
01074 
01075     if (const ConstantExpr *CE = dyn_cast<ConstantExpr>(C)) {
01076       visit(CE->getOpcode(), *CE);
01077       SDValue N1 = NodeMap[V];
01078       assert(N1.getNode() && "visit didn't populate the NodeMap!");
01079       return N1;
01080     }
01081 
01082     if (isa<ConstantStruct>(C) || isa<ConstantArray>(C)) {
01083       SmallVector<SDValue, 4> Constants;
01084       for (User::const_op_iterator OI = C->op_begin(), OE = C->op_end();
01085            OI != OE; ++OI) {
01086         SDNode *Val = getValue(*OI).getNode();
01087         // If the operand is an empty aggregate, there are no values.
01088         if (!Val) continue;
01089         // Add each leaf value from the operand to the Constants list
01090         // to form a flattened list of all the values.
01091         for (unsigned i = 0, e = Val->getNumValues(); i != e; ++i)
01092           Constants.push_back(SDValue(Val, i));
01093       }
01094 
01095       return DAG.getMergeValues(&Constants[0], Constants.size(),
01096                                 getCurSDLoc());
01097     }
01098 
01099     if (const ConstantDataSequential *CDS =
01100           dyn_cast<ConstantDataSequential>(C)) {
01101       SmallVector<SDValue, 4> Ops;
01102       for (unsigned i = 0, e = CDS->getNumElements(); i != e; ++i) {
01103         SDNode *Val = getValue(CDS->getElementAsConstant(i)).getNode();
01104         // Add each leaf value from the operand to the Constants list
01105         // to form a flattened list of all the values.
01106         for (unsigned i = 0, e = Val->getNumValues(); i != e; ++i)
01107           Ops.push_back(SDValue(Val, i));
01108       }
01109 
01110       if (isa<ArrayType>(CDS->getType()))
01111         return DAG.getMergeValues(&Ops[0], Ops.size(), getCurSDLoc());
01112       return NodeMap[V] = DAG.getNode(ISD::BUILD_VECTOR, getCurSDLoc(),
01113                                       VT, &Ops[0], Ops.size());
01114     }
01115 
01116     if (C->getType()->isStructTy() || C->getType()->isArrayTy()) {
01117       assert((isa<ConstantAggregateZero>(C) || isa<UndefValue>(C)) &&
01118              "Unknown struct or array constant!");
01119 
01120       SmallVector<EVT, 4> ValueVTs;
01121       ComputeValueVTs(*TLI, C->getType(), ValueVTs);
01122       unsigned NumElts = ValueVTs.size();
01123       if (NumElts == 0)
01124         return SDValue(); // empty struct
01125       SmallVector<SDValue, 4> Constants(NumElts);
01126       for (unsigned i = 0; i != NumElts; ++i) {
01127         EVT EltVT = ValueVTs[i];
01128         if (isa<UndefValue>(C))
01129           Constants[i] = DAG.getUNDEF(EltVT);
01130         else if (EltVT.isFloatingPoint())
01131           Constants[i] = DAG.getConstantFP(0, EltVT);
01132         else
01133           Constants[i] = DAG.getConstant(0, EltVT);
01134       }
01135 
01136       return DAG.getMergeValues(&Constants[0], NumElts,
01137                                 getCurSDLoc());
01138     }
01139 
01140     if (const BlockAddress *BA = dyn_cast<BlockAddress>(C))
01141       return DAG.getBlockAddress(BA, VT);
01142 
01143     VectorType *VecTy = cast<VectorType>(V->getType());
01144     unsigned NumElements = VecTy->getNumElements();
01145 
01146     // Now that we know the number and type of the elements, get that number of
01147     // elements into the Ops array based on what kind of constant it is.
01148     SmallVector<SDValue, 16> Ops;
01149     if (const ConstantVector *CV = dyn_cast<ConstantVector>(C)) {
01150       for (unsigned i = 0; i != NumElements; ++i)
01151         Ops.push_back(getValue(CV->getOperand(i)));
01152     } else {
01153       assert(isa<ConstantAggregateZero>(C) && "Unknown vector constant!");
01154       EVT EltVT = TLI->getValueType(VecTy->getElementType());
01155 
01156       SDValue Op;
01157       if (EltVT.isFloatingPoint())
01158         Op = DAG.getConstantFP(0, EltVT);
01159       else
01160         Op = DAG.getConstant(0, EltVT);
01161       Ops.assign(NumElements, Op);
01162     }
01163 
01164     // Create a BUILD_VECTOR node.
01165     return NodeMap[V] = DAG.getNode(ISD::BUILD_VECTOR, getCurSDLoc(),
01166                                     VT, &Ops[0], Ops.size());
01167   }
01168 
01169   // If this is a static alloca, generate it as the frameindex instead of
01170   // computation.
01171   if (const AllocaInst *AI = dyn_cast<AllocaInst>(V)) {
01172     DenseMap<const AllocaInst*, int>::iterator SI =
01173       FuncInfo.StaticAllocaMap.find(AI);
01174     if (SI != FuncInfo.StaticAllocaMap.end())
01175       return DAG.getFrameIndex(SI->second, TLI->getPointerTy());
01176   }
01177 
01178   // If this is an instruction which fast-isel has deferred, select it now.
01179   if (const Instruction *Inst = dyn_cast<Instruction>(V)) {
01180     unsigned InReg = FuncInfo.InitializeRegForValue(Inst);
01181     RegsForValue RFV(*DAG.getContext(), *TLI, InReg, Inst->getType());
01182     SDValue Chain = DAG.getEntryNode();
01183     return RFV.getCopyFromRegs(DAG, FuncInfo, getCurSDLoc(), Chain, nullptr, V);
01184   }
01185 
01186   llvm_unreachable("Can't get register for value!");
01187 }
01188 
01189 void SelectionDAGBuilder::visitRet(const ReturnInst &I) {
01190   const TargetLowering *TLI = TM.getTargetLowering();
01191   SDValue Chain = getControlRoot();
01192   SmallVector<ISD::OutputArg, 8> Outs;
01193   SmallVector<SDValue, 8> OutVals;
01194 
01195   if (!FuncInfo.CanLowerReturn) {
01196     unsigned DemoteReg = FuncInfo.DemoteRegister;
01197     const Function *F = I.getParent()->getParent();
01198 
01199     // Emit a store of the return value through the virtual register.
01200     // Leave Outs empty so that LowerReturn won't try to load return
01201     // registers the usual way.
01202     SmallVector<EVT, 1> PtrValueVTs;
01203     ComputeValueVTs(*TLI, PointerType::getUnqual(F->getReturnType()),
01204                     PtrValueVTs);
01205 
01206     SDValue RetPtr = DAG.getRegister(DemoteReg, PtrValueVTs[0]);
01207     SDValue RetOp = getValue(I.getOperand(0));
01208 
01209     SmallVector<EVT, 4> ValueVTs;
01210     SmallVector<uint64_t, 4> Offsets;
01211     ComputeValueVTs(*TLI, I.getOperand(0)->getType(), ValueVTs, &Offsets);
01212     unsigned NumValues = ValueVTs.size();
01213 
01214     SmallVector<SDValue, 4> Chains(NumValues);
01215     for (unsigned i = 0; i != NumValues; ++i) {
01216       SDValue Add = DAG.getNode(ISD::ADD, getCurSDLoc(),
01217                                 RetPtr.getValueType(), RetPtr,
01218                                 DAG.getIntPtrConstant(Offsets[i]));
01219       Chains[i] =
01220         DAG.getStore(Chain, getCurSDLoc(),
01221                      SDValue(RetOp.getNode(), RetOp.getResNo() + i),
01222                      // FIXME: better loc info would be nice.
01223                      Add, MachinePointerInfo(), false, false, 0);
01224     }
01225 
01226     Chain = DAG.getNode(ISD::TokenFactor, getCurSDLoc(),
01227                         MVT::Other, &Chains[0], NumValues);
01228   } else if (I.getNumOperands() != 0) {
01229     SmallVector<EVT, 4> ValueVTs;
01230     ComputeValueVTs(*TLI, I.getOperand(0)->getType(), ValueVTs);
01231     unsigned NumValues = ValueVTs.size();
01232     if (NumValues) {
01233       SDValue RetOp = getValue(I.getOperand(0));
01234       for (unsigned j = 0, f = NumValues; j != f; ++j) {
01235         EVT VT = ValueVTs[j];
01236 
01237         ISD::NodeType ExtendKind = ISD::ANY_EXTEND;
01238 
01239         const Function *F = I.getParent()->getParent();
01240         if (F->getAttributes().hasAttribute(AttributeSet::ReturnIndex,
01241                                             Attribute::SExt))
01242           ExtendKind = ISD::SIGN_EXTEND;
01243         else if (F->getAttributes().hasAttribute(AttributeSet::ReturnIndex,
01244                                                  Attribute::ZExt))
01245           ExtendKind = ISD::ZERO_EXTEND;
01246 
01247         if (ExtendKind != ISD::ANY_EXTEND && VT.isInteger())
01248           VT = TLI->getTypeForExtArgOrReturn(VT.getSimpleVT(), ExtendKind);
01249 
01250         unsigned NumParts = TLI->getNumRegisters(*DAG.getContext(), VT);
01251         MVT PartVT = TLI->getRegisterType(*DAG.getContext(), VT);
01252         SmallVector<SDValue, 4> Parts(NumParts);
01253         getCopyToParts(DAG, getCurSDLoc(),
01254                        SDValue(RetOp.getNode(), RetOp.getResNo() + j),
01255                        &Parts[0], NumParts, PartVT, &I, ExtendKind);
01256 
01257         // 'inreg' on function refers to return value
01258         ISD::ArgFlagsTy Flags = ISD::ArgFlagsTy();
01259         if (F->getAttributes().hasAttribute(AttributeSet::ReturnIndex,
01260                                             Attribute::InReg))
01261           Flags.setInReg();
01262 
01263         // Propagate extension type if any
01264         if (ExtendKind == ISD::SIGN_EXTEND)
01265           Flags.setSExt();
01266         else if (ExtendKind == ISD::ZERO_EXTEND)
01267           Flags.setZExt();
01268 
01269         for (unsigned i = 0; i < NumParts; ++i) {
01270           Outs.push_back(ISD::OutputArg(Flags, Parts[i].getValueType(),
01271                                         VT, /*isfixed=*/true, 0, 0));
01272           OutVals.push_back(Parts[i]);
01273         }
01274       }
01275     }
01276   }
01277 
01278   bool isVarArg = DAG.getMachineFunction().getFunction()->isVarArg();
01279   CallingConv::ID CallConv =
01280     DAG.getMachineFunction().getFunction()->getCallingConv();
01281   Chain = TM.getTargetLowering()->LowerReturn(Chain, CallConv, isVarArg,
01282                                               Outs, OutVals, getCurSDLoc(),
01283                                               DAG);
01284 
01285   // Verify that the target's LowerReturn behaved as expected.
01286   assert(Chain.getNode() && Chain.getValueType() == MVT::Other &&
01287          "LowerReturn didn't return a valid chain!");
01288 
01289   // Update the DAG with the new chain value resulting from return lowering.
01290   DAG.setRoot(Chain);
01291 }
01292 
01293 /// CopyToExportRegsIfNeeded - If the given value has virtual registers
01294 /// created for it, emit nodes to copy the value into the virtual
01295 /// registers.
01296 void SelectionDAGBuilder::CopyToExportRegsIfNeeded(const Value *V) {
01297   // Skip empty types
01298   if (V->getType()->isEmptyTy())
01299     return;
01300 
01301   DenseMap<const Value *, unsigned>::iterator VMI = FuncInfo.ValueMap.find(V);
01302   if (VMI != FuncInfo.ValueMap.end()) {
01303     assert(!V->use_empty() && "Unused value assigned virtual registers!");
01304     CopyValueToVirtualRegister(V, VMI->second);
01305   }
01306 }
01307 
01308 /// ExportFromCurrentBlock - If this condition isn't known to be exported from
01309 /// the current basic block, add it to ValueMap now so that we'll get a
01310 /// CopyTo/FromReg.
01311 void SelectionDAGBuilder::ExportFromCurrentBlock(const Value *V) {
01312   // No need to export constants.
01313   if (!isa<Instruction>(V) && !isa<Argument>(V)) return;
01314 
01315   // Already exported?
01316   if (FuncInfo.isExportedInst(V)) return;
01317 
01318   unsigned Reg = FuncInfo.InitializeRegForValue(V);
01319   CopyValueToVirtualRegister(V, Reg);
01320 }
01321 
01322 bool SelectionDAGBuilder::isExportableFromCurrentBlock(const Value *V,
01323                                                      const BasicBlock *FromBB) {
01324   // The operands of the setcc have to be in this block.  We don't know
01325   // how to export them from some other block.
01326   if (const Instruction *VI = dyn_cast<Instruction>(V)) {
01327     // Can export from current BB.
01328     if (VI->getParent() == FromBB)
01329       return true;
01330 
01331     // Is already exported, noop.
01332     return FuncInfo.isExportedInst(V);
01333   }
01334 
01335   // If this is an argument, we can export it if the BB is the entry block or
01336   // if it is already exported.
01337   if (isa<Argument>(V)) {
01338     if (FromBB == &FromBB->getParent()->getEntryBlock())
01339       return true;
01340 
01341     // Otherwise, can only export this if it is already exported.
01342     return FuncInfo.isExportedInst(V);
01343   }
01344 
01345   // Otherwise, constants can always be exported.
01346   return true;
01347 }
01348 
01349 /// Return branch probability calculated by BranchProbabilityInfo for IR blocks.
01350 uint32_t SelectionDAGBuilder::getEdgeWeight(const MachineBasicBlock *Src,
01351                                             const MachineBasicBlock *Dst) const {
01352   BranchProbabilityInfo *BPI = FuncInfo.BPI;
01353   if (!BPI)
01354     return 0;
01355   const BasicBlock *SrcBB = Src->getBasicBlock();
01356   const BasicBlock *DstBB = Dst->getBasicBlock();
01357   return BPI->getEdgeWeight(SrcBB, DstBB);
01358 }
01359 
01360 void SelectionDAGBuilder::
01361 addSuccessorWithWeight(MachineBasicBlock *Src, MachineBasicBlock *Dst,
01362                        uint32_t Weight /* = 0 */) {
01363   if (!Weight)
01364     Weight = getEdgeWeight(Src, Dst);
01365   Src->addSuccessor(Dst, Weight);
01366 }
01367 
01368 
01369 static bool InBlock(const Value *V, const BasicBlock *BB) {
01370   if (const Instruction *I = dyn_cast<Instruction>(V))
01371     return I->getParent() == BB;
01372   return true;
01373 }
01374 
01375 /// EmitBranchForMergedCondition - Helper method for FindMergedConditions.
01376 /// This function emits a branch and is used at the leaves of an OR or an
01377 /// AND operator tree.
01378 ///
01379 void
01380 SelectionDAGBuilder::EmitBranchForMergedCondition(const Value *Cond,
01381                                                   MachineBasicBlock *TBB,
01382                                                   MachineBasicBlock *FBB,
01383                                                   MachineBasicBlock *CurBB,
01384                                                   MachineBasicBlock *SwitchBB,
01385                                                   uint32_t TWeight,
01386                                                   uint32_t FWeight) {
01387   const BasicBlock *BB = CurBB->getBasicBlock();
01388 
01389   // If the leaf of the tree is a comparison, merge the condition into
01390   // the caseblock.
01391   if (const CmpInst *BOp = dyn_cast<CmpInst>(Cond)) {
01392     // The operands of the cmp have to be in this block.  We don't know
01393     // how to export them from some other block.  If this is the first block
01394     // of the sequence, no exporting is needed.
01395     if (CurBB == SwitchBB ||
01396         (isExportableFromCurrentBlock(BOp->getOperand(0), BB) &&
01397          isExportableFromCurrentBlock(BOp->getOperand(1), BB))) {
01398       ISD::CondCode Condition;
01399       if (const ICmpInst *IC = dyn_cast<ICmpInst>(Cond)) {
01400         Condition = getICmpCondCode(IC->getPredicate());
01401       } else if (const FCmpInst *FC = dyn_cast<FCmpInst>(Cond)) {
01402         Condition = getFCmpCondCode(FC->getPredicate());
01403         if (TM.Options.NoNaNsFPMath)
01404           Condition = getFCmpCodeWithoutNaN(Condition);
01405       } else {
01406         Condition = ISD::SETEQ; // silence warning.
01407         llvm_unreachable("Unknown compare instruction");
01408       }
01409 
01410       CaseBlock CB(Condition, BOp->getOperand(0), BOp->getOperand(1), nullptr,
01411                    TBB, FBB, CurBB, TWeight, FWeight);
01412       SwitchCases.push_back(CB);
01413       return;
01414     }
01415   }
01416 
01417   // Create a CaseBlock record representing this branch.
01418   CaseBlock CB(ISD::SETEQ, Cond, ConstantInt::getTrue(*DAG.getContext()),
01419                nullptr, TBB, FBB, CurBB, TWeight, FWeight);
01420   SwitchCases.push_back(CB);
01421 }
01422 
01423 /// Scale down both weights to fit into uint32_t.
01424 static void ScaleWeights(uint64_t &NewTrue, uint64_t &NewFalse) {
01425   uint64_t NewMax = (NewTrue > NewFalse) ? NewTrue : NewFalse;
01426   uint32_t Scale = (NewMax / UINT32_MAX) + 1;
01427   NewTrue = NewTrue / Scale;
01428   NewFalse = NewFalse / Scale;
01429 }
01430 
01431 /// FindMergedConditions - If Cond is an expression like
01432 void SelectionDAGBuilder::FindMergedConditions(const Value *Cond,
01433                                                MachineBasicBlock *TBB,
01434                                                MachineBasicBlock *FBB,
01435                                                MachineBasicBlock *CurBB,
01436                                                MachineBasicBlock *SwitchBB,
01437                                                unsigned Opc, uint32_t TWeight,
01438                                                uint32_t FWeight) {
01439   // If this node is not part of the or/and tree, emit it as a branch.
01440   const Instruction *BOp = dyn_cast<Instruction>(Cond);
01441   if (!BOp || !(isa<BinaryOperator>(BOp) || isa<CmpInst>(BOp)) ||
01442       (unsigned)BOp->getOpcode() != Opc || !BOp->hasOneUse() ||
01443       BOp->getParent() != CurBB->getBasicBlock() ||
01444       !InBlock(BOp->getOperand(0), CurBB->getBasicBlock()) ||
01445       !InBlock(BOp->getOperand(1), CurBB->getBasicBlock())) {
01446     EmitBranchForMergedCondition(Cond, TBB, FBB, CurBB, SwitchBB,
01447                                  TWeight, FWeight);
01448     return;
01449   }
01450 
01451   //  Create TmpBB after CurBB.
01452   MachineFunction::iterator BBI = CurBB;
01453   MachineFunction &MF = DAG.getMachineFunction();
01454   MachineBasicBlock *TmpBB = MF.CreateMachineBasicBlock(CurBB->getBasicBlock());
01455   CurBB->getParent()->insert(++BBI, TmpBB);
01456 
01457   if (Opc == Instruction::Or) {
01458     // Codegen X | Y as:
01459     // BB1:
01460     //   jmp_if_X TBB
01461     //   jmp TmpBB
01462     // TmpBB:
01463     //   jmp_if_Y TBB
01464     //   jmp FBB
01465     //
01466 
01467     // We have flexibility in setting Prob for BB1 and Prob for TmpBB.
01468     // The requirement is that
01469     //   TrueProb for BB1 + (FalseProb for BB1 * TrueProb for TmpBB)
01470     //     = TrueProb for orignal BB.
01471     // Assuming the orignal weights are A and B, one choice is to set BB1's
01472     // weights to A and A+2B, and set TmpBB's weights to A and 2B. This choice
01473     // assumes that
01474     //   TrueProb for BB1 == FalseProb for BB1 * TrueProb for TmpBB.
01475     // Another choice is to assume TrueProb for BB1 equals to TrueProb for
01476     // TmpBB, but the math is more complicated.
01477 
01478     uint64_t NewTrueWeight = TWeight;
01479     uint64_t NewFalseWeight = (uint64_t)TWeight + 2 * (uint64_t)FWeight;
01480     ScaleWeights(NewTrueWeight, NewFalseWeight);
01481     // Emit the LHS condition.
01482     FindMergedConditions(BOp->getOperand(0), TBB, TmpBB, CurBB, SwitchBB, Opc,
01483                          NewTrueWeight, NewFalseWeight);
01484 
01485     NewTrueWeight = TWeight;
01486     NewFalseWeight = 2 * (uint64_t)FWeight;
01487     ScaleWeights(NewTrueWeight, NewFalseWeight);
01488     // Emit the RHS condition into TmpBB.
01489     FindMergedConditions(BOp->getOperand(1), TBB, FBB, TmpBB, SwitchBB, Opc,
01490                          NewTrueWeight, NewFalseWeight);
01491   } else {
01492     assert(Opc == Instruction::And && "Unknown merge op!");
01493     // Codegen X & Y as:
01494     // BB1:
01495     //   jmp_if_X TmpBB
01496     //   jmp FBB
01497     // TmpBB:
01498     //   jmp_if_Y TBB
01499     //   jmp FBB
01500     //
01501     //  This requires creation of TmpBB after CurBB.
01502 
01503     // We have flexibility in setting Prob for BB1 and Prob for TmpBB.
01504     // The requirement is that
01505     //   FalseProb for BB1 + (TrueProb for BB1 * FalseProb for TmpBB)
01506     //     = FalseProb for orignal BB.
01507     // Assuming the orignal weights are A and B, one choice is to set BB1's
01508     // weights to 2A+B and B, and set TmpBB's weights to 2A and B. This choice
01509     // assumes that
01510     //   FalseProb for BB1 == TrueProb for BB1 * FalseProb for TmpBB.
01511 
01512     uint64_t NewTrueWeight = 2 * (uint64_t)TWeight + (uint64_t)FWeight;
01513     uint64_t NewFalseWeight = FWeight;
01514     ScaleWeights(NewTrueWeight, NewFalseWeight);
01515     // Emit the LHS condition.
01516     FindMergedConditions(BOp->getOperand(0), TmpBB, FBB, CurBB, SwitchBB, Opc,
01517                          NewTrueWeight, NewFalseWeight);
01518 
01519     NewTrueWeight = 2 * (uint64_t)TWeight;
01520     NewFalseWeight = FWeight;
01521     ScaleWeights(NewTrueWeight, NewFalseWeight);
01522     // Emit the RHS condition into TmpBB.
01523     FindMergedConditions(BOp->getOperand(1), TBB, FBB, TmpBB, SwitchBB, Opc,
01524                          NewTrueWeight, NewFalseWeight);
01525   }
01526 }
01527 
01528 /// If the set of cases should be emitted as a series of branches, return true.
01529 /// If we should emit this as a bunch of and/or'd together conditions, return
01530 /// false.
01531 bool
01532 SelectionDAGBuilder::ShouldEmitAsBranches(const std::vector<CaseBlock> &Cases) {
01533   if (Cases.size() != 2) return true;
01534 
01535   // If this is two comparisons of the same values or'd or and'd together, they
01536   // will get folded into a single comparison, so don't emit two blocks.
01537   if ((Cases[0].CmpLHS == Cases[1].CmpLHS &&
01538        Cases[0].CmpRHS == Cases[1].CmpRHS) ||
01539       (Cases[0].CmpRHS == Cases[1].CmpLHS &&
01540        Cases[0].CmpLHS == Cases[1].CmpRHS)) {
01541     return false;
01542   }
01543 
01544   // Handle: (X != null) | (Y != null) --> (X|Y) != 0
01545   // Handle: (X == null) & (Y == null) --> (X|Y) == 0
01546   if (Cases[0].CmpRHS == Cases[1].CmpRHS &&
01547       Cases[0].CC == Cases[1].CC &&
01548       isa<Constant>(Cases[0].CmpRHS) &&
01549       cast<Constant>(Cases[0].CmpRHS)->isNullValue()) {
01550     if (Cases[0].CC == ISD::SETEQ && Cases[0].TrueBB == Cases[1].ThisBB)
01551       return false;
01552     if (Cases[0].CC == ISD::SETNE && Cases[0].FalseBB == Cases[1].ThisBB)
01553       return false;
01554   }
01555 
01556   return true;
01557 }
01558 
01559 void SelectionDAGBuilder::visitBr(const BranchInst &I) {
01560   MachineBasicBlock *BrMBB = FuncInfo.MBB;
01561 
01562   // Update machine-CFG edges.
01563   MachineBasicBlock *Succ0MBB = FuncInfo.MBBMap[I.getSuccessor(0)];
01564 
01565   // Figure out which block is immediately after the current one.
01566   MachineBasicBlock *NextBlock = nullptr;
01567   MachineFunction::iterator BBI = BrMBB;
01568   if (++BBI != FuncInfo.MF->end())
01569     NextBlock = BBI;
01570 
01571   if (I.isUnconditional()) {
01572     // Update machine-CFG edges.
01573     BrMBB->addSuccessor(Succ0MBB);
01574 
01575     // If this is not a fall-through branch or optimizations are switched off,
01576     // emit the branch.
01577     if (Succ0MBB != NextBlock || TM.getOptLevel() == CodeGenOpt::None)
01578       DAG.setRoot(DAG.getNode(ISD::BR, getCurSDLoc(),
01579                               MVT::Other, getControlRoot(),
01580                               DAG.getBasicBlock(Succ0MBB)));
01581 
01582     return;
01583   }
01584 
01585   // If this condition is one of the special cases we handle, do special stuff
01586   // now.
01587   const Value *CondVal = I.getCondition();
01588   MachineBasicBlock *Succ1MBB = FuncInfo.MBBMap[I.getSuccessor(1)];
01589 
01590   // If this is a series of conditions that are or'd or and'd together, emit
01591   // this as a sequence of branches instead of setcc's with and/or operations.
01592   // As long as jumps are not expensive, this should improve performance.
01593   // For example, instead of something like:
01594   //     cmp A, B
01595   //     C = seteq
01596   //     cmp D, E
01597   //     F = setle
01598   //     or C, F
01599   //     jnz foo
01600   // Emit:
01601   //     cmp A, B
01602   //     je foo
01603   //     cmp D, E
01604   //     jle foo
01605   //
01606   if (const BinaryOperator *BOp = dyn_cast<BinaryOperator>(CondVal)) {
01607     if (!TM.getTargetLowering()->isJumpExpensive() &&
01608         BOp->hasOneUse() &&
01609         (BOp->getOpcode() == Instruction::And ||
01610          BOp->getOpcode() == Instruction::Or)) {
01611       FindMergedConditions(BOp, Succ0MBB, Succ1MBB, BrMBB, BrMBB,
01612                            BOp->getOpcode(), getEdgeWeight(BrMBB, Succ0MBB),
01613                            getEdgeWeight(BrMBB, Succ1MBB));
01614       // If the compares in later blocks need to use values not currently
01615       // exported from this block, export them now.  This block should always
01616       // be the first entry.
01617       assert(SwitchCases[0].ThisBB == BrMBB && "Unexpected lowering!");
01618 
01619       // Allow some cases to be rejected.
01620       if (ShouldEmitAsBranches(SwitchCases)) {
01621         for (unsigned i = 1, e = SwitchCases.size(); i != e; ++i) {
01622           ExportFromCurrentBlock(SwitchCases[i].CmpLHS);
01623           ExportFromCurrentBlock(SwitchCases[i].CmpRHS);
01624         }
01625 
01626         // Emit the branch for this block.
01627         visitSwitchCase(SwitchCases[0], BrMBB);
01628         SwitchCases.erase(SwitchCases.begin());
01629         return;
01630       }
01631 
01632       // Okay, we decided not to do this, remove any inserted MBB's and clear
01633       // SwitchCases.
01634       for (unsigned i = 1, e = SwitchCases.size(); i != e; ++i)
01635         FuncInfo.MF->erase(SwitchCases[i].ThisBB);
01636 
01637       SwitchCases.clear();
01638     }
01639   }
01640 
01641   // Create a CaseBlock record representing this branch.
01642   CaseBlock CB(ISD::SETEQ, CondVal, ConstantInt::getTrue(*DAG.getContext()),
01643                nullptr, Succ0MBB, Succ1MBB, BrMBB);
01644 
01645   // Use visitSwitchCase to actually insert the fast branch sequence for this
01646   // cond branch.
01647   visitSwitchCase(CB, BrMBB);
01648 }
01649 
01650 /// visitSwitchCase - Emits the necessary code to represent a single node in
01651 /// the binary search tree resulting from lowering a switch instruction.
01652 void SelectionDAGBuilder::visitSwitchCase(CaseBlock &CB,
01653                                           MachineBasicBlock *SwitchBB) {
01654   SDValue Cond;
01655   SDValue CondLHS = getValue(CB.CmpLHS);
01656   SDLoc dl = getCurSDLoc();
01657 
01658   // Build the setcc now.
01659   if (!CB.CmpMHS) {
01660     // Fold "(X == true)" to X and "(X == false)" to !X to
01661     // handle common cases produced by branch lowering.
01662     if (CB.CmpRHS == ConstantInt::getTrue(*DAG.getContext()) &&
01663         CB.CC == ISD::SETEQ)
01664       Cond = CondLHS;
01665     else if (CB.CmpRHS == ConstantInt::getFalse(*DAG.getContext()) &&
01666              CB.CC == ISD::SETEQ) {
01667       SDValue True = DAG.getConstant(1, CondLHS.getValueType());
01668       Cond = DAG.getNode(ISD::XOR, dl, CondLHS.getValueType(), CondLHS, True);
01669     } else
01670       Cond = DAG.getSetCC(dl, MVT::i1, CondLHS, getValue(CB.CmpRHS), CB.CC);
01671   } else {
01672     assert(CB.CC == ISD::SETLE && "Can handle only LE ranges now");
01673 
01674     const APInt& Low = cast<ConstantInt>(CB.CmpLHS)->getValue();
01675     const APInt& High  = cast<ConstantInt>(CB.CmpRHS)->getValue();
01676 
01677     SDValue CmpOp = getValue(CB.CmpMHS);
01678     EVT VT = CmpOp.getValueType();
01679 
01680     if (cast<ConstantInt>(CB.CmpLHS)->isMinValue(true)) {
01681       Cond = DAG.getSetCC(dl, MVT::i1, CmpOp, DAG.getConstant(High, VT),
01682                           ISD::SETLE);
01683     } else {
01684       SDValue SUB = DAG.getNode(ISD::SUB, dl,
01685                                 VT, CmpOp, DAG.getConstant(Low, VT));
01686       Cond = DAG.getSetCC(dl, MVT::i1, SUB,
01687                           DAG.getConstant(High-Low, VT), ISD::SETULE);
01688     }
01689   }
01690 
01691   // Update successor info
01692   addSuccessorWithWeight(SwitchBB, CB.TrueBB, CB.TrueWeight);
01693   // TrueBB and FalseBB are always different unless the incoming IR is
01694   // degenerate. This only happens when running llc on weird IR.
01695   if (CB.TrueBB != CB.FalseBB)
01696     addSuccessorWithWeight(SwitchBB, CB.FalseBB, CB.FalseWeight);
01697 
01698   // Set NextBlock to be the MBB immediately after the current one, if any.
01699   // This is used to avoid emitting unnecessary branches to the next block.
01700   MachineBasicBlock *NextBlock = nullptr;
01701   MachineFunction::iterator BBI = SwitchBB;
01702   if (++BBI != FuncInfo.MF->end())
01703     NextBlock = BBI;
01704 
01705   // If the lhs block is the next block, invert the condition so that we can
01706   // fall through to the lhs instead of the rhs block.
01707   if (CB.TrueBB == NextBlock) {
01708     std::swap(CB.TrueBB, CB.FalseBB);
01709     SDValue True = DAG.getConstant(1, Cond.getValueType());
01710     Cond = DAG.getNode(ISD::XOR, dl, Cond.getValueType(), Cond, True);
01711   }
01712 
01713   SDValue BrCond = DAG.getNode(ISD::BRCOND, dl,
01714                                MVT::Other, getControlRoot(), Cond,
01715                                DAG.getBasicBlock(CB.TrueBB));
01716 
01717   // Insert the false branch. Do this even if it's a fall through branch,
01718   // this makes it easier to do DAG optimizations which require inverting
01719   // the branch condition.
01720   BrCond = DAG.getNode(ISD::BR, dl, MVT::Other, BrCond,
01721                        DAG.getBasicBlock(CB.FalseBB));
01722 
01723   DAG.setRoot(BrCond);
01724 }
01725 
01726 /// visitJumpTable - Emit JumpTable node in the current MBB
01727 void SelectionDAGBuilder::visitJumpTable(JumpTable &JT) {
01728   // Emit the code for the jump table
01729   assert(JT.Reg != -1U && "Should lower JT Header first!");
01730   EVT PTy = TM.getTargetLowering()->getPointerTy();
01731   SDValue Index = DAG.getCopyFromReg(getControlRoot(), getCurSDLoc(),
01732                                      JT.Reg, PTy);
01733   SDValue Table = DAG.getJumpTable(JT.JTI, PTy);
01734   SDValue BrJumpTable = DAG.getNode(ISD::BR_JT, getCurSDLoc(),
01735                                     MVT::Other, Index.getValue(1),
01736                                     Table, Index);
01737   DAG.setRoot(BrJumpTable);
01738 }
01739 
01740 /// visitJumpTableHeader - This function emits necessary code to produce index
01741 /// in the JumpTable from switch case.
01742 void SelectionDAGBuilder::visitJumpTableHeader(JumpTable &JT,
01743                                                JumpTableHeader &JTH,
01744                                                MachineBasicBlock *SwitchBB) {
01745   // Subtract the lowest switch case value from the value being switched on and
01746   // conditional branch to default mbb if the result is greater than the
01747   // difference between smallest and largest cases.
01748   SDValue SwitchOp = getValue(JTH.SValue);
01749   EVT VT = SwitchOp.getValueType();
01750   SDValue Sub = DAG.getNode(ISD::SUB, getCurSDLoc(), VT, SwitchOp,
01751                             DAG.getConstant(JTH.First, VT));
01752 
01753   // The SDNode we just created, which holds the value being switched on minus
01754   // the smallest case value, needs to be copied to a virtual register so it
01755   // can be used as an index into the jump table in a subsequent basic block.
01756   // This value may be smaller or larger than the target's pointer type, and
01757   // therefore require extension or truncating.
01758   const TargetLowering *TLI = TM.getTargetLowering();
01759   SwitchOp = DAG.getZExtOrTrunc(Sub, getCurSDLoc(), TLI->getPointerTy());
01760 
01761   unsigned JumpTableReg = FuncInfo.CreateReg(TLI->getPointerTy());
01762   SDValue CopyTo = DAG.getCopyToReg(getControlRoot(), getCurSDLoc(),
01763                                     JumpTableReg, SwitchOp);
01764   JT.Reg = JumpTableReg;
01765 
01766   // Emit the range check for the jump table, and branch to the default block
01767   // for the switch statement if the value being switched on exceeds the largest
01768   // case in the switch.
01769   SDValue CMP = DAG.getSetCC(getCurSDLoc(),
01770                              TLI->getSetCCResultType(*DAG.getContext(),
01771                                                      Sub.getValueType()),
01772                              Sub,
01773                              DAG.getConstant(JTH.Last - JTH.First,VT),
01774                              ISD::SETUGT);
01775 
01776   // Set NextBlock to be the MBB immediately after the current one, if any.
01777   // This is used to avoid emitting unnecessary branches to the next block.
01778   MachineBasicBlock *NextBlock = nullptr;
01779   MachineFunction::iterator BBI = SwitchBB;
01780 
01781   if (++BBI != FuncInfo.MF->end())
01782     NextBlock = BBI;
01783 
01784   SDValue BrCond = DAG.getNode(ISD::BRCOND, getCurSDLoc(),
01785                                MVT::Other, CopyTo, CMP,
01786                                DAG.getBasicBlock(JT.Default));
01787 
01788   if (JT.MBB != NextBlock)
01789     BrCond = DAG.getNode(ISD::BR, getCurSDLoc(), MVT::Other, BrCond,
01790                          DAG.getBasicBlock(JT.MBB));
01791 
01792   DAG.setRoot(BrCond);
01793 }
01794 
01795 /// Codegen a new tail for a stack protector check ParentMBB which has had its
01796 /// tail spliced into a stack protector check success bb.
01797 ///
01798 /// For a high level explanation of how this fits into the stack protector
01799 /// generation see the comment on the declaration of class
01800 /// StackProtectorDescriptor.
01801 void SelectionDAGBuilder::visitSPDescriptorParent(StackProtectorDescriptor &SPD,
01802                                                   MachineBasicBlock *ParentBB) {
01803 
01804   // First create the loads to the guard/stack slot for the comparison.
01805   const TargetLowering *TLI = TM.getTargetLowering();
01806   EVT PtrTy = TLI->getPointerTy();
01807 
01808   MachineFrameInfo *MFI = ParentBB->getParent()->getFrameInfo();
01809   int FI = MFI->getStackProtectorIndex();
01810 
01811   const Value *IRGuard = SPD.getGuard();
01812   SDValue GuardPtr = getValue(IRGuard);
01813   SDValue StackSlotPtr = DAG.getFrameIndex(FI, PtrTy);
01814 
01815   unsigned Align =
01816     TLI->getDataLayout()->getPrefTypeAlignment(IRGuard->getType());
01817   SDValue Guard = DAG.getLoad(PtrTy, getCurSDLoc(), DAG.getEntryNode(),
01818                               GuardPtr, MachinePointerInfo(IRGuard, 0),
01819                               true, false, false, Align);
01820 
01821   SDValue StackSlot = DAG.getLoad(PtrTy, getCurSDLoc(), DAG.getEntryNode(),
01822                                   StackSlotPtr,
01823                                   MachinePointerInfo::getFixedStack(FI),
01824                                   true, false, false, Align);
01825 
01826   // Perform the comparison via a subtract/getsetcc.
01827   EVT VT = Guard.getValueType();
01828   SDValue Sub = DAG.getNode(ISD::SUB, getCurSDLoc(), VT, Guard, StackSlot);
01829 
01830   SDValue Cmp = DAG.getSetCC(getCurSDLoc(),
01831                              TLI->getSetCCResultType(*DAG.getContext(),
01832                                                      Sub.getValueType()),
01833                              Sub, DAG.getConstant(0, VT),
01834                              ISD::SETNE);
01835 
01836   // If the sub is not 0, then we know the guard/stackslot do not equal, so
01837   // branch to failure MBB.
01838   SDValue BrCond = DAG.getNode(ISD::BRCOND, getCurSDLoc(),
01839                                MVT::Other, StackSlot.getOperand(0),
01840                                Cmp, DAG.getBasicBlock(SPD.getFailureMBB()));
01841   // Otherwise branch to success MBB.
01842   SDValue Br = DAG.getNode(ISD::BR, getCurSDLoc(),
01843                            MVT::Other, BrCond,
01844                            DAG.getBasicBlock(SPD.getSuccessMBB()));
01845 
01846   DAG.setRoot(Br);
01847 }
01848 
01849 /// Codegen the failure basic block for a stack protector check.
01850 ///
01851 /// A failure stack protector machine basic block consists simply of a call to
01852 /// __stack_chk_fail().
01853 ///
01854 /// For a high level explanation of how this fits into the stack protector
01855 /// generation see the comment on the declaration of class
01856 /// StackProtectorDescriptor.
01857 void
01858 SelectionDAGBuilder::visitSPDescriptorFailure(StackProtectorDescriptor &SPD) {
01859   const TargetLowering *TLI = TM.getTargetLowering();
01860   SDValue Chain = TLI->makeLibCall(DAG, RTLIB::STACKPROTECTOR_CHECK_FAIL,
01861                                    MVT::isVoid, nullptr, 0, false,
01862                                    getCurSDLoc(), false, false).second;
01863   DAG.setRoot(Chain);
01864 }
01865 
01866 /// visitBitTestHeader - This function emits necessary code to produce value
01867 /// suitable for "bit tests"
01868 void SelectionDAGBuilder::visitBitTestHeader(BitTestBlock &B,
01869                                              MachineBasicBlock *SwitchBB) {
01870   // Subtract the minimum value
01871   SDValue SwitchOp = getValue(B.SValue);
01872   EVT VT = SwitchOp.getValueType();
01873   SDValue Sub = DAG.getNode(ISD::SUB, getCurSDLoc(), VT, SwitchOp,
01874                             DAG.getConstant(B.First, VT));
01875 
01876   // Check range
01877   const TargetLowering *TLI = TM.getTargetLowering();
01878   SDValue RangeCmp = DAG.getSetCC(getCurSDLoc(),
01879                                   TLI->getSetCCResultType(*DAG.getContext(),
01880                                                          Sub.getValueType()),
01881                                   Sub, DAG.getConstant(B.Range, VT),
01882                                   ISD::SETUGT);
01883 
01884   // Determine the type of the test operands.
01885   bool UsePtrType = false;
01886   if (!TLI->isTypeLegal(VT))
01887     UsePtrType = true;
01888   else {
01889     for (unsigned i = 0, e = B.Cases.size(); i != e; ++i)
01890       if (!isUIntN(VT.getSizeInBits(), B.Cases[i].Mask)) {
01891         // Switch table case range are encoded into series of masks.
01892         // Just use pointer type, it's guaranteed to fit.
01893         UsePtrType = true;
01894         break;
01895       }
01896   }
01897   if (UsePtrType) {
01898     VT = TLI->getPointerTy();
01899     Sub = DAG.getZExtOrTrunc(Sub, getCurSDLoc(), VT);
01900   }
01901 
01902   B.RegVT = VT.getSimpleVT();
01903   B.Reg = FuncInfo.CreateReg(B.RegVT);
01904   SDValue CopyTo = DAG.getCopyToReg(getControlRoot(), getCurSDLoc(),
01905                                     B.Reg, Sub);
01906 
01907   // Set NextBlock to be the MBB immediately after the current one, if any.
01908   // This is used to avoid emitting unnecessary branches to the next block.
01909   MachineBasicBlock *NextBlock = nullptr;
01910   MachineFunction::iterator BBI = SwitchBB;
01911   if (++BBI != FuncInfo.MF->end())
01912     NextBlock = BBI;
01913 
01914   MachineBasicBlock* MBB = B.Cases[0].ThisBB;
01915 
01916   addSuccessorWithWeight(SwitchBB, B.Default);
01917   addSuccessorWithWeight(SwitchBB, MBB);
01918 
01919   SDValue BrRange = DAG.getNode(ISD::BRCOND, getCurSDLoc(),
01920                                 MVT::Other, CopyTo, RangeCmp,
01921                                 DAG.getBasicBlock(B.Default));
01922 
01923   if (MBB != NextBlock)
01924     BrRange = DAG.getNode(ISD::BR, getCurSDLoc(), MVT::Other, CopyTo,
01925                           DAG.getBasicBlock(MBB));
01926 
01927   DAG.setRoot(BrRange);
01928 }
01929 
01930 /// visitBitTestCase - this function produces one "bit test"
01931 void SelectionDAGBuilder::visitBitTestCase(BitTestBlock &BB,
01932                                            MachineBasicBlock* NextMBB,
01933                                            uint32_t BranchWeightToNext,
01934                                            unsigned Reg,
01935                                            BitTestCase &B,
01936                                            MachineBasicBlock *SwitchBB) {
01937   MVT VT = BB.RegVT;
01938   SDValue ShiftOp = DAG.getCopyFromReg(getControlRoot(), getCurSDLoc(),
01939                                        Reg, VT);
01940   SDValue Cmp;
01941   unsigned PopCount = CountPopulation_64(B.Mask);
01942   const TargetLowering *TLI = TM.getTargetLowering();
01943   if (PopCount == 1) {
01944     // Testing for a single bit; just compare the shift count with what it
01945     // would need to be to shift a 1 bit in that position.
01946     Cmp = DAG.getSetCC(getCurSDLoc(),
01947                        TLI->getSetCCResultType(*DAG.getContext(), VT),
01948                        ShiftOp,
01949                        DAG.getConstant(countTrailingZeros(B.Mask), VT),
01950                        ISD::SETEQ);
01951   } else if (PopCount == BB.Range) {
01952     // There is only one zero bit in the range, test for it directly.
01953     Cmp = DAG.getSetCC(getCurSDLoc(),
01954                        TLI->getSetCCResultType(*DAG.getContext(), VT),
01955                        ShiftOp,
01956                        DAG.getConstant(CountTrailingOnes_64(B.Mask), VT),
01957                        ISD::SETNE);
01958   } else {
01959     // Make desired shift
01960     SDValue SwitchVal = DAG.getNode(ISD::SHL, getCurSDLoc(), VT,
01961                                     DAG.getConstant(1, VT), ShiftOp);
01962 
01963     // Emit bit tests and jumps
01964     SDValue AndOp = DAG.getNode(ISD::AND, getCurSDLoc(),
01965                                 VT, SwitchVal, DAG.getConstant(B.Mask, VT));
01966     Cmp = DAG.getSetCC(getCurSDLoc(),
01967                        TLI->getSetCCResultType(*DAG.getContext(), VT),
01968                        AndOp, DAG.getConstant(0, VT),
01969                        ISD::SETNE);
01970   }
01971 
01972   // The branch weight from SwitchBB to B.TargetBB is B.ExtraWeight.
01973   addSuccessorWithWeight(SwitchBB, B.TargetBB, B.ExtraWeight);
01974   // The branch weight from SwitchBB to NextMBB is BranchWeightToNext.
01975   addSuccessorWithWeight(SwitchBB, NextMBB, BranchWeightToNext);
01976 
01977   SDValue BrAnd = DAG.getNode(ISD::BRCOND, getCurSDLoc(),
01978                               MVT::Other, getControlRoot(),
01979                               Cmp, DAG.getBasicBlock(B.TargetBB));
01980 
01981   // Set NextBlock to be the MBB immediately after the current one, if any.
01982   // This is used to avoid emitting unnecessary branches to the next block.
01983   MachineBasicBlock *NextBlock = nullptr;
01984   MachineFunction::iterator BBI = SwitchBB;
01985   if (++BBI != FuncInfo.MF->end())
01986     NextBlock = BBI;
01987 
01988   if (NextMBB != NextBlock)
01989     BrAnd = DAG.getNode(ISD::BR, getCurSDLoc(), MVT::Other, BrAnd,
01990                         DAG.getBasicBlock(NextMBB));
01991 
01992   DAG.setRoot(BrAnd);
01993 }
01994 
01995 void SelectionDAGBuilder::visitInvoke(const InvokeInst &I) {
01996   MachineBasicBlock *InvokeMBB = FuncInfo.MBB;
01997 
01998   // Retrieve successors.
01999   MachineBasicBlock *Return = FuncInfo.MBBMap[I.getSuccessor(0)];
02000   MachineBasicBlock *LandingPad = FuncInfo.MBBMap[I.getSuccessor(1)];
02001 
02002   const Value *Callee(I.getCalledValue());
02003   const Function *Fn = dyn_cast<Function>(Callee);
02004   if (isa<InlineAsm>(Callee))
02005     visitInlineAsm(&I);
02006   else if (Fn && Fn->isIntrinsic()) {
02007     assert(Fn->getIntrinsicID() == Intrinsic::donothing);
02008     // Ignore invokes to @llvm.donothing: jump directly to the next BB.
02009   } else
02010     LowerCallTo(&I, getValue(Callee), false, LandingPad);
02011 
02012   // If the value of the invoke is used outside of its defining block, make it
02013   // available as a virtual register.
02014   CopyToExportRegsIfNeeded(&I);
02015 
02016   // Update successor info
02017   addSuccessorWithWeight(InvokeMBB, Return);
02018   addSuccessorWithWeight(InvokeMBB, LandingPad);
02019 
02020   // Drop into normal successor.
02021   DAG.setRoot(DAG.getNode(ISD::BR, getCurSDLoc(),
02022                           MVT::Other, getControlRoot(),
02023                           DAG.getBasicBlock(Return)));
02024 }
02025 
02026 void SelectionDAGBuilder::visitResume(const ResumeInst &RI) {
02027   llvm_unreachable("SelectionDAGBuilder shouldn't visit resume instructions!");
02028 }
02029 
02030 void SelectionDAGBuilder::visitLandingPad(const LandingPadInst &LP) {
02031   assert(FuncInfo.MBB->isLandingPad() &&
02032          "Call to landingpad not in landing pad!");
02033 
02034   MachineBasicBlock *MBB = FuncInfo.MBB;
02035   MachineModuleInfo &MMI = DAG.getMachineFunction().getMMI();
02036   AddLandingPadInfo(LP, MMI, MBB);
02037 
02038   // If there aren't registers to copy the values into (e.g., during SjLj
02039   // exceptions), then don't bother to create these DAG nodes.
02040   const TargetLowering *TLI = TM.getTargetLowering();
02041   if (TLI->getExceptionPointerRegister() == 0 &&
02042       TLI->getExceptionSelectorRegister() == 0)
02043     return;
02044 
02045   SmallVector<EVT, 2> ValueVTs;
02046   ComputeValueVTs(*TLI, LP.getType(), ValueVTs);
02047   assert(ValueVTs.size() == 2 && "Only two-valued landingpads are supported");
02048 
02049   // Get the two live-in registers as SDValues. The physregs have already been
02050   // copied into virtual registers.
02051   SDValue Ops[2];
02052   Ops[0] = DAG.getZExtOrTrunc(
02053     DAG.getCopyFromReg(DAG.getEntryNode(), getCurSDLoc(),
02054                        FuncInfo.ExceptionPointerVirtReg, TLI->getPointerTy()),
02055     getCurSDLoc(), ValueVTs[0]);
02056   Ops[1] = DAG.getZExtOrTrunc(
02057     DAG.getCopyFromReg(DAG.getEntryNode(), getCurSDLoc(),
02058                        FuncInfo.ExceptionSelectorVirtReg, TLI->getPointerTy()),
02059     getCurSDLoc(), ValueVTs[1]);
02060 
02061   // Merge into one.
02062   SDValue Res = DAG.getNode(ISD::MERGE_VALUES, getCurSDLoc(),
02063                             DAG.getVTList(ValueVTs),
02064                             &Ops[0], 2);
02065   setValue(&LP, Res);
02066 }
02067 
02068 /// handleSmallSwitchCaseRange - Emit a series of specific tests (suitable for
02069 /// small case ranges).
02070 bool SelectionDAGBuilder::handleSmallSwitchRange(CaseRec& CR,
02071                                                  CaseRecVector& WorkList,
02072                                                  const Value* SV,
02073                                                  MachineBasicBlock *Default,
02074                                                  MachineBasicBlock *SwitchBB) {
02075   // Size is the number of Cases represented by this range.
02076   size_t Size = CR.Range.second - CR.Range.first;
02077   if (Size > 3)
02078     return false;
02079 
02080   // Get the MachineFunction which holds the current MBB.  This is used when
02081   // inserting any additional MBBs necessary to represent the switch.
02082   MachineFunction *CurMF = FuncInfo.MF;
02083 
02084   // Figure out which block is immediately after the current one.
02085   MachineBasicBlock *NextBlock = nullptr;
02086   MachineFunction::iterator BBI = CR.CaseBB;
02087 
02088   if (++BBI != FuncInfo.MF->end())
02089     NextBlock = BBI;
02090 
02091   BranchProbabilityInfo *BPI = FuncInfo.BPI;
02092   // If any two of the cases has the same destination, and if one value
02093   // is the same as the other, but has one bit unset that the other has set,
02094   // use bit manipulation to do two compares at once.  For example:
02095   // "if (X == 6 || X == 4)" -> "if ((X|2) == 6)"
02096   // TODO: This could be extended to merge any 2 cases in switches with 3 cases.
02097   // TODO: Handle cases where CR.CaseBB != SwitchBB.
02098   if (Size == 2 && CR.CaseBB == SwitchBB) {
02099     Case &Small = *CR.Range.first;
02100     Case &Big = *(CR.Range.second-1);
02101 
02102     if (Small.Low == Small.High && Big.Low == Big.High && Small.BB == Big.BB) {
02103       const APInt& SmallValue = cast<ConstantInt>(Small.Low)->getValue();
02104       const APInt& BigValue = cast<ConstantInt>(Big.Low)->getValue();
02105 
02106       // Check that there is only one bit different.
02107       if (BigValue.countPopulation() == SmallValue.countPopulation() + 1 &&
02108           (SmallValue | BigValue) == BigValue) {
02109         // Isolate the common bit.
02110         APInt CommonBit = BigValue & ~SmallValue;
02111         assert((SmallValue | CommonBit) == BigValue &&
02112                CommonBit.countPopulation() == 1 && "Not a common bit?");
02113 
02114         SDValue CondLHS = getValue(SV);
02115         EVT VT = CondLHS.getValueType();
02116         SDLoc DL = getCurSDLoc();
02117 
02118         SDValue Or = DAG.getNode(ISD::OR, DL, VT, CondLHS,
02119                                  DAG.getConstant(CommonBit, VT));
02120         SDValue Cond = DAG.getSetCC(DL, MVT::i1,
02121                                     Or, DAG.getConstant(BigValue, VT),
02122                                     ISD::SETEQ);
02123 
02124         // Update successor info.
02125         // Both Small and Big will jump to Small.BB, so we sum up the weights.
02126         addSuccessorWithWeight(SwitchBB, Small.BB,
02127                                Small.ExtraWeight + Big.ExtraWeight);
02128         addSuccessorWithWeight(SwitchBB, Default,
02129           // The default destination is the first successor in IR.
02130           BPI ? BPI->getEdgeWeight(SwitchBB->getBasicBlock(), (unsigned)0) : 0);
02131 
02132         // Insert the true branch.
02133         SDValue BrCond = DAG.getNode(ISD::BRCOND, DL, MVT::Other,
02134                                      getControlRoot(), Cond,
02135                                      DAG.getBasicBlock(Small.BB));
02136 
02137         // Insert the false branch.
02138         BrCond = DAG.getNode(ISD::BR, DL, MVT::Other, BrCond,
02139                              DAG.getBasicBlock(Default));
02140 
02141         DAG.setRoot(BrCond);
02142         return true;
02143       }
02144     }
02145   }
02146 
02147   // Order cases by weight so the most likely case will be checked first.
02148   uint32_t UnhandledWeights = 0;
02149   if (BPI) {
02150     for (CaseItr I = CR.Range.first, IE = CR.Range.second; I != IE; ++I) {
02151       uint32_t IWeight = I->ExtraWeight;
02152       UnhandledWeights += IWeight;
02153       for (CaseItr J = CR.Range.first; J < I; ++J) {
02154         uint32_t JWeight = J->ExtraWeight;
02155         if (IWeight > JWeight)
02156           std::swap(*I, *J);
02157       }
02158     }
02159   }
02160   // Rearrange the case blocks so that the last one falls through if possible.
02161   Case &BackCase = *(CR.Range.second-1);
02162   if (Size > 1 &&
02163       NextBlock && Default != NextBlock && BackCase.BB != NextBlock) {
02164     // The last case block won't fall through into 'NextBlock' if we emit the
02165     // branches in this order.  See if rearranging a case value would help.
02166     // We start at the bottom as it's the case with the least weight.
02167     for (Case *I = &*(CR.Range.second-2), *E = &*CR.Range.first-1; I != E; --I)
02168       if (I->BB == NextBlock) {
02169         std::swap(*I, BackCase);
02170         break;
02171       }
02172   }
02173 
02174   // Create a CaseBlock record representing a conditional branch to
02175   // the Case's target mbb if the value being switched on SV is equal
02176   // to C.
02177   MachineBasicBlock *CurBlock = CR.CaseBB;
02178   for (CaseItr I = CR.Range.first, E = CR.Range.second; I != E; ++I) {
02179     MachineBasicBlock *FallThrough;
02180     if (I != E-1) {
02181       FallThrough = CurMF->CreateMachineBasicBlock(CurBlock->getBasicBlock());
02182       CurMF->insert(BBI, FallThrough);
02183 
02184       // Put SV in a virtual register to make it available from the new blocks.
02185       ExportFromCurrentBlock(SV);
02186     } else {
02187       // If the last case doesn't match, go to the default block.
02188       FallThrough = Default;
02189     }
02190 
02191     const Value *RHS, *LHS, *MHS;
02192     ISD::CondCode CC;
02193     if (I->High == I->Low) {
02194       // This is just small small case range :) containing exactly 1 case
02195       CC = ISD::SETEQ;
02196       LHS = SV; RHS = I->High; MHS = nullptr;
02197     } else {
02198       CC = ISD::SETLE;
02199       LHS = I->Low; MHS = SV; RHS = I->High;
02200     }
02201 
02202     // The false weight should be sum of all un-handled cases.
02203     UnhandledWeights -= I->ExtraWeight;
02204     CaseBlock CB(CC, LHS, RHS, MHS, /* truebb */ I->BB, /* falsebb */ FallThrough,
02205                  /* me */ CurBlock,
02206                  /* trueweight */ I->ExtraWeight,
02207                  /* falseweight */ UnhandledWeights);
02208 
02209     // If emitting the first comparison, just call visitSwitchCase to emit the
02210     // code into the current block.  Otherwise, push the CaseBlock onto the
02211     // vector to be later processed by SDISel, and insert the node's MBB
02212     // before the next MBB.
02213     if (CurBlock == SwitchBB)
02214       visitSwitchCase(CB, SwitchBB);
02215     else
02216       SwitchCases.push_back(CB);
02217 
02218     CurBlock = FallThrough;
02219   }
02220 
02221   return true;
02222 }
02223 
02224 static inline bool areJTsAllowed(const TargetLowering &TLI) {
02225   return TLI.supportJumpTables() &&
02226           (TLI.isOperationLegalOrCustom(ISD::BR_JT, MVT::Other) ||
02227            TLI.isOperationLegalOrCustom(ISD::BRIND, MVT::Other));
02228 }
02229 
02230 static APInt ComputeRange(const APInt &First, const APInt &Last) {
02231   uint32_t BitWidth = std::max(Last.getBitWidth(), First.getBitWidth()) + 1;
02232   APInt LastExt = Last.sext(BitWidth), FirstExt = First.sext(BitWidth);
02233   return (LastExt - FirstExt + 1ULL);
02234 }
02235 
02236 /// handleJTSwitchCase - Emit jumptable for current switch case range
02237 bool SelectionDAGBuilder::handleJTSwitchCase(CaseRec &CR,
02238                                              CaseRecVector &WorkList,
02239                                              const Value *SV,
02240                                              MachineBasicBlock *Default,
02241                                              MachineBasicBlock *SwitchBB) {
02242   Case& FrontCase = *CR.Range.first;
02243   Case& BackCase  = *(CR.Range.second-1);
02244 
02245   const APInt &First = cast<ConstantInt>(FrontCase.Low)->getValue();
02246   const APInt &Last  = cast<ConstantInt>(BackCase.High)->getValue();
02247 
02248   APInt TSize(First.getBitWidth(), 0);
02249   for (CaseItr I = CR.Range.first, E = CR.Range.second; I != E; ++I)
02250     TSize += I->size();
02251 
02252   const TargetLowering *TLI = TM.getTargetLowering();
02253   if (!areJTsAllowed(*TLI) || TSize.ult(TLI->getMinimumJumpTableEntries()))
02254     return false;
02255 
02256   APInt Range = ComputeRange(First, Last);
02257   // The density is TSize / Range. Require at least 40%.
02258   // It should not be possible for IntTSize to saturate for sane code, but make
02259   // sure we handle Range saturation correctly.
02260   uint64_t IntRange = Range.getLimitedValue(UINT64_MAX/10);
02261   uint64_t IntTSize = TSize.getLimitedValue(UINT64_MAX/10);
02262   if (IntTSize * 10 < IntRange * 4)
02263     return false;
02264 
02265   DEBUG(dbgs() << "Lowering jump table\n"
02266                << "First entry: " << First << ". Last entry: " << Last << '\n'
02267                << "Range: " << Range << ". Size: " << TSize << ".\n\n");
02268 
02269   // Get the MachineFunction which holds the current MBB.  This is used when
02270   // inserting any additional MBBs necessary to represent the switch.
02271   MachineFunction *CurMF = FuncInfo.MF;
02272 
02273   // Figure out which block is immediately after the current one.
02274   MachineFunction::iterator BBI = CR.CaseBB;
02275   ++BBI;
02276 
02277   const BasicBlock *LLVMBB = CR.CaseBB->getBasicBlock();
02278 
02279   // Create a new basic block to hold the code for loading the address
02280   // of the jump table, and jumping to it.  Update successor information;
02281   // we will either branch to the default case for the switch, or the jump
02282   // table.
02283   MachineBasicBlock *JumpTableBB = CurMF->CreateMachineBasicBlock(LLVMBB);
02284   CurMF->insert(BBI, JumpTableBB);
02285 
02286   addSuccessorWithWeight(CR.CaseBB, Default);
02287   addSuccessorWithWeight(CR.CaseBB, JumpTableBB);
02288 
02289   // Build a vector of destination BBs, corresponding to each target
02290   // of the jump table. If the value of the jump table slot corresponds to
02291   // a case statement, push the case's BB onto the vector, otherwise, push
02292   // the default BB.
02293   std::vector<MachineBasicBlock*> DestBBs;
02294   APInt TEI = First;
02295   for (CaseItr I = CR.Range.first, E = CR.Range.second; I != E; ++TEI) {
02296     const APInt &Low = cast<ConstantInt>(I->Low)->getValue();
02297     const APInt &High = cast<ConstantInt>(I->High)->getValue();
02298 
02299     if (Low.sle(TEI) && TEI.sle(High)) {
02300       DestBBs.push_back(I->BB);
02301       if (TEI==High)
02302         ++I;
02303     } else {
02304       DestBBs.push_back(Default);
02305     }
02306   }
02307 
02308   // Calculate weight for each unique destination in CR.
02309   DenseMap<MachineBasicBlock*, uint32_t> DestWeights;
02310   if (FuncInfo.BPI)
02311     for (CaseItr I = CR.Range.first, E = CR.Range.second; I != E; ++I) {
02312       DenseMap<MachineBasicBlock*, uint32_t>::iterator Itr =
02313           DestWeights.find(I->BB);
02314       if (Itr != DestWeights.end())
02315         Itr->second += I->ExtraWeight;
02316       else
02317         DestWeights[I->BB] = I->ExtraWeight;
02318     }
02319 
02320   // Update successor info. Add one edge to each unique successor.
02321   BitVector SuccsHandled(CR.CaseBB->getParent()->getNumBlockIDs());
02322   for (std::vector<MachineBasicBlock*>::iterator I = DestBBs.begin(),
02323          E = DestBBs.end(); I != E; ++I) {
02324     if (!SuccsHandled[(*I)->getNumber()]) {
02325       SuccsHandled[(*I)->getNumber()] = true;
02326       DenseMap<MachineBasicBlock*, uint32_t>::iterator Itr =
02327           DestWeights.find(*I);
02328       addSuccessorWithWeight(JumpTableBB, *I,
02329                              Itr != DestWeights.end() ? Itr->second : 0);
02330     }
02331   }
02332 
02333   // Create a jump table index for this jump table.
02334   unsigned JTEncoding = TLI->getJumpTableEncoding();
02335   unsigned JTI = CurMF->getOrCreateJumpTableInfo(JTEncoding)
02336                        ->createJumpTableIndex(DestBBs);
02337 
02338   // Set the jump table information so that we can codegen it as a second
02339   // MachineBasicBlock
02340   JumpTable JT(-1U, JTI, JumpTableBB, Default);
02341   JumpTableHeader JTH(First, Last, SV, CR.CaseBB, (CR.CaseBB == SwitchBB));
02342   if (CR.CaseBB == SwitchBB)
02343     visitJumpTableHeader(JT, JTH, SwitchBB);
02344 
02345   JTCases.push_back(JumpTableBlock(JTH, JT));
02346   return true;
02347 }
02348 
02349 /// handleBTSplitSwitchCase - emit comparison and split binary search tree into
02350 /// 2 subtrees.
02351 bool SelectionDAGBuilder::handleBTSplitSwitchCase(CaseRec& CR,
02352                                                   CaseRecVector& WorkList,
02353                                                   const Value* SV,
02354                                                   MachineBasicBlock* Default,
02355                                                   MachineBasicBlock* SwitchBB) {
02356   // Get the MachineFunction which holds the current MBB.  This is used when
02357   // inserting any additional MBBs necessary to represent the switch.
02358   MachineFunction *CurMF = FuncInfo.MF;
02359 
02360   // Figure out which block is immediately after the current one.
02361   MachineFunction::iterator BBI = CR.CaseBB;
02362   ++BBI;
02363 
02364   Case& FrontCase = *CR.Range.first;
02365   Case& BackCase  = *(CR.Range.second-1);
02366   const BasicBlock *LLVMBB = CR.CaseBB->getBasicBlock();
02367 
02368   // Size is the number of Cases represented by this range.
02369   unsigned Size = CR.Range.second - CR.Range.first;
02370 
02371   const APInt &First = cast<ConstantInt>(FrontCase.Low)->getValue();
02372   const APInt &Last  = cast<ConstantInt>(BackCase.High)->getValue();
02373   double FMetric = 0;
02374   CaseItr Pivot = CR.Range.first + Size/2;
02375 
02376   // Select optimal pivot, maximizing sum density of LHS and RHS. This will
02377   // (heuristically) allow us to emit JumpTable's later.
02378   APInt TSize(First.getBitWidth(), 0);
02379   for (CaseItr I = CR.Range.first, E = CR.Range.second;
02380        I!=E; ++I)
02381     TSize += I->size();
02382 
02383   APInt LSize = FrontCase.size();
02384   APInt RSize = TSize-LSize;
02385   DEBUG(dbgs() << "Selecting best pivot: \n"
02386                << "First: " << First << ", Last: " << Last <<'\n'
02387                << "LSize: " << LSize << ", RSize: " << RSize << '\n');
02388   for (CaseItr I = CR.Range.first, J=I+1, E = CR.Range.second;
02389        J!=E; ++I, ++J) {
02390     const APInt &LEnd = cast<ConstantInt>(I->High)->getValue();
02391     const APInt &RBegin = cast<ConstantInt>(J->Low)->getValue();
02392     APInt Range = ComputeRange(LEnd, RBegin);
02393     assert((Range - 2ULL).isNonNegative() &&
02394            "Invalid case distance");
02395     // Use volatile double here to avoid excess precision issues on some hosts,
02396     // e.g. that use 80-bit X87 registers.
02397     volatile double LDensity =
02398        (double)LSize.roundToDouble() /
02399                            (LEnd - First + 1ULL).roundToDouble();
02400     volatile double RDensity =
02401       (double)RSize.roundToDouble() /
02402                            (Last - RBegin + 1ULL).roundToDouble();
02403     volatile double Metric = Range.logBase2()*(LDensity+RDensity);
02404     // Should always split in some non-trivial place
02405     DEBUG(dbgs() <<"=>Step\n"
02406                  << "LEnd: " << LEnd << ", RBegin: " << RBegin << '\n'
02407                  << "LDensity: " << LDensity
02408                  << ", RDensity: " << RDensity << '\n'
02409                  << "Metric: " << Metric << '\n');
02410     if (FMetric < Metric) {
02411       Pivot = J;
02412       FMetric = Metric;
02413       DEBUG(dbgs() << "Current metric set to: " << FMetric << '\n');
02414     }
02415 
02416     LSize += J->size();
02417     RSize -= J->size();
02418   }
02419 
02420   const TargetLowering *TLI = TM.getTargetLowering();
02421   if (areJTsAllowed(*TLI)) {
02422     // If our case is dense we *really* should handle it earlier!
02423     assert((FMetric > 0) && "Should handle dense range earlier!");
02424   } else {
02425     Pivot = CR.Range.first + Size/2;
02426   }
02427 
02428   CaseRange LHSR(CR.Range.first, Pivot);
02429   CaseRange RHSR(Pivot, CR.Range.second);
02430   const Constant *C = Pivot->Low;
02431   MachineBasicBlock *FalseBB = nullptr, *TrueBB = nullptr;
02432 
02433   // We know that we branch to the LHS if the Value being switched on is
02434   // less than the Pivot value, C.  We use this to optimize our binary
02435   // tree a bit, by recognizing that if SV is greater than or equal to the
02436   // LHS's Case Value, and that Case Value is exactly one less than the
02437   // Pivot's Value, then we can branch directly to the LHS's Target,
02438   // rather than creating a leaf node for it.
02439   if ((LHSR.second - LHSR.first) == 1 &&
02440       LHSR.first->High == CR.GE &&
02441       cast<ConstantInt>(C)->getValue() ==
02442       (cast<ConstantInt>(CR.GE)->getValue() + 1LL)) {
02443     TrueBB = LHSR.first->BB;
02444   } else {
02445     TrueBB = CurMF->CreateMachineBasicBlock(LLVMBB);
02446     CurMF->insert(BBI, TrueBB);
02447     WorkList.push_back(CaseRec(TrueBB, C, CR.GE, LHSR));
02448 
02449     // Put SV in a virtual register to make it available from the new blocks.
02450     ExportFromCurrentBlock(SV);
02451   }
02452 
02453   // Similar to the optimization above, if the Value being switched on is
02454   // known to be less than the Constant CR.LT, and the current Case Value
02455   // is CR.LT - 1, then we can branch directly to the target block for
02456   // the current Case Value, rather than emitting a RHS leaf node for it.
02457   if ((RHSR.second - RHSR.first) == 1 && CR.LT &&
02458       cast<ConstantInt>(RHSR.first->Low)->getValue() ==
02459       (cast<ConstantInt>(CR.LT)->getValue() - 1LL)) {
02460     FalseBB = RHSR.first->BB;
02461   } else {
02462     FalseBB = CurMF->CreateMachineBasicBlock(LLVMBB);
02463     CurMF->insert(BBI, FalseBB);
02464     WorkList.push_back(CaseRec(FalseBB,CR.LT,C,RHSR));
02465 
02466     // Put SV in a virtual register to make it available from the new blocks.
02467     ExportFromCurrentBlock(SV);
02468   }
02469 
02470   // Create a CaseBlock record representing a conditional branch to
02471   // the LHS node if the value being switched on SV is less than C.
02472   // Otherwise, branch to LHS.
02473   CaseBlock CB(ISD::SETLT, SV, C, nullptr, TrueBB, FalseBB, CR.CaseBB);
02474 
02475   if (CR.CaseBB == SwitchBB)
02476     visitSwitchCase(CB, SwitchBB);
02477   else
02478     SwitchCases.push_back(CB);
02479 
02480   return true;
02481 }
02482 
02483 /// handleBitTestsSwitchCase - if current case range has few destination and
02484 /// range span less, than machine word bitwidth, encode case range into series
02485 /// of masks and emit bit tests with these masks.
02486 bool SelectionDAGBuilder::handleBitTestsSwitchCase(CaseRec& CR,
02487                                                    CaseRecVector& WorkList,
02488                                                    const Value* SV,
02489                                                    MachineBasicBlock* Default,
02490                                                    MachineBasicBlock* SwitchBB) {
02491   const TargetLowering *TLI = TM.getTargetLowering();
02492   EVT PTy = TLI->getPointerTy();
02493   unsigned IntPtrBits = PTy.getSizeInBits();
02494 
02495   Case& FrontCase = *CR.Range.first;
02496   Case& BackCase  = *(CR.Range.second-1);
02497 
02498   // Get the MachineFunction which holds the current MBB.  This is used when
02499   // inserting any additional MBBs necessary to represent the switch.
02500   MachineFunction *CurMF = FuncInfo.MF;
02501 
02502   // If target does not have legal shift left, do not emit bit tests at all.
02503   if (!TLI->isOperationLegal(ISD::SHL, PTy))
02504     return false;
02505 
02506   size_t numCmps = 0;
02507   for (CaseItr I = CR.Range.first, E = CR.Range.second;
02508        I!=E; ++I) {
02509     // Single case counts one, case range - two.
02510     numCmps += (I->Low == I->High ? 1 : 2);
02511   }
02512 
02513   // Count unique destinations
02514   SmallSet<MachineBasicBlock*, 4> Dests;
02515   for (CaseItr I = CR.Range.first, E = CR.Range.second; I!=E; ++I) {
02516     Dests.insert(I->BB);
02517     if (Dests.size() > 3)
02518       // Don't bother the code below, if there are too much unique destinations
02519       return false;
02520   }
02521   DEBUG(dbgs() << "Total number of unique destinations: "
02522         << Dests.size() << '\n'
02523         << "Total number of comparisons: " << numCmps << '\n');
02524 
02525   // Compute span of values.
02526   const APInt& minValue = cast<ConstantInt>(FrontCase.Low)->getValue();
02527   const APInt& maxValue = cast<ConstantInt>(BackCase.High)->getValue();
02528   APInt cmpRange = maxValue - minValue;
02529 
02530   DEBUG(dbgs() << "Compare range: " << cmpRange << '\n'
02531                << "Low bound: " << minValue << '\n'
02532                << "High bound: " << maxValue << '\n');
02533 
02534   if (cmpRange.uge(IntPtrBits) ||
02535       (!(Dests.size() == 1 && numCmps >= 3) &&
02536        !(Dests.size() == 2 && numCmps >= 5) &&
02537        !(Dests.size() >= 3 && numCmps >= 6)))
02538     return false;
02539 
02540   DEBUG(dbgs() << "Emitting bit tests\n");
02541   APInt lowBound = APInt::getNullValue(cmpRange.getBitWidth());
02542 
02543   // Optimize the case where all the case values fit in a
02544   // word without having to subtract minValue. In this case,
02545   // we can optimize away the subtraction.
02546   if (minValue.isNonNegative() && maxValue.slt(IntPtrBits)) {
02547     cmpRange = maxValue;
02548   } else {
02549     lowBound = minValue;
02550   }
02551 
02552   CaseBitsVector CasesBits;
02553   unsigned i, count = 0;
02554 
02555   for (CaseItr I = CR.Range.first, E = CR.Range.second; I!=E; ++I) {
02556     MachineBasicBlock* Dest = I->BB;
02557     for (i = 0; i < count; ++i)
02558       if (Dest == CasesBits[i].BB)
02559         break;
02560 
02561     if (i == count) {
02562       assert((count < 3) && "Too much destinations to test!");
02563       CasesBits.push_back(CaseBits(0, Dest, 0, 0/*Weight*/));
02564       count++;
02565     }
02566 
02567     const APInt& lowValue = cast<ConstantInt>(I->Low)->getValue();
02568     const APInt& highValue = cast<ConstantInt>(I->High)->getValue();
02569 
02570     uint64_t lo = (lowValue - lowBound).getZExtValue();
02571     uint64_t hi = (highValue - lowBound).getZExtValue();
02572     CasesBits[i].ExtraWeight += I->ExtraWeight;
02573 
02574     for (uint64_t j = lo; j <= hi; j++) {
02575       CasesBits[i].Mask |=  1ULL << j;
02576       CasesBits[i].Bits++;
02577     }
02578 
02579   }
02580   std::sort(CasesBits.begin(), CasesBits.end(), CaseBitsCmp());
02581 
02582   BitTestInfo BTC;
02583 
02584   // Figure out which block is immediately after the current one.
02585   MachineFunction::iterator BBI = CR.CaseBB;
02586   ++BBI;
02587 
02588   const BasicBlock *LLVMBB = CR.CaseBB->getBasicBlock();
02589 
02590   DEBUG(dbgs() << "Cases:\n");
02591   for (unsigned i = 0, e = CasesBits.size(); i!=e; ++i) {
02592     DEBUG(dbgs() << "Mask: " << CasesBits[i].Mask
02593                  << ", Bits: " << CasesBits[i].Bits
02594                  << ", BB: " << CasesBits[i].BB << '\n');
02595 
02596     MachineBasicBlock *CaseBB = CurMF->CreateMachineBasicBlock(LLVMBB);
02597     CurMF->insert(BBI, CaseBB);
02598     BTC.push_back(BitTestCase(CasesBits[i].Mask,
02599                               CaseBB,
02600                               CasesBits[i].BB, CasesBits[i].ExtraWeight));
02601 
02602     // Put SV in a virtual register to make it available from the new blocks.
02603     ExportFromCurrentBlock(SV);
02604   }
02605 
02606   BitTestBlock BTB(lowBound, cmpRange, SV,
02607                    -1U, MVT::Other, (CR.CaseBB == SwitchBB),
02608                    CR.CaseBB, Default, BTC);
02609 
02610   if (CR.CaseBB == SwitchBB)
02611     visitBitTestHeader(BTB, SwitchBB);
02612 
02613   BitTestCases.push_back(BTB);
02614 
02615   return true;
02616 }
02617 
02618 /// Clusterify - Transform simple list of Cases into list of CaseRange's
02619 size_t SelectionDAGBuilder::Clusterify(CaseVector& Cases,
02620                                        const SwitchInst& SI) {
02621   size_t numCmps = 0;
02622 
02623   BranchProbabilityInfo *BPI = FuncInfo.BPI;
02624   // Start with "simple" cases
02625   for (SwitchInst::ConstCaseIt i = SI.case_begin(), e = SI.case_end();
02626        i != e; ++i) {
02627     const BasicBlock *SuccBB = i.getCaseSuccessor();
02628     MachineBasicBlock *SMBB = FuncInfo.MBBMap[SuccBB];
02629 
02630     uint32_t ExtraWeight =
02631       BPI ? BPI->getEdgeWeight(SI.getParent(), i.getSuccessorIndex()) : 0;
02632 
02633     Cases.push_back(Case(i.getCaseValue(), i.getCaseValue(),
02634                          SMBB, ExtraWeight));
02635   }
02636   std::sort(Cases.begin(), Cases.end(), CaseCmp());
02637 
02638   // Merge case into clusters
02639   if (Cases.size() >= 2)
02640     // Must recompute end() each iteration because it may be
02641     // invalidated by erase if we hold on to it
02642     for (CaseItr I = Cases.begin(), J = std::next(Cases.begin());
02643          J != Cases.end(); ) {
02644       const APInt& nextValue = cast<ConstantInt>(J->Low)->getValue();
02645       const APInt& currentValue = cast<ConstantInt>(I->High)->getValue();
02646       MachineBasicBlock* nextBB = J->BB;
02647       MachineBasicBlock* currentBB = I->BB;
02648 
02649       // If the two neighboring cases go to the same destination, merge them
02650       // into a single case.
02651       if ((nextValue - currentValue == 1) && (currentBB == nextBB)) {
02652         I->High = J->High;
02653         I->ExtraWeight += J->ExtraWeight;
02654         J = Cases.erase(J);
02655       } else {
02656         I = J++;
02657       }
02658     }
02659 
02660   for (CaseItr I=Cases.begin(), E=Cases.end(); I!=E; ++I, ++numCmps) {
02661     if (I->Low != I->High)
02662       // A range counts double, since it requires two compares.
02663       ++numCmps;
02664   }
02665 
02666   return numCmps;
02667 }
02668 
02669 void SelectionDAGBuilder::UpdateSplitBlock(MachineBasicBlock *First,
02670                                            MachineBasicBlock *Last) {
02671   // Update JTCases.
02672   for (unsigned i = 0, e = JTCases.size(); i != e; ++i)
02673     if (JTCases[i].first.HeaderBB == First)
02674       JTCases[i].first.HeaderBB = Last;
02675 
02676   // Update BitTestCases.
02677   for (unsigned i = 0, e = BitTestCases.size(); i != e; ++i)
02678     if (BitTestCases[i].Parent == First)
02679       BitTestCases[i].Parent = Last;
02680 }
02681 
02682 void SelectionDAGBuilder::visitSwitch(const SwitchInst &SI) {
02683   MachineBasicBlock *SwitchMBB = FuncInfo.MBB;
02684 
02685   // Figure out which block is immediately after the current one.
02686   MachineBasicBlock *NextBlock = nullptr;
02687   MachineBasicBlock *Default = FuncInfo.MBBMap[SI.getDefaultDest()];
02688 
02689   // If there is only the default destination, branch to it if it is not the
02690   // next basic block.  Otherwise, just fall through.
02691   if (!SI.getNumCases()) {
02692     // Update machine-CFG edges.
02693 
02694     // If this is not a fall-through branch, emit the branch.
02695     SwitchMBB->addSuccessor(Default);
02696     if (Default != NextBlock)
02697       DAG.setRoot(DAG.getNode(ISD::BR, getCurSDLoc(),
02698                               MVT::Other, getControlRoot(),
02699                               DAG.getBasicBlock(Default)));
02700 
02701     return;
02702   }
02703 
02704   // If there are any non-default case statements, create a vector of Cases
02705   // representing each one, and sort the vector so that we can efficiently
02706   // create a binary search tree from them.
02707   CaseVector Cases;
02708   size_t numCmps = Clusterify(Cases, SI);
02709   DEBUG(dbgs() << "Clusterify finished. Total clusters: " << Cases.size()
02710                << ". Total compares: " << numCmps << '\n');
02711   (void)numCmps;
02712 
02713   // Get the Value to be switched on and default basic blocks, which will be
02714   // inserted into CaseBlock records, representing basic blocks in the binary
02715   // search tree.
02716   const Value *SV = SI.getCondition();
02717 
02718   // Push the initial CaseRec onto the worklist
02719   CaseRecVector WorkList;
02720   WorkList.push_back(CaseRec(SwitchMBB,nullptr,nullptr,
02721                              CaseRange(Cases.begin(),Cases.end())));
02722 
02723   while (!WorkList.empty()) {
02724     // Grab a record representing a case range to process off the worklist
02725     CaseRec CR = WorkList.back();
02726     WorkList.pop_back();
02727 
02728     if (handleBitTestsSwitchCase(CR, WorkList, SV, Default, SwitchMBB))
02729       continue;
02730 
02731     // If the range has few cases (two or less) emit a series of specific
02732     // tests.
02733     if (handleSmallSwitchRange(CR, WorkList, SV, Default, SwitchMBB))
02734       continue;
02735 
02736     // If the switch has more than N blocks, and is at least 40% dense, and the
02737     // target supports indirect branches, then emit a jump table rather than
02738     // lowering the switch to a binary tree of conditional branches.
02739     // N defaults to 4 and is controlled via TLS.getMinimumJumpTableEntries().
02740     if (handleJTSwitchCase(CR, WorkList, SV, Default, SwitchMBB))
02741       continue;
02742 
02743     // Emit binary tree. We need to pick a pivot, and push left and right ranges
02744     // onto the worklist. Leafs are handled via handleSmallSwitchRange() call.
02745     handleBTSplitSwitchCase(CR, WorkList, SV, Default, SwitchMBB);
02746   }
02747 }
02748 
02749 void SelectionDAGBuilder::visitIndirectBr(const IndirectBrInst &I) {
02750   MachineBasicBlock *IndirectBrMBB = FuncInfo.MBB;
02751 
02752   // Update machine-CFG edges with unique successors.
02753   SmallSet<BasicBlock*, 32> Done;
02754   for (unsigned i = 0, e = I.getNumSuccessors(); i != e; ++i) {
02755     BasicBlock *BB = I.getSuccessor(i);
02756     bool Inserted = Done.insert(BB);
02757     if (!Inserted)
02758         continue;
02759 
02760     MachineBasicBlock *Succ = FuncInfo.MBBMap[BB];
02761     addSuccessorWithWeight(IndirectBrMBB, Succ);
02762   }
02763 
02764   DAG.setRoot(DAG.getNode(ISD::BRIND, getCurSDLoc(),
02765                           MVT::Other, getControlRoot(),
02766                           getValue(I.getAddress())));
02767 }
02768 
02769 void SelectionDAGBuilder::visitUnreachable(const UnreachableInst &I) {
02770   if (DAG.getTarget().Options.TrapUnreachable)
02771     DAG.setRoot(DAG.getNode(ISD::TRAP, getCurSDLoc(), MVT::Other, DAG.getRoot()));
02772 }
02773 
02774 void SelectionDAGBuilder::visitFSub(const User &I) {
02775   // -0.0 - X --> fneg
02776   Type *Ty = I.getType();
02777   if (isa<Constant>(I.getOperand(0)) &&
02778       I.getOperand(0) == ConstantFP::getZeroValueForNegation(Ty)) {
02779     SDValue Op2 = getValue(I.getOperand(1));
02780     setValue(&I, DAG.getNode(ISD::FNEG, getCurSDLoc(),
02781                              Op2.getValueType(), Op2));
02782     return;
02783   }
02784 
02785   visitBinary(I, ISD::FSUB);
02786 }
02787 
02788 void SelectionDAGBuilder::visitBinary(const User &I, unsigned OpCode) {
02789   SDValue Op1 = getValue(I.getOperand(0));
02790   SDValue Op2 = getValue(I.getOperand(1));
02791   setValue(&I, DAG.getNode(OpCode, getCurSDLoc(),
02792                            Op1.getValueType(), Op1, Op2));
02793 }
02794 
02795 void SelectionDAGBuilder::visitShift(const User &I, unsigned Opcode) {
02796   SDValue Op1 = getValue(I.getOperand(0));
02797   SDValue Op2 = getValue(I.getOperand(1));
02798 
02799   EVT ShiftTy = TM.getTargetLowering()->getShiftAmountTy(Op2.getValueType());
02800 
02801   // Coerce the shift amount to the right type if we can.
02802   if (!I.getType()->isVectorTy() && Op2.getValueType() != ShiftTy) {
02803     unsigned ShiftSize = ShiftTy.getSizeInBits();
02804     unsigned Op2Size = Op2.getValueType().getSizeInBits();
02805     SDLoc DL = getCurSDLoc();
02806 
02807     // If the operand is smaller than the shift count type, promote it.
02808     if (ShiftSize > Op2Size)
02809       Op2 = DAG.getNode(ISD::ZERO_EXTEND, DL, ShiftTy, Op2);
02810 
02811     // If the operand is larger than the shift count type but the shift
02812     // count type has enough bits to represent any shift value, truncate
02813     // it now. This is a common case and it exposes the truncate to
02814     // optimization early.
02815     else if (ShiftSize >= Log2_32_Ceil(Op2.getValueType().getSizeInBits()))
02816       Op2 = DAG.getNode(ISD::TRUNCATE, DL, ShiftTy, Op2);
02817     // Otherwise we'll need to temporarily settle for some other convenient
02818     // type.  Type legalization will make adjustments once the shiftee is split.
02819     else
02820       Op2 = DAG.getZExtOrTrunc(Op2, DL, MVT::i32);
02821   }
02822 
02823   setValue(&I, DAG.getNode(Opcode, getCurSDLoc(),
02824                            Op1.getValueType(), Op1, Op2));
02825 }
02826 
02827 void SelectionDAGBuilder::visitSDiv(const User &I) {
02828   SDValue Op1 = getValue(I.getOperand(0));
02829   SDValue Op2 = getValue(I.getOperand(1));
02830 
02831   // Turn exact SDivs into multiplications.
02832   // FIXME: This should be in DAGCombiner, but it doesn't have access to the
02833   // exact bit.
02834   if (isa<BinaryOperator>(&I) && cast<BinaryOperator>(&I)->isExact() &&
02835       !isa<ConstantSDNode>(Op1) &&
02836       isa<ConstantSDNode>(Op2) && !cast<ConstantSDNode>(Op2)->isNullValue())
02837     setValue(&I, TM.getTargetLowering()->BuildExactSDIV(Op1, Op2,
02838                                                         getCurSDLoc(), DAG));
02839   else
02840     setValue(&I, DAG.getNode(ISD::SDIV, getCurSDLoc(), Op1.getValueType(),
02841                              Op1, Op2));
02842 }
02843 
02844 void SelectionDAGBuilder::visitICmp(const User &I) {
02845   ICmpInst::Predicate predicate = ICmpInst::BAD_ICMP_PREDICATE;
02846   if (const ICmpInst *IC = dyn_cast<ICmpInst>(&I))
02847     predicate = IC->getPredicate();
02848   else if (const ConstantExpr *IC = dyn_cast<ConstantExpr>(&I))
02849     predicate = ICmpInst::Predicate(IC->getPredicate());
02850   SDValue Op1 = getValue(I.getOperand(0));
02851   SDValue Op2 = getValue(I.getOperand(1));
02852   ISD::CondCode Opcode = getICmpCondCode(predicate);
02853 
02854   EVT DestVT = TM.getTargetLowering()->getValueType(I.getType());
02855   setValue(&I, DAG.getSetCC(getCurSDLoc(), DestVT, Op1, Op2, Opcode));
02856 }
02857 
02858 void SelectionDAGBuilder::visitFCmp(const User &I) {
02859   FCmpInst::Predicate predicate = FCmpInst::BAD_FCMP_PREDICATE;
02860   if (const FCmpInst *FC = dyn_cast<FCmpInst>(&I))
02861     predicate = FC->getPredicate();
02862   else if (const ConstantExpr *FC = dyn_cast<ConstantExpr>(&I))
02863     predicate = FCmpInst::Predicate(FC->getPredicate());
02864   SDValue Op1 = getValue(I.getOperand(0));
02865   SDValue Op2 = getValue(I.getOperand(1));
02866   ISD::CondCode Condition = getFCmpCondCode(predicate);
02867   if (TM.Options.NoNaNsFPMath)
02868     Condition = getFCmpCodeWithoutNaN(Condition);
02869   EVT DestVT = TM.getTargetLowering()->getValueType(I.getType());
02870   setValue(&I, DAG.getSetCC(getCurSDLoc(), DestVT, Op1, Op2, Condition));
02871 }
02872 
02873 void SelectionDAGBuilder::visitSelect(const User &I) {
02874   SmallVector<EVT, 4> ValueVTs;
02875   ComputeValueVTs(*TM.getTargetLowering(), I.getType(), ValueVTs);
02876   unsigned NumValues = ValueVTs.size();
02877   if (NumValues == 0) return;
02878 
02879   SmallVector<SDValue, 4> Values(NumValues);
02880   SDValue Cond     = getValue(I.getOperand(0));
02881   SDValue TrueVal  = getValue(I.getOperand(1));
02882   SDValue FalseVal = getValue(I.getOperand(2));
02883   ISD::NodeType OpCode = Cond.getValueType().isVector() ?
02884     ISD::VSELECT : ISD::SELECT;
02885 
02886   for (unsigned i = 0; i != NumValues; ++i)
02887     Values[i] = DAG.getNode(OpCode, getCurSDLoc(),
02888                             TrueVal.getNode()->getValueType(TrueVal.getResNo()+i),
02889                             Cond,
02890                             SDValue(TrueVal.getNode(),
02891                                     TrueVal.getResNo() + i),
02892                             SDValue(FalseVal.getNode(),
02893                                     FalseVal.getResNo() + i));
02894 
02895   setValue(&I, DAG.getNode(ISD::MERGE_VALUES, getCurSDLoc(),
02896                            DAG.getVTList(ValueVTs),
02897                            &Values[0], NumValues));
02898 }
02899 
02900 void SelectionDAGBuilder::visitTrunc(const User &I) {
02901   // TruncInst cannot be a no-op cast because sizeof(src) > sizeof(dest).
02902   SDValue N = getValue(I.getOperand(0));
02903   EVT DestVT = TM.getTargetLowering()->getValueType(I.getType());
02904   setValue(&I, DAG.getNode(ISD::TRUNCATE, getCurSDLoc(), DestVT, N));
02905 }
02906 
02907 void SelectionDAGBuilder::visitZExt(const User &I) {
02908   // ZExt cannot be a no-op cast because sizeof(src) < sizeof(dest).
02909   // ZExt also can't be a cast to bool for same reason. So, nothing much to do
02910   SDValue N = getValue(I.getOperand(0));
02911   EVT DestVT = TM.getTargetLowering()->getValueType(I.getType());
02912   setValue(&I, DAG.getNode(ISD::ZERO_EXTEND, getCurSDLoc(), DestVT, N));
02913 }
02914 
02915 void SelectionDAGBuilder::visitSExt(const User &I) {
02916   // SExt cannot be a no-op cast because sizeof(src) < sizeof(dest).
02917   // SExt also can't be a cast to bool for same reason. So, nothing much to do
02918   SDValue N = getValue(I.getOperand(0));
02919   EVT DestVT = TM.getTargetLowering()->getValueType(I.getType());
02920   setValue(&I, DAG.getNode(ISD::SIGN_EXTEND, getCurSDLoc(), DestVT, N));
02921 }
02922 
02923 void SelectionDAGBuilder::visitFPTrunc(const User &I) {
02924   // FPTrunc is never a no-op cast, no need to check
02925   SDValue N = getValue(I.getOperand(0));
02926   const TargetLowering *TLI = TM.getTargetLowering();
02927   EVT DestVT = TLI->getValueType(I.getType());
02928   setValue(&I, DAG.getNode(ISD::FP_ROUND, getCurSDLoc(),
02929                            DestVT, N,
02930                            DAG.getTargetConstant(0, TLI->getPointerTy())));
02931 }
02932 
02933 void SelectionDAGBuilder::visitFPExt(const User &I) {
02934   // FPExt is never a no-op cast, no need to check
02935   SDValue N = getValue(I.getOperand(0));
02936   EVT DestVT = TM.getTargetLowering()->getValueType(I.getType());
02937   setValue(&I, DAG.getNode(ISD::FP_EXTEND, getCurSDLoc(), DestVT, N));
02938 }
02939 
02940 void SelectionDAGBuilder::visitFPToUI(const User &I) {
02941   // FPToUI is never a no-op cast, no need to check
02942   SDValue N = getValue(I.getOperand(0));
02943   EVT DestVT = TM.getTargetLowering()->getValueType(I.getType());
02944   setValue(&I, DAG.getNode(ISD::FP_TO_UINT, getCurSDLoc(), DestVT, N));
02945 }
02946 
02947 void SelectionDAGBuilder::visitFPToSI(const User &I) {
02948   // FPToSI is never a no-op cast, no need to check
02949   SDValue N = getValue(I.getOperand(0));
02950   EVT DestVT = TM.getTargetLowering()->getValueType(I.getType());
02951   setValue(&I, DAG.getNode(ISD::FP_TO_SINT, getCurSDLoc(), DestVT, N));
02952 }
02953 
02954 void SelectionDAGBuilder::visitUIToFP(const User &I) {
02955   // UIToFP is never a no-op cast, no need to check
02956   SDValue N = getValue(I.getOperand(0));
02957   EVT DestVT = TM.getTargetLowering()->getValueType(I.getType());
02958   setValue(&I, DAG.getNode(ISD::UINT_TO_FP, getCurSDLoc(), DestVT, N));
02959 }
02960 
02961 void SelectionDAGBuilder::visitSIToFP(const User &I) {
02962   // SIToFP is never a no-op cast, no need to check
02963   SDValue N = getValue(I.getOperand(0));
02964   EVT DestVT = TM.getTargetLowering()->getValueType(I.getType());
02965   setValue(&I, DAG.getNode(ISD::SINT_TO_FP, getCurSDLoc(), DestVT, N));
02966 }
02967 
02968 void SelectionDAGBuilder::visitPtrToInt(const User &I) {
02969   // What to do depends on the size of the integer and the size of the pointer.
02970   // We can either truncate, zero extend, or no-op, accordingly.
02971   SDValue N = getValue(I.getOperand(0));
02972   EVT DestVT = TM.getTargetLowering()->getValueType(I.getType());
02973   setValue(&I, DAG.getZExtOrTrunc(N, getCurSDLoc(), DestVT));
02974 }
02975 
02976 void SelectionDAGBuilder::visitIntToPtr(const User &I) {
02977   // What to do depends on the size of the integer and the size of the pointer.
02978   // We can either truncate, zero extend, or no-op, accordingly.
02979   SDValue N = getValue(I.getOperand(0));
02980   EVT DestVT = TM.getTargetLowering()->getValueType(I.getType());
02981   setValue(&I, DAG.getZExtOrTrunc(N, getCurSDLoc(), DestVT));
02982 }
02983 
02984 void SelectionDAGBuilder::visitBitCast(const User &I) {
02985   SDValue N = getValue(I.getOperand(0));
02986   EVT DestVT = TM.getTargetLowering()->getValueType(I.getType());
02987 
02988   // BitCast assures us that source and destination are the same size so this is
02989   // either a BITCAST or a no-op.
02990   if (DestVT != N.getValueType())
02991     setValue(&I, DAG.getNode(ISD::BITCAST, getCurSDLoc(),
02992                              DestVT, N)); // convert types.
02993   // Check if the original LLVM IR Operand was a ConstantInt, because getValue()
02994   // might fold any kind of constant expression to an integer constant and that
02995   // is not what we are looking for. Only regcognize a bitcast of a genuine
02996   // constant integer as an opaque constant.
02997   else if(ConstantInt *C = dyn_cast<ConstantInt>(I.getOperand(0)))
02998     setValue(&I, DAG.getConstant(C->getValue(), DestVT, /*isTarget=*/false,
02999                                  /*isOpaque*/true));
03000   else
03001     setValue(&I, N);            // noop cast.
03002 }
03003 
03004 void SelectionDAGBuilder::visitAddrSpaceCast(const User &I) {
03005   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
03006   const Value *SV = I.getOperand(0);
03007   SDValue N = getValue(SV);
03008   EVT DestVT = TM.getTargetLowering()->getValueType(I.getType());
03009 
03010   unsigned SrcAS = SV->getType()->getPointerAddressSpace();
03011   unsigned DestAS = I.getType()->getPointerAddressSpace();
03012 
03013   if (!TLI.isNoopAddrSpaceCast(SrcAS, DestAS))
03014     N = DAG.getAddrSpaceCast(getCurSDLoc(), DestVT, N, SrcAS, DestAS);
03015 
03016   setValue(&I, N);
03017 }
03018 
03019 void SelectionDAGBuilder::visitInsertElement(const User &I) {
03020   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
03021   SDValue InVec = getValue(I.getOperand(0));
03022   SDValue InVal = getValue(I.getOperand(1));
03023   SDValue InIdx = DAG.getSExtOrTrunc(getValue(I.getOperand(2)),
03024                                      getCurSDLoc(), TLI.getVectorIdxTy());
03025   setValue(&I, DAG.getNode(ISD::INSERT_VECTOR_ELT, getCurSDLoc(),
03026                            TM.getTargetLowering()->getValueType(I.getType()),
03027                            InVec, InVal, InIdx));
03028 }
03029 
03030 void SelectionDAGBuilder::visitExtractElement(const User &I) {
03031   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
03032   SDValue InVec = getValue(I.getOperand(0));
03033   SDValue InIdx = DAG.getSExtOrTrunc(getValue(I.getOperand(1)),
03034                                      getCurSDLoc(), TLI.getVectorIdxTy());
03035   setValue(&I, DAG.getNode(ISD::EXTRACT_VECTOR_ELT, getCurSDLoc(),
03036                            TM.getTargetLowering()->getValueType(I.getType()),
03037                            InVec, InIdx));
03038 }
03039 
03040 // Utility for visitShuffleVector - Return true if every element in Mask,
03041 // beginning from position Pos and ending in Pos+Size, falls within the
03042 // specified sequential range [L, L+Pos). or is undef.
03043 static bool isSequentialInRange(const SmallVectorImpl<int> &Mask,
03044                                 unsigned Pos, unsigned Size, int Low) {
03045   for (unsigned i = Pos, e = Pos+Size; i != e; ++i, ++Low)
03046     if (Mask[i] >= 0 && Mask[i] != Low)
03047       return false;
03048   return true;
03049 }
03050 
03051 void SelectionDAGBuilder::visitShuffleVector(const User &I) {
03052   SDValue Src1 = getValue(I.getOperand(0));
03053   SDValue Src2 = getValue(I.getOperand(1));
03054 
03055   SmallVector<int, 8> Mask;
03056   ShuffleVectorInst::getShuffleMask(cast<Constant>(I.getOperand(2)), Mask);
03057   unsigned MaskNumElts = Mask.size();
03058 
03059   const TargetLowering *TLI = TM.getTargetLowering();
03060   EVT VT = TLI->getValueType(I.getType());
03061   EVT SrcVT = Src1.getValueType();
03062   unsigned SrcNumElts = SrcVT.getVectorNumElements();
03063 
03064   if (SrcNumElts == MaskNumElts) {
03065     setValue(&I, DAG.getVectorShuffle(VT, getCurSDLoc(), Src1, Src2,
03066                                       &Mask[0]));
03067     return;
03068   }
03069 
03070   // Normalize the shuffle vector since mask and vector length don't match.
03071   if (SrcNumElts < MaskNumElts && MaskNumElts % SrcNumElts == 0) {
03072     // Mask is longer than the source vectors and is a multiple of the source
03073     // vectors.  We can use concatenate vector to make the mask and vectors
03074     // lengths match.
03075     if (SrcNumElts*2 == MaskNumElts) {
03076       // First check for Src1 in low and Src2 in high
03077       if (isSequentialInRange(Mask, 0, SrcNumElts, 0) &&
03078           isSequentialInRange(Mask, SrcNumElts, SrcNumElts, SrcNumElts)) {
03079         // The shuffle is concatenating two vectors together.
03080         setValue(&I, DAG.getNode(ISD::CONCAT_VECTORS, getCurSDLoc(),
03081                                  VT, Src1, Src2));
03082         return;
03083       }
03084       // Then check for Src2 in low and Src1 in high
03085       if (isSequentialInRange(Mask, 0, SrcNumElts, SrcNumElts) &&
03086           isSequentialInRange(Mask, SrcNumElts, SrcNumElts, 0)) {
03087         // The shuffle is concatenating two vectors together.
03088         setValue(&I, DAG.getNode(ISD::CONCAT_VECTORS, getCurSDLoc(),
03089                                  VT, Src2, Src1));
03090         return;
03091       }
03092     }
03093 
03094     // Pad both vectors with undefs to make them the same length as the mask.
03095     unsigned NumConcat = MaskNumElts / SrcNumElts;
03096     bool Src1U = Src1.getOpcode() == ISD::UNDEF;
03097     bool Src2U = Src2.getOpcode() == ISD::UNDEF;
03098     SDValue UndefVal = DAG.getUNDEF(SrcVT);
03099 
03100     SmallVector<SDValue, 8> MOps1(NumConcat, UndefVal);
03101     SmallVector<SDValue, 8> MOps2(NumConcat, UndefVal);
03102     MOps1[0] = Src1;
03103     MOps2[0] = Src2;
03104 
03105     Src1 = Src1U ? DAG.getUNDEF(VT) : DAG.getNode(ISD::CONCAT_VECTORS,
03106                                                   getCurSDLoc(), VT,
03107                                                   &MOps1[0], NumConcat);
03108     Src2 = Src2U ? DAG.getUNDEF(VT) : DAG.getNode(ISD::CONCAT_VECTORS,
03109                                                   getCurSDLoc(), VT,
03110                                                   &MOps2[0], NumConcat);
03111 
03112     // Readjust mask for new input vector length.
03113     SmallVector<int, 8> MappedOps;
03114     for (unsigned i = 0; i != MaskNumElts; ++i) {
03115       int Idx = Mask[i];
03116       if (Idx >= (int)SrcNumElts)
03117         Idx -= SrcNumElts - MaskNumElts;
03118       MappedOps.push_back(Idx);
03119     }
03120 
03121     setValue(&I, DAG.getVectorShuffle(VT, getCurSDLoc(), Src1, Src2,
03122                                       &MappedOps[0]));
03123     return;
03124   }
03125 
03126   if (SrcNumElts > MaskNumElts) {
03127     // Analyze the access pattern of the vector to see if we can extract
03128     // two subvectors and do the shuffle. The analysis is done by calculating
03129     // the range of elements the mask access on both vectors.
03130     int MinRange[2] = { static_cast<int>(SrcNumElts),
03131                         static_cast<int>(SrcNumElts)};
03132     int MaxRange[2] = {-1, -1};
03133 
03134     for (unsigned i = 0; i != MaskNumElts; ++i) {
03135       int Idx = Mask[i];
03136       unsigned Input = 0;
03137       if (Idx < 0)
03138         continue;
03139 
03140       if (Idx >= (int)SrcNumElts) {
03141         Input = 1;
03142         Idx -= SrcNumElts;
03143       }
03144       if (Idx > MaxRange[Input])
03145         MaxRange[Input] = Idx;
03146       if (Idx < MinRange[Input])
03147         MinRange[Input] = Idx;
03148     }
03149 
03150     // Check if the access is smaller than the vector size and can we find
03151     // a reasonable extract index.
03152     int RangeUse[2] = { -1, -1 };  // 0 = Unused, 1 = Extract, -1 = Can not
03153                                    // Extract.
03154     int StartIdx[2];  // StartIdx to extract from
03155     for (unsigned Input = 0; Input < 2; ++Input) {
03156       if (MinRange[Input] >= (int)SrcNumElts && MaxRange[Input] < 0) {
03157         RangeUse[Input] = 0; // Unused
03158         StartIdx[Input] = 0;
03159         continue;
03160       }
03161 
03162       // Find a good start index that is a multiple of the mask length. Then
03163       // see if the rest of the elements are in range.
03164       StartIdx[Input] = (MinRange[Input]/MaskNumElts)*MaskNumElts;
03165       if (MaxRange[Input] - StartIdx[Input] < (int)MaskNumElts &&
03166           StartIdx[Input] + MaskNumElts <= SrcNumElts)
03167         RangeUse[Input] = 1; // Extract from a multiple of the mask length.
03168     }
03169 
03170     if (RangeUse[0] == 0 && RangeUse[1] == 0) {
03171       setValue(&I, DAG.getUNDEF(VT)); // Vectors are not used.
03172       return;
03173     }
03174     if (RangeUse[0] >= 0 && RangeUse[1] >= 0) {
03175       // Extract appropriate subvector and generate a vector shuffle
03176       for (unsigned Input = 0; Input < 2; ++Input) {
03177         SDValue &Src = Input == 0 ? Src1 : Src2;
03178         if (RangeUse[Input] == 0)
03179           Src = DAG.getUNDEF(VT);
03180         else
03181           Src = DAG.getNode(ISD::EXTRACT_SUBVECTOR, getCurSDLoc(), VT,
03182                             Src, DAG.getConstant(StartIdx[Input],
03183                                                  TLI->getVectorIdxTy()));
03184       }
03185 
03186       // Calculate new mask.
03187       SmallVector<int, 8> MappedOps;
03188       for (unsigned i = 0; i != MaskNumElts; ++i) {
03189         int Idx = Mask[i];
03190         if (Idx >= 0) {
03191           if (Idx < (int)SrcNumElts)
03192             Idx -= StartIdx[0];
03193           else
03194             Idx -= SrcNumElts + StartIdx[1] - MaskNumElts;
03195         }
03196         MappedOps.push_back(Idx);
03197       }
03198 
03199       setValue(&I, DAG.getVectorShuffle(VT, getCurSDLoc(), Src1, Src2,
03200                                         &MappedOps[0]));
03201       return;
03202     }
03203   }
03204 
03205   // We can't use either concat vectors or extract subvectors so fall back to
03206   // replacing the shuffle with extract and build vector.
03207   // to insert and build vector.
03208   EVT EltVT = VT.getVectorElementType();
03209   EVT IdxVT = TLI->getVectorIdxTy();
03210   SmallVector<SDValue,8> Ops;
03211   for (unsigned i = 0; i != MaskNumElts; ++i) {
03212     int Idx = Mask[i];
03213     SDValue Res;
03214 
03215     if (Idx < 0) {
03216       Res = DAG.getUNDEF(EltVT);
03217     } else {
03218       SDValue &Src = Idx < (int)SrcNumElts ? Src1 : Src2;
03219       if (Idx >= (int)SrcNumElts) Idx -= SrcNumElts;
03220 
03221       Res = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, getCurSDLoc(),
03222                         EltVT, Src, DAG.getConstant(Idx, IdxVT));
03223     }
03224 
03225     Ops.push_back(Res);
03226   }
03227 
03228   setValue(&I, DAG.getNode(ISD::BUILD_VECTOR, getCurSDLoc(),
03229                            VT, &Ops[0], Ops.size()));
03230 }
03231 
03232 void SelectionDAGBuilder::visitInsertValue(const InsertValueInst &I) {
03233   const Value *Op0 = I.getOperand(0);
03234   const Value *Op1 = I.getOperand(1);
03235   Type *AggTy = I.getType();
03236   Type *ValTy = Op1->getType();
03237   bool IntoUndef = isa<UndefValue>(Op0);
03238   bool FromUndef = isa<UndefValue>(Op1);
03239 
03240   unsigned LinearIndex = ComputeLinearIndex(AggTy, I.getIndices());
03241 
03242   const TargetLowering *TLI = TM.getTargetLowering();
03243   SmallVector<EVT, 4> AggValueVTs;
03244   ComputeValueVTs(*TLI, AggTy, AggValueVTs);
03245   SmallVector<EVT, 4> ValValueVTs;
03246   ComputeValueVTs(*TLI, ValTy, ValValueVTs);
03247 
03248   unsigned NumAggValues = AggValueVTs.size();
03249   unsigned NumValValues = ValValueVTs.size();
03250   SmallVector<SDValue, 4> Values(NumAggValues);
03251 
03252   SDValue Agg = getValue(Op0);
03253   unsigned i = 0;
03254   // Copy the beginning value(s) from the original aggregate.
03255   for (; i != LinearIndex; ++i)
03256     Values[i] = IntoUndef ? DAG.getUNDEF(AggValueVTs[i]) :
03257                 SDValue(Agg.getNode(), Agg.getResNo() + i);
03258   // Copy values from the inserted value(s).
03259   if (NumValValues) {
03260     SDValue Val = getValue(Op1);
03261     for (; i != LinearIndex + NumValValues; ++i)
03262       Values[i] = FromUndef ? DAG.getUNDEF(AggValueVTs[i]) :
03263                   SDValue(Val.getNode(), Val.getResNo() + i - LinearIndex);
03264   }
03265   // Copy remaining value(s) from the original aggregate.
03266   for (; i != NumAggValues; ++i)
03267     Values[i] = IntoUndef ? DAG.getUNDEF(AggValueVTs[i]) :
03268                 SDValue(Agg.getNode(), Agg.getResNo() + i);
03269 
03270   setValue(&I, DAG.getNode(ISD::MERGE_VALUES, getCurSDLoc(),
03271                            DAG.getVTList(AggValueVTs),
03272                            &Values[0], NumAggValues));
03273 }
03274 
03275 void SelectionDAGBuilder::visitExtractValue(const ExtractValueInst &I) {
03276   const Value *Op0 = I.getOperand(0);
03277   Type *AggTy = Op0->getType();
03278   Type *ValTy = I.getType();
03279   bool OutOfUndef = isa<UndefValue>(Op0);
03280 
03281   unsigned LinearIndex = ComputeLinearIndex(AggTy, I.getIndices());
03282 
03283   const TargetLowering *TLI = TM.getTargetLowering();
03284   SmallVector<EVT, 4> ValValueVTs;
03285   ComputeValueVTs(*TLI, ValTy, ValValueVTs);
03286 
03287   unsigned NumValValues = ValValueVTs.size();
03288 
03289   // Ignore a extractvalue that produces an empty object
03290   if (!NumValValues) {
03291     setValue(&I, DAG.getUNDEF(MVT(MVT::Other)));
03292     return;
03293   }
03294 
03295   SmallVector<SDValue, 4> Values(NumValValues);
03296 
03297   SDValue Agg = getValue(Op0);
03298   // Copy out the selected value(s).
03299   for (unsigned i = LinearIndex; i != LinearIndex + NumValValues; ++i)
03300     Values[i - LinearIndex] =
03301       OutOfUndef ?
03302         DAG.getUNDEF(Agg.getNode()->getValueType(Agg.getResNo() + i)) :
03303         SDValue(Agg.getNode(), Agg.getResNo() + i);
03304 
03305   setValue(&I, DAG.getNode(ISD::MERGE_VALUES, getCurSDLoc(),
03306                            DAG.getVTList(ValValueVTs),
03307                            &Values[0], NumValValues));
03308 }
03309 
03310 void SelectionDAGBuilder::visitGetElementPtr(const User &I) {
03311   Value *Op0 = I.getOperand(0);
03312   // Note that the pointer operand may be a vector of pointers. Take the scalar
03313   // element which holds a pointer.
03314   Type *Ty = Op0->getType()->getScalarType();
03315   unsigned AS = Ty->getPointerAddressSpace();
03316   SDValue N = getValue(Op0);
03317 
03318   for (GetElementPtrInst::const_op_iterator OI = I.op_begin()+1, E = I.op_end();
03319        OI != E; ++OI) {
03320     const Value *Idx = *OI;
03321     if (StructType *StTy = dyn_cast<StructType>(Ty)) {
03322       unsigned Field = cast<Constant>(Idx)->getUniqueInteger().getZExtValue();
03323       if (Field) {
03324         // N = N + Offset
03325         uint64_t Offset = DL->getStructLayout(StTy)->getElementOffset(Field);
03326         N = DAG.getNode(ISD::ADD, getCurSDLoc(), N.getValueType(), N,
03327                         DAG.getConstant(Offset, N.getValueType()));
03328       }
03329 
03330       Ty = StTy->getElementType(Field);
03331     } else {
03332       Ty = cast<SequentialType>(Ty)->getElementType();
03333 
03334       // If this is a constant subscript, handle it quickly.
03335       const TargetLowering *TLI = TM.getTargetLowering();
03336       if (const ConstantInt *CI = dyn_cast<ConstantInt>(Idx)) {
03337         if (CI->isZero()) continue;
03338         uint64_t Offs =
03339             DL->getTypeAllocSize(Ty)*cast<ConstantInt>(CI)->getSExtValue();
03340         SDValue OffsVal;
03341         EVT PTy = TLI->getPointerTy(AS);
03342         unsigned PtrBits = PTy.getSizeInBits();
03343         if (PtrBits < 64)
03344           OffsVal = DAG.getNode(ISD::TRUNCATE, getCurSDLoc(), PTy,
03345                                 DAG.getConstant(Offs, MVT::i64));
03346         else
03347           OffsVal = DAG.getConstant(Offs, PTy);
03348 
03349         N = DAG.getNode(ISD::ADD, getCurSDLoc(), N.getValueType(), N,
03350                         OffsVal);
03351         continue;
03352       }
03353 
03354       // N = N + Idx * ElementSize;
03355       APInt ElementSize = APInt(TLI->getPointerSizeInBits(AS),
03356                                 DL->getTypeAllocSize(Ty));
03357       SDValue IdxN = getValue(Idx);
03358 
03359       // If the index is smaller or larger than intptr_t, truncate or extend
03360       // it.
03361       IdxN = DAG.getSExtOrTrunc(IdxN, getCurSDLoc(), N.getValueType());
03362 
03363       // If this is a multiply by a power of two, turn it into a shl
03364       // immediately.  This is a very common case.
03365       if (ElementSize != 1) {
03366         if (ElementSize.isPowerOf2()) {
03367           unsigned Amt = ElementSize.logBase2();
03368           IdxN = DAG.getNode(ISD::SHL, getCurSDLoc(),
03369                              N.getValueType(), IdxN,
03370                              DAG.getConstant(Amt, IdxN.getValueType()));
03371         } else {
03372           SDValue Scale = DAG.getConstant(ElementSize, IdxN.getValueType());
03373           IdxN = DAG.getNode(ISD::MUL, getCurSDLoc(),
03374                              N.getValueType(), IdxN, Scale);
03375         }
03376       }
03377 
03378       N = DAG.getNode(ISD::ADD, getCurSDLoc(),
03379                       N.getValueType(), N, IdxN);
03380     }
03381   }
03382 
03383   setValue(&I, N);
03384 }
03385 
03386 void SelectionDAGBuilder::visitAlloca(const AllocaInst &I) {
03387   // If this is a fixed sized alloca in the entry block of the function,
03388   // allocate it statically on the stack.
03389   if (FuncInfo.StaticAllocaMap.count(&I))
03390     return;   // getValue will auto-populate this.
03391 
03392   Type *Ty = I.getAllocatedType();
03393   const TargetLowering *TLI = TM.getTargetLowering();
03394   uint64_t TySize = TLI->getDataLayout()->getTypeAllocSize(Ty);
03395   unsigned Align =
03396     std::max((unsigned)TLI->getDataLayout()->getPrefTypeAlignment(Ty),
03397              I.getAlignment());
03398 
03399   SDValue AllocSize = getValue(I.getArraySize());
03400 
03401   EVT IntPtr = TLI->getPointerTy();
03402   if (AllocSize.getValueType() != IntPtr)
03403     AllocSize = DAG.getZExtOrTrunc(AllocSize, getCurSDLoc(), IntPtr);
03404 
03405   AllocSize = DAG.getNode(ISD::MUL, getCurSDLoc(), IntPtr,
03406                           AllocSize,
03407                           DAG.getConstant(TySize, IntPtr));
03408 
03409   // Handle alignment.  If the requested alignment is less than or equal to
03410   // the stack alignment, ignore it.  If the size is greater than or equal to
03411   // the stack alignment, we note this in the DYNAMIC_STACKALLOC node.
03412   unsigned StackAlign = TM.getFrameLowering()->getStackAlignment();
03413   if (Align <= StackAlign)
03414     Align = 0;
03415 
03416   // Round the size of the allocation up to the stack alignment size
03417   // by add SA-1 to the size.
03418   AllocSize = DAG.getNode(ISD::ADD, getCurSDLoc(),
03419                           AllocSize.getValueType(), AllocSize,
03420                           DAG.getIntPtrConstant(StackAlign-1));
03421 
03422   // Mask out the low bits for alignment purposes.
03423   AllocSize = DAG.getNode(ISD::AND, getCurSDLoc(),
03424                           AllocSize.getValueType(), AllocSize,
03425                           DAG.getIntPtrConstant(~(uint64_t)(StackAlign-1)));
03426 
03427   SDValue Ops[] = { getRoot(), AllocSize, DAG.getIntPtrConstant(Align) };
03428   SDVTList VTs = DAG.getVTList(AllocSize.getValueType(), MVT::Other);
03429   SDValue DSA = DAG.getNode(ISD::DYNAMIC_STACKALLOC, getCurSDLoc(),
03430                             VTs, Ops, 3);
03431   setValue(&I, DSA);
03432   DAG.setRoot(DSA.getValue(1));
03433 
03434   assert(FuncInfo.MF->getFrameInfo()->hasVarSizedObjects());
03435 }
03436 
03437 void SelectionDAGBuilder::visitLoad(const LoadInst &I) {
03438   if (I.isAtomic())
03439     return visitAtomicLoad(I);
03440 
03441   const Value *SV = I.getOperand(0);
03442   SDValue Ptr = getValue(SV);
03443 
03444   Type *Ty = I.getType();
03445 
03446   bool isVolatile = I.isVolatile();
03447   bool isNonTemporal = I.getMetadata("nontemporal") != nullptr;
03448   bool isInvariant = I.getMetadata("invariant.load") != nullptr;
03449   unsigned Alignment = I.getAlignment();
03450   const MDNode *TBAAInfo = I.getMetadata(LLVMContext::MD_tbaa);
03451   const MDNode *Ranges = I.getMetadata(LLVMContext::MD_range);
03452 
03453   SmallVector<EVT, 4> ValueVTs;
03454   SmallVector<uint64_t, 4> Offsets;
03455   ComputeValueVTs(*TM.getTargetLowering(), Ty, ValueVTs, &Offsets);
03456   unsigned NumValues = ValueVTs.size();
03457   if (NumValues == 0)
03458     return;
03459 
03460   SDValue Root;
03461   bool ConstantMemory = false;
03462   if (isVolatile || NumValues > MaxParallelChains)
03463     // Serialize volatile loads with other side effects.
03464     Root = getRoot();
03465   else if (AA->pointsToConstantMemory(
03466              AliasAnalysis::Location(SV, AA->getTypeStoreSize(Ty), TBAAInfo))) {
03467     // Do not serialize (non-volatile) loads of constant memory with anything.
03468     Root = DAG.getEntryNode();
03469     ConstantMemory = true;
03470   } else {
03471     // Do not serialize non-volatile loads against each other.
03472     Root = DAG.getRoot();
03473   }
03474 
03475   const TargetLowering *TLI = TM.getTargetLowering();
03476   if (isVolatile)
03477     Root = TLI->prepareVolatileOrAtomicLoad(Root, getCurSDLoc(), DAG);
03478 
03479   SmallVector<SDValue, 4> Values(NumValues);
03480   SmallVector<SDValue, 4> Chains(std::min(unsigned(MaxParallelChains),
03481                                           NumValues));
03482   EVT PtrVT = Ptr.getValueType();
03483   unsigned ChainI = 0;
03484   for (unsigned i = 0; i != NumValues; ++i, ++ChainI) {
03485     // Serializing loads here may result in excessive register pressure, and
03486     // TokenFactor places arbitrary choke points on the scheduler. SD scheduling
03487     // could recover a bit by hoisting nodes upward in the chain by recognizing
03488     // they are side-effect free or do not alias. The optimizer should really
03489     // avoid this case by converting large object/array copies to llvm.memcpy
03490     // (MaxParallelChains should always remain as failsafe).
03491     if (ChainI == MaxParallelChains) {
03492       assert(PendingLoads.empty() && "PendingLoads must be serialized first");
03493       SDValue Chain = DAG.getNode(ISD::TokenFactor, getCurSDLoc(),
03494                                   MVT::Other, &Chains[0], ChainI);
03495       Root = Chain;
03496       ChainI = 0;
03497     }
03498     SDValue A = DAG.getNode(ISD::ADD, getCurSDLoc(),
03499                             PtrVT, Ptr,
03500                             DAG.getConstant(Offsets[i], PtrVT));
03501     SDValue L = DAG.getLoad(ValueVTs[i], getCurSDLoc(), Root,
03502                             A, MachinePointerInfo(SV, Offsets[i]), isVolatile,
03503                             isNonTemporal, isInvariant, Alignment, TBAAInfo,
03504                             Ranges);
03505 
03506     Values[i] = L;
03507     Chains[ChainI] = L.getValue(1);
03508   }
03509 
03510   if (!ConstantMemory) {
03511     SDValue Chain = DAG.getNode(ISD::TokenFactor, getCurSDLoc(),
03512                                 MVT::Other, &Chains[0], ChainI);
03513     if (isVolatile)
03514       DAG.setRoot(Chain);
03515     else
03516       PendingLoads.push_back(Chain);
03517   }
03518 
03519   setValue(&I, DAG.getNode(ISD::MERGE_VALUES, getCurSDLoc(),
03520                            DAG.getVTList(ValueVTs),
03521                            &Values[0], NumValues));
03522 }
03523 
03524 void SelectionDAGBuilder::visitStore(const StoreInst &I) {
03525   if (I.isAtomic())
03526     return visitAtomicStore(I);
03527 
03528   const Value *SrcV = I.getOperand(0);
03529   const Value *PtrV = I.getOperand(1);
03530 
03531   SmallVector<EVT, 4> ValueVTs;
03532   SmallVector<uint64_t, 4> Offsets;
03533   ComputeValueVTs(*TM.getTargetLowering(), SrcV->getType(), ValueVTs, &Offsets);
03534   unsigned NumValues = ValueVTs.size();
03535   if (NumValues == 0)
03536     return;
03537 
03538   // Get the lowered operands. Note that we do this after
03539   // checking if NumResults is zero, because with zero results
03540   // the operands won't have values in the map.
03541   SDValue Src = getValue(SrcV);
03542   SDValue Ptr = getValue(PtrV);
03543 
03544   SDValue Root = getRoot();
03545   SmallVector<SDValue, 4> Chains(std::min(unsigned(MaxParallelChains),
03546                                           NumValues));
03547   EVT PtrVT = Ptr.getValueType();
03548   bool isVolatile = I.isVolatile();
03549   bool isNonTemporal = I.getMetadata("nontemporal") != nullptr;
03550   unsigned Alignment = I.getAlignment();
03551   const MDNode *TBAAInfo = I.getMetadata(LLVMContext::MD_tbaa);
03552 
03553   unsigned ChainI = 0;
03554   for (unsigned i = 0; i != NumValues; ++i, ++ChainI) {
03555     // See visitLoad comments.
03556     if (ChainI == MaxParallelChains) {
03557       SDValue Chain = DAG.getNode(ISD::TokenFactor, getCurSDLoc(),
03558                                   MVT::Other, &Chains[0], ChainI);
03559       Root = Chain;
03560       ChainI = 0;
03561     }
03562     SDValue Add = DAG.getNode(ISD::ADD, getCurSDLoc(), PtrVT, Ptr,
03563                               DAG.getConstant(Offsets[i], PtrVT));
03564     SDValue St = DAG.getStore(Root, getCurSDLoc(),
03565                               SDValue(Src.getNode(), Src.getResNo() + i),
03566                               Add, MachinePointerInfo(PtrV, Offsets[i]),
03567                               isVolatile, isNonTemporal, Alignment, TBAAInfo);
03568     Chains[ChainI] = St;
03569   }
03570 
03571   SDValue StoreNode = DAG.getNode(ISD::TokenFactor, getCurSDLoc(),
03572                                   MVT::Other, &Chains[0], ChainI);
03573   DAG.setRoot(StoreNode);
03574 }
03575 
03576 static SDValue InsertFenceForAtomic(SDValue Chain, AtomicOrdering Order,
03577                                     SynchronizationScope Scope,
03578                                     bool Before, SDLoc dl,
03579                                     SelectionDAG &DAG,
03580                                     const TargetLowering &TLI) {
03581   // Fence, if necessary
03582   if (Before) {
03583     if (Order == AcquireRelease || Order == SequentiallyConsistent)
03584       Order = Release;
03585     else if (Order == Acquire || Order == Monotonic)
03586       return Chain;
03587   } else {
03588     if (Order == AcquireRelease)
03589       Order = Acquire;
03590     else if (Order == Release || Order == Monotonic)
03591       return Chain;
03592   }
03593   SDValue Ops[3];
03594   Ops[0] = Chain;
03595   Ops[1] = DAG.getConstant(Order, TLI.getPointerTy());
03596   Ops[2] = DAG.getConstant(Scope, TLI.getPointerTy());
03597   return DAG.getNode(ISD::ATOMIC_FENCE, dl, MVT::Other, Ops, 3);
03598 }
03599 
03600 void SelectionDAGBuilder::visitAtomicCmpXchg(const AtomicCmpXchgInst &I) {
03601   SDLoc dl = getCurSDLoc();
03602   AtomicOrdering SuccessOrder = I.getSuccessOrdering();
03603   AtomicOrdering FailureOrder = I.getFailureOrdering();
03604   SynchronizationScope Scope = I.getSynchScope();
03605 
03606   SDValue InChain = getRoot();
03607 
03608   const TargetLowering *TLI = TM.getTargetLowering();
03609   if (TLI->getInsertFencesForAtomic())
03610     InChain = InsertFenceForAtomic(InChain, SuccessOrder, Scope, true, dl,
03611                                    DAG, *TLI);
03612 
03613   SDValue L =
03614     DAG.getAtomic(ISD::ATOMIC_CMP_SWAP, dl,
03615                   getValue(I.getCompareOperand()).getSimpleValueType(),
03616                   InChain,
03617                   getValue(I.getPointerOperand()),
03618                   getValue(I.getCompareOperand()),
03619                   getValue(I.getNewValOperand()),
03620                   MachinePointerInfo(I.getPointerOperand()), 0 /* Alignment */,
03621                   TLI->getInsertFencesForAtomic() ? Monotonic : SuccessOrder,
03622                   TLI->getInsertFencesForAtomic() ? Monotonic : FailureOrder,
03623                   Scope);
03624 
03625   SDValue OutChain = L.getValue(1);
03626 
03627   if (TLI->getInsertFencesForAtomic())
03628     OutChain = InsertFenceForAtomic(OutChain, SuccessOrder, Scope, false, dl,
03629                                     DAG, *TLI);
03630 
03631   setValue(&I, L);
03632   DAG.setRoot(OutChain);
03633 }
03634 
03635 void SelectionDAGBuilder::visitAtomicRMW(const AtomicRMWInst &I) {
03636   SDLoc dl = getCurSDLoc();
03637   ISD::NodeType NT;
03638   switch (I.getOperation()) {
03639   default: llvm_unreachable("Unknown atomicrmw operation");
03640   case AtomicRMWInst::Xchg: NT = ISD::ATOMIC_SWAP; break;
03641   case AtomicRMWInst::Add:  NT = ISD::ATOMIC_LOAD_ADD; break;
03642   case AtomicRMWInst::Sub:  NT = ISD::ATOMIC_LOAD_SUB; break;
03643   case AtomicRMWInst::And:  NT = ISD::ATOMIC_LOAD_AND; break;
03644   case AtomicRMWInst::Nand: NT = ISD::ATOMIC_LOAD_NAND; break;
03645   case AtomicRMWInst::Or:   NT = ISD::ATOMIC_LOAD_OR; break;
03646   case AtomicRMWInst::Xor:  NT = ISD::ATOMIC_LOAD_XOR; break;
03647   case AtomicRMWInst::Max:  NT = ISD::ATOMIC_LOAD_MAX; break;
03648   case AtomicRMWInst::Min:  NT = ISD::ATOMIC_LOAD_MIN; break;
03649   case AtomicRMWInst::UMax: NT = ISD::ATOMIC_LOAD_UMAX; break;
03650   case AtomicRMWInst::UMin: NT = ISD::ATOMIC_LOAD_UMIN; break;
03651   }
03652   AtomicOrdering Order = I.getOrdering();
03653   SynchronizationScope Scope = I.getSynchScope();
03654 
03655   SDValue InChain = getRoot();
03656 
03657   const TargetLowering *TLI = TM.getTargetLowering();
03658   if (TLI->getInsertFencesForAtomic())
03659     InChain = InsertFenceForAtomic(InChain, Order, Scope, true, dl,
03660                                    DAG, *TLI);
03661 
03662   SDValue L =
03663     DAG.getAtomic(NT, dl,
03664                   getValue(I.getValOperand()).getSimpleValueType(),
03665                   InChain,
03666                   getValue(I.getPointerOperand()),
03667                   getValue(I.getValOperand()),
03668                   I.getPointerOperand(), 0 /* Alignment */,
03669                   TLI->getInsertFencesForAtomic() ? Monotonic : Order,
03670                   Scope);
03671 
03672   SDValue OutChain = L.getValue(1);
03673 
03674   if (TLI->getInsertFencesForAtomic())
03675     OutChain = InsertFenceForAtomic(OutChain, Order, Scope, false, dl,
03676                                     DAG, *TLI);
03677 
03678   setValue(&I, L);
03679   DAG.setRoot(OutChain);
03680 }
03681 
03682 void SelectionDAGBuilder::visitFence(const FenceInst &I) {
03683   SDLoc dl = getCurSDLoc();
03684   const TargetLowering *TLI = TM.getTargetLowering();
03685   SDValue Ops[3];
03686   Ops[0] = getRoot();
03687   Ops[1] = DAG.getConstant(I.getOrdering(), TLI->getPointerTy());
03688   Ops[2] = DAG.getConstant(I.getSynchScope(), TLI->getPointerTy());
03689   DAG.setRoot(DAG.getNode(ISD::ATOMIC_FENCE, dl, MVT::Other, Ops, 3));
03690 }
03691 
03692 void SelectionDAGBuilder::visitAtomicLoad(const LoadInst &I) {
03693   SDLoc dl = getCurSDLoc();
03694   AtomicOrdering Order = I.getOrdering();
03695   SynchronizationScope Scope = I.getSynchScope();
03696 
03697   SDValue InChain = getRoot();
03698 
03699   const TargetLowering *TLI = TM.getTargetLowering();
03700   EVT VT = TLI->getValueType(I.getType());
03701 
03702   if (I.getAlignment() < VT.getSizeInBits() / 8)
03703     report_fatal_error("Cannot generate unaligned atomic load");
03704 
03705   MachineMemOperand *MMO =
03706       DAG.getMachineFunction().
03707       getMachineMemOperand(MachinePointerInfo(I.getPointerOperand()),
03708                            MachineMemOperand::MOVolatile |
03709                            MachineMemOperand::MOLoad,
03710                            VT.getStoreSize(),
03711                            I.getAlignment() ? I.getAlignment() :
03712                                               DAG.getEVTAlignment(VT));
03713 
03714   InChain = TLI->prepareVolatileOrAtomicLoad(InChain, dl, DAG);
03715   SDValue L =
03716       DAG.getAtomic(ISD::ATOMIC_LOAD, dl, VT, VT, InChain,
03717                     getValue(I.getPointerOperand()), MMO,
03718                     TLI->getInsertFencesForAtomic() ? Monotonic : Order,
03719                     Scope);
03720 
03721   SDValue OutChain = L.getValue(1);
03722 
03723   if (TLI->getInsertFencesForAtomic())
03724     OutChain = InsertFenceForAtomic(OutChain, Order, Scope, false, dl,
03725                                     DAG, *TLI);
03726 
03727   setValue(&I, L);
03728   DAG.setRoot(OutChain);
03729 }
03730 
03731 void SelectionDAGBuilder::visitAtomicStore(const StoreInst &I) {
03732   SDLoc dl = getCurSDLoc();
03733 
03734   AtomicOrdering Order = I.getOrdering();
03735   SynchronizationScope Scope = I.getSynchScope();
03736 
03737   SDValue InChain = getRoot();
03738 
03739   const TargetLowering *TLI = TM.getTargetLowering();
03740   EVT VT = TLI->getValueType(I.getValueOperand()->getType());
03741 
03742   if (I.getAlignment() < VT.getSizeInBits() / 8)
03743     report_fatal_error("Cannot generate unaligned atomic store");
03744 
03745   if (TLI->getInsertFencesForAtomic())
03746     InChain = InsertFenceForAtomic(InChain, Order, Scope, true, dl,
03747                                    DAG, *TLI);
03748 
03749   SDValue OutChain =
03750     DAG.getAtomic(ISD::ATOMIC_STORE, dl, VT,
03751                   InChain,
03752                   getValue(I.getPointerOperand()),
03753                   getValue(I.getValueOperand()),
03754                   I.getPointerOperand(), I.getAlignment(),
03755                   TLI->getInsertFencesForAtomic() ? Monotonic : Order,
03756                   Scope);
03757 
03758   if (TLI->getInsertFencesForAtomic())
03759     OutChain = InsertFenceForAtomic(OutChain, Order, Scope, false, dl,
03760                                     DAG, *TLI);
03761 
03762   DAG.setRoot(OutChain);
03763 }
03764 
03765 /// visitTargetIntrinsic - Lower a call of a target intrinsic to an INTRINSIC
03766 /// node.
03767 void SelectionDAGBuilder::visitTargetIntrinsic(const CallInst &I,
03768                                                unsigned Intrinsic) {
03769   bool HasChain = !I.doesNotAccessMemory();
03770   bool OnlyLoad = HasChain && I.onlyReadsMemory();
03771 
03772   // Build the operand list.
03773   SmallVector<SDValue, 8> Ops;
03774   if (HasChain) {  // If this intrinsic has side-effects, chainify it.
03775     if (OnlyLoad) {
03776       // We don't need to serialize loads against other loads.
03777       Ops.push_back(DAG.getRoot());
03778     } else {
03779       Ops.push_back(getRoot());
03780     }
03781   }
03782 
03783   // Info is set by getTgtMemInstrinsic
03784   TargetLowering::IntrinsicInfo Info;
03785   const TargetLowering *TLI = TM.getTargetLowering();
03786   bool IsTgtIntrinsic = TLI->getTgtMemIntrinsic(Info, I, Intrinsic);
03787 
03788   // Add the intrinsic ID as an integer operand if it's not a target intrinsic.
03789   if (!IsTgtIntrinsic || Info.opc == ISD::INTRINSIC_VOID ||
03790       Info.opc == ISD::INTRINSIC_W_CHAIN)
03791     Ops.push_back(DAG.getTargetConstant(Intrinsic, TLI->getPointerTy()));
03792 
03793   // Add all operands of the call to the operand list.
03794   for (unsigned i = 0, e = I.getNumArgOperands(); i != e; ++i) {
03795     SDValue Op = getValue(I.getArgOperand(i));
03796     Ops.push_back(Op);
03797   }
03798 
03799   SmallVector<EVT, 4> ValueVTs;
03800   ComputeValueVTs(*TLI, I.getType(), ValueVTs);
03801 
03802   if (HasChain)
03803     ValueVTs.push_back(MVT::Other);
03804 
03805   SDVTList VTs = DAG.getVTList(ValueVTs);
03806 
03807   // Create the node.
03808   SDValue Result;
03809   if (IsTgtIntrinsic) {
03810     // This is target intrinsic that touches memory
03811     Result = DAG.getMemIntrinsicNode(Info.opc, getCurSDLoc(),
03812                                      VTs, &Ops[0], Ops.size(),
03813                                      Info.memVT,
03814                                    MachinePointerInfo(Info.ptrVal, Info.offset),
03815                                      Info.align, Info.vol,
03816                                      Info.readMem, Info.writeMem);
03817   } else if (!HasChain) {
03818     Result = DAG.getNode(ISD::INTRINSIC_WO_CHAIN, getCurSDLoc(),
03819                          VTs, &Ops[0], Ops.size());
03820   } else if (!I.getType()->isVoidTy()) {
03821     Result = DAG.getNode(ISD::INTRINSIC_W_CHAIN, getCurSDLoc(),
03822                          VTs, &Ops[0], Ops.size());
03823   } else {
03824     Result = DAG.getNode(ISD::INTRINSIC_VOID, getCurSDLoc(),
03825                          VTs, &Ops[0], Ops.size());
03826   }
03827 
03828   if (HasChain) {
03829     SDValue Chain = Result.getValue(Result.getNode()->getNumValues()-1);
03830     if (OnlyLoad)
03831       PendingLoads.push_back(Chain);
03832     else
03833       DAG.setRoot(Chain);
03834   }
03835 
03836   if (!I.getType()->isVoidTy()) {
03837     if (VectorType *PTy = dyn_cast<VectorType>(I.getType())) {
03838       EVT VT = TLI->getValueType(PTy);
03839       Result = DAG.getNode(ISD::BITCAST, getCurSDLoc(), VT, Result);
03840     }
03841 
03842     setValue(&I, Result);
03843   }
03844 }
03845 
03846 /// GetSignificand - Get the significand and build it into a floating-point
03847 /// number with exponent of 1:
03848 ///
03849 ///   Op = (Op & 0x007fffff) | 0x3f800000;
03850 ///
03851 /// where Op is the hexadecimal representation of floating point value.
03852 static SDValue
03853 GetSignificand(SelectionDAG &DAG, SDValue Op, SDLoc dl) {
03854   SDValue t1 = DAG.getNode(ISD::AND, dl, MVT::i32, Op,
03855                            DAG.getConstant(0x007fffff, MVT::i32));
03856   SDValue t2 = DAG.getNode(ISD::OR, dl, MVT::i32, t1,
03857                            DAG.getConstant(0x3f800000, MVT::i32));
03858   return DAG.getNode(ISD::BITCAST, dl, MVT::f32, t2);
03859 }
03860 
03861 /// GetExponent - Get the exponent:
03862 ///
03863 ///   (float)(int)(((Op & 0x7f800000) >> 23) - 127);
03864 ///
03865 /// where Op is the hexadecimal representation of floating point value.
03866 static SDValue
03867 GetExponent(SelectionDAG &DAG, SDValue Op, const TargetLowering &TLI,
03868             SDLoc dl) {
03869   SDValue t0 = DAG.getNode(ISD::AND, dl, MVT::i32, Op,
03870                            DAG.getConstant(0x7f800000, MVT::i32));
03871   SDValue t1 = DAG.getNode(ISD::SRL, dl, MVT::i32, t0,
03872                            DAG.getConstant(23, TLI.getPointerTy()));
03873   SDValue t2 = DAG.getNode(ISD::SUB, dl, MVT::i32, t1,
03874                            DAG.getConstant(127, MVT::i32));
03875   return DAG.getNode(ISD::SINT_TO_FP, dl, MVT::f32, t2);
03876 }
03877 
03878 /// getF32Constant - Get 32-bit floating point constant.
03879 static SDValue
03880 getF32Constant(SelectionDAG &DAG, unsigned Flt) {
03881   return DAG.getConstantFP(APFloat(APFloat::IEEEsingle, APInt(32, Flt)),
03882                            MVT::f32);
03883 }
03884 
03885 /// expandExp - Lower an exp intrinsic. Handles the special sequences for
03886 /// limited-precision mode.
03887 static SDValue expandExp(SDLoc dl, SDValue Op, SelectionDAG &DAG,
03888                          const TargetLowering &TLI) {
03889   if (Op.getValueType() == MVT::f32 &&
03890       LimitFloatPrecision > 0 && LimitFloatPrecision <= 18) {
03891 
03892     // Put the exponent in the right bit position for later addition to the
03893     // final result:
03894     //
03895     //   #define LOG2OFe 1.4426950f
03896     //   IntegerPartOfX = ((int32_t)(X * LOG2OFe));
03897     SDValue t0 = DAG.getNode(ISD::FMUL, dl, MVT::f32, Op,
03898                              getF32Constant(DAG, 0x3fb8aa3b));
03899     SDValue IntegerPartOfX = DAG.getNode(ISD::FP_TO_SINT, dl, MVT::i32, t0);
03900 
03901     //   FractionalPartOfX = (X * LOG2OFe) - (float)IntegerPartOfX;
03902     SDValue t1 = DAG.getNode(ISD::SINT_TO_FP, dl, MVT::f32, IntegerPartOfX);
03903     SDValue X = DAG.getNode(ISD::FSUB, dl, MVT::f32, t0, t1);
03904 
03905     //   IntegerPartOfX <<= 23;
03906     IntegerPartOfX = DAG.getNode(ISD::SHL, dl, MVT::i32, IntegerPartOfX,
03907                                  DAG.getConstant(23, TLI.getPointerTy()));
03908 
03909     SDValue TwoToFracPartOfX;
03910     if (LimitFloatPrecision <= 6) {
03911       // For floating-point precision of 6:
03912       //
03913       //   TwoToFractionalPartOfX =
03914       //     0.997535578f +
03915       //       (0.735607626f + 0.252464424f * x) * x;
03916       //
03917       // error 0.0144103317, which is 6 bits
03918       SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X,
03919                                getF32Constant(DAG, 0x3e814304));
03920       SDValue t3 = DAG.getNode(ISD::FADD, dl, MVT::f32, t2,
03921                                getF32Constant(DAG, 0x3f3c50c8));
03922       SDValue t4 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t3, X);
03923       TwoToFracPartOfX = DAG.getNode(ISD::FADD, dl, MVT::f32, t4,
03924                                      getF32Constant(DAG, 0x3f7f5e7e));
03925     } else if (LimitFloatPrecision <= 12) {
03926       // For floating-point precision of 12:
03927       //
03928       //   TwoToFractionalPartOfX =
03929       //     0.999892986f +
03930       //       (0.696457318f +
03931       //         (0.224338339f + 0.792043434e-1f * x) * x) * x;
03932       //
03933       // 0.000107046256 error, which is 13 to 14 bits
03934       SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X,
03935                                getF32Constant(DAG, 0x3da235e3));
03936       SDValue t3 = DAG.getNode(ISD::FADD, dl, MVT::f32, t2,
03937                                getF32Constant(DAG, 0x3e65b8f3));
03938       SDValue t4 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t3, X);
03939       SDValue t5 = DAG.getNode(ISD::FADD, dl, MVT::f32, t4,
03940                                getF32Constant(DAG, 0x3f324b07));
03941       SDValue t6 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t5, X);
03942       TwoToFracPartOfX = DAG.getNode(ISD::FADD, dl, MVT::f32, t6,
03943                                      getF32Constant(DAG, 0x3f7ff8fd));
03944     } else { // LimitFloatPrecision <= 18
03945       // For floating-point precision of 18:
03946       //
03947       //   TwoToFractionalPartOfX =
03948       //     0.999999982f +
03949       //       (0.693148872f +
03950       //         (0.240227044f +
03951       //           (0.554906021e-1f +
03952       //             (0.961591928e-2f +
03953       //               (0.136028312e-2f + 0.157059148e-3f *x)*x)*x)*x)*x)*x;
03954       //
03955       // error 2.47208000*10^(-7), which is better than 18 bits
03956       SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X,
03957                                getF32Constant(DAG, 0x3924b03e));
03958       SDValue t3 = DAG.getNode(ISD::FADD, dl, MVT::f32, t2,
03959                                getF32Constant(DAG, 0x3ab24b87));
03960       SDValue t4 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t3, X);
03961       SDValue t5 = DAG.getNode(ISD::FADD, dl, MVT::f32, t4,
03962                                getF32Constant(DAG, 0x3c1d8c17));
03963       SDValue t6 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t5, X);
03964       SDValue t7 = DAG.getNode(ISD::FADD, dl, MVT::f32, t6,
03965                                getF32Constant(DAG, 0x3d634a1d));
03966       SDValue t8 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t7, X);
03967       SDValue t9 = DAG.getNode(ISD::FADD, dl, MVT::f32, t8,
03968                                getF32Constant(DAG, 0x3e75fe14));
03969       SDValue t10 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t9, X);
03970       SDValue t11 = DAG.getNode(ISD::FADD, dl, MVT::f32, t10,
03971                                 getF32Constant(DAG, 0x3f317234));
03972       SDValue t12 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t11, X);
03973       TwoToFracPartOfX = DAG.getNode(ISD::FADD, dl, MVT::f32, t12,
03974                                      getF32Constant(DAG, 0x3f800000));
03975     }
03976 
03977     // Add the exponent into the result in integer domain.
03978     SDValue t13 = DAG.getNode(ISD::BITCAST, dl, MVT::i32, TwoToFracPartOfX);
03979     return DAG.getNode(ISD::BITCAST, dl, MVT::f32,
03980                        DAG.getNode(ISD::ADD, dl, MVT::i32,
03981                                    t13, IntegerPartOfX));
03982   }
03983 
03984   // No special expansion.
03985   return DAG.getNode(ISD::FEXP, dl, Op.getValueType(), Op);
03986 }
03987 
03988 /// expandLog - Lower a log intrinsic. Handles the special sequences for
03989 /// limited-precision mode.
03990 static SDValue expandLog(SDLoc dl, SDValue Op, SelectionDAG &DAG,
03991                          const TargetLowering &TLI) {
03992   if (Op.getValueType() == MVT::f32 &&
03993       LimitFloatPrecision > 0 && LimitFloatPrecision <= 18) {
03994     SDValue Op1 = DAG.getNode(ISD::BITCAST, dl, MVT::i32, Op);
03995 
03996     // Scale the exponent by log(2) [0.69314718f].
03997     SDValue Exp = GetExponent(DAG, Op1, TLI, dl);
03998     SDValue LogOfExponent = DAG.getNode(ISD::FMUL, dl, MVT::f32, Exp,
03999                                         getF32Constant(DAG, 0x3f317218));
04000 
04001     // Get the significand and build it into a floating-point number with
04002     // exponent of 1.
04003     SDValue X = GetSignificand(DAG, Op1, dl);
04004 
04005     SDValue LogOfMantissa;
04006     if (LimitFloatPrecision <= 6) {
04007       // For floating-point precision of 6:
04008       //
04009       //   LogofMantissa =
04010       //     -1.1609546f +
04011       //       (1.4034025f - 0.23903021f * x) * x;
04012       //
04013       // error 0.0034276066, which is better than 8 bits
04014       SDValue t0 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X,
04015                                getF32Constant(DAG, 0xbe74c456));
04016       SDValue t1 = DAG.getNode(ISD::FADD, dl, MVT::f32, t0,
04017                                getF32Constant(DAG, 0x3fb3a2b1));
04018       SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t1, X);
04019       LogOfMantissa = DAG.getNode(ISD::FSUB, dl, MVT::f32, t2,
04020                                   getF32Constant(DAG, 0x3f949a29));
04021     } else if (LimitFloatPrecision <= 12) {
04022       // For floating-point precision of 12:
04023       //
04024       //   LogOfMantissa =
04025       //     -1.7417939f +
04026       //       (2.8212026f +
04027       //         (-1.4699568f +
04028       //           (0.44717955f - 0.56570851e-1f * x) * x) * x) * x;
04029       //
04030       // error 0.000061011436, which is 14 bits
04031       SDValue t0 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X,
04032                                getF32Constant(DAG, 0xbd67b6d6));
04033       SDValue t1 = DAG.getNode(ISD::FADD, dl, MVT::f32, t0,
04034                                getF32Constant(DAG, 0x3ee4f4b8));
04035       SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t1, X);
04036       SDValue t3 = DAG.getNode(ISD::FSUB, dl, MVT::f32, t2,
04037                                getF32Constant(DAG, 0x3fbc278b));
04038       SDValue t4 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t3, X);
04039       SDValue t5 = DAG.getNode(ISD::FADD, dl, MVT::f32, t4,
04040                                getF32Constant(DAG, 0x40348e95));
04041       SDValue t6 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t5, X);
04042       LogOfMantissa = DAG.getNode(ISD::FSUB, dl, MVT::f32, t6,
04043                                   getF32Constant(DAG, 0x3fdef31a));
04044     } else { // LimitFloatPrecision <= 18
04045       // For floating-point precision of 18:
04046       //
04047       //   LogOfMantissa =
04048       //     -2.1072184f +
04049       //       (4.2372794f +
04050       //         (-3.7029485f +
04051       //           (2.2781945f +
04052       //             (-0.87823314f +
04053       //               (0.19073739f - 0.17809712e-1f * x) * x) * x) * x) * x)*x;
04054       //
04055       // error 0.0000023660568, which is better than 18 bits
04056       SDValue t0 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X,
04057                                getF32Constant(DAG, 0xbc91e5ac));
04058       SDValue t1 = DAG.getNode(ISD::FADD, dl, MVT::f32, t0,
04059                                getF32Constant(DAG, 0x3e4350aa));
04060       SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t1, X);
04061       SDValue t3 = DAG.getNode(ISD::FSUB, dl, MVT::f32, t2,
04062                                getF32Constant(DAG, 0x3f60d3e3));
04063       SDValue t4 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t3, X);
04064       SDValue t5 = DAG.getNode(ISD::FADD, dl, MVT::f32, t4,
04065                                getF32Constant(DAG, 0x4011cdf0));
04066       SDValue t6 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t5, X);
04067       SDValue t7 = DAG.getNode(ISD::FSUB, dl, MVT::f32, t6,
04068                                getF32Constant(DAG, 0x406cfd1c));
04069       SDValue t8 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t7, X);
04070       SDValue t9 = DAG.getNode(ISD::FADD, dl, MVT::f32, t8,
04071                                getF32Constant(DAG, 0x408797cb));
04072       SDValue t10 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t9, X);
04073       LogOfMantissa = DAG.getNode(ISD::FSUB, dl, MVT::f32, t10,
04074                                   getF32Constant(DAG, 0x4006dcab));
04075     }
04076 
04077     return DAG.getNode(ISD::FADD, dl, MVT::f32, LogOfExponent, LogOfMantissa);
04078   }
04079 
04080   // No special expansion.
04081   return DAG.getNode(ISD::FLOG, dl, Op.getValueType(), Op);
04082 }
04083 
04084 /// expandLog2 - Lower a log2 intrinsic. Handles the special sequences for
04085 /// limited-precision mode.
04086 static SDValue expandLog2(SDLoc dl, SDValue Op, SelectionDAG &DAG,
04087                           const TargetLowering &TLI) {
04088   if (Op.getValueType() == MVT::f32 &&
04089       LimitFloatPrecision > 0 && LimitFloatPrecision <= 18) {
04090     SDValue Op1 = DAG.getNode(ISD::BITCAST, dl, MVT::i32, Op);
04091 
04092     // Get the exponent.
04093     SDValue LogOfExponent = GetExponent(DAG, Op1, TLI, dl);
04094 
04095     // Get the significand and build it into a floating-point number with
04096     // exponent of 1.
04097     SDValue X = GetSignificand(DAG, Op1, dl);
04098 
04099     // Different possible minimax approximations of significand in
04100     // floating-point for various degrees of accuracy over [1,2].
04101     SDValue Log2ofMantissa;
04102     if (LimitFloatPrecision <= 6) {
04103       // For floating-point precision of 6:
04104       //
04105       //   Log2ofMantissa = -1.6749035f + (2.0246817f - .34484768f * x) * x;
04106       //
04107       // error 0.0049451742, which is more than 7 bits
04108       SDValue t0 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X,
04109                                getF32Constant(DAG, 0xbeb08fe0));
04110       SDValue t1 = DAG.getNode(ISD::FADD, dl, MVT::f32, t0,
04111                                getF32Constant(DAG, 0x40019463));
04112       SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t1, X);
04113       Log2ofMantissa = DAG.getNode(ISD::FSUB, dl, MVT::f32, t2,
04114                                    getF32Constant(DAG, 0x3fd6633d));
04115     } else if (LimitFloatPrecision <= 12) {
04116       // For floating-point precision of 12:
04117       //
04118       //   Log2ofMantissa =
04119       //     -2.51285454f +
04120       //       (4.07009056f +
04121       //         (-2.12067489f +
04122       //           (.645142248f - 0.816157886e-1f * x) * x) * x) * x;
04123       //
04124       // error 0.0000876136000, which is better than 13 bits
04125       SDValue t0 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X,
04126                                getF32Constant(DAG, 0xbda7262e));
04127       SDValue t1 = DAG.getNode(ISD::FADD, dl, MVT::f32, t0,
04128                                getF32Constant(DAG, 0x3f25280b));
04129       SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t1, X);
04130       SDValue t3 = DAG.getNode(ISD::FSUB, dl, MVT::f32, t2,
04131                                getF32Constant(DAG, 0x4007b923));
04132       SDValue t4 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t3, X);
04133       SDValue t5 = DAG.getNode(ISD::FADD, dl, MVT::f32, t4,
04134                                getF32Constant(DAG, 0x40823e2f));
04135       SDValue t6 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t5, X);
04136       Log2ofMantissa = DAG.getNode(ISD::FSUB, dl, MVT::f32, t6,
04137                                    getF32Constant(DAG, 0x4020d29c));
04138     } else { // LimitFloatPrecision <= 18
04139       // For floating-point precision of 18:
04140       //
04141       //   Log2ofMantissa =
04142       //     -3.0400495f +
04143       //       (6.1129976f +
04144       //         (-5.3420409f +
04145       //           (3.2865683f +
04146       //             (-1.2669343f +
04147       //               (0.27515199f -
04148       //                 0.25691327e-1f * x) * x) * x) * x) * x) * x;
04149       //
04150       // error 0.0000018516, which is better than 18 bits
04151       SDValue t0 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X,
04152                                getF32Constant(DAG, 0xbcd2769e));
04153       SDValue t1 = DAG.getNode(ISD::FADD, dl, MVT::f32, t0,
04154                                getF32Constant(DAG, 0x3e8ce0b9));
04155       SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t1, X);
04156       SDValue t3 = DAG.getNode(ISD::FSUB, dl, MVT::f32, t2,
04157                                getF32Constant(DAG, 0x3fa22ae7));
04158       SDValue t4 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t3, X);
04159       SDValue t5 = DAG.getNode(ISD::FADD, dl, MVT::f32, t4,
04160                                getF32Constant(DAG, 0x40525723));
04161       SDValue t6 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t5, X);
04162       SDValue t7 = DAG.getNode(ISD::FSUB, dl, MVT::f32, t6,
04163                                getF32Constant(DAG, 0x40aaf200));
04164       SDValue t8 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t7, X);
04165       SDValue t9 = DAG.getNode(ISD::FADD, dl, MVT::f32, t8,
04166                                getF32Constant(DAG, 0x40c39dad));
04167       SDValue t10 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t9, X);
04168       Log2ofMantissa = DAG.getNode(ISD::FSUB, dl, MVT::f32, t10,
04169                                    getF32Constant(DAG, 0x4042902c));
04170     }
04171 
04172     return DAG.getNode(ISD::FADD, dl, MVT::f32, LogOfExponent, Log2ofMantissa);
04173   }
04174 
04175   // No special expansion.
04176   return DAG.getNode(ISD::FLOG2, dl, Op.getValueType(), Op);
04177 }
04178 
04179 /// expandLog10 - Lower a log10 intrinsic. Handles the special sequences for
04180 /// limited-precision mode.
04181 static SDValue expandLog10(SDLoc dl, SDValue Op, SelectionDAG &DAG,
04182                            const TargetLowering &TLI) {
04183   if (Op.getValueType() == MVT::f32 &&
04184       LimitFloatPrecision > 0 && LimitFloatPrecision <= 18) {
04185     SDValue Op1 = DAG.getNode(ISD::BITCAST, dl, MVT::i32, Op);
04186 
04187     // Scale the exponent by log10(2) [0.30102999f].
04188     SDValue Exp = GetExponent(DAG, Op1, TLI, dl);
04189     SDValue LogOfExponent = DAG.getNode(ISD::FMUL, dl, MVT::f32, Exp,
04190                                         getF32Constant(DAG, 0x3e9a209a));
04191 
04192     // Get the significand and build it into a floating-point number with
04193     // exponent of 1.
04194     SDValue X = GetSignificand(DAG, Op1, dl);
04195 
04196     SDValue Log10ofMantissa;
04197     if (LimitFloatPrecision <= 6) {
04198       // For floating-point precision of 6:
04199       //
04200       //   Log10ofMantissa =
04201       //     -0.50419619f +
04202       //       (0.60948995f - 0.10380950f * x) * x;
04203       //
04204       // error 0.0014886165, which is 6 bits
04205       SDValue t0 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X,
04206                                getF32Constant(DAG, 0xbdd49a13));
04207       SDValue t1 = DAG.getNode(ISD::FADD, dl, MVT::f32, t0,
04208                                getF32Constant(DAG, 0x3f1c0789));
04209       SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t1, X);
04210       Log10ofMantissa = DAG.getNode(ISD::FSUB, dl, MVT::f32, t2,
04211                                     getF32Constant(DAG, 0x3f011300));
04212     } else if (LimitFloatPrecision <= 12) {
04213       // For floating-point precision of 12:
04214       //
04215       //   Log10ofMantissa =
04216       //     -0.64831180f +
04217       //       (0.91751397f +
04218       //         (-0.31664806f + 0.47637168e-1f * x) * x) * x;
04219       //
04220       // error 0.00019228036, which is better than 12 bits
04221       SDValue t0 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X,
04222                                getF32Constant(DAG, 0x3d431f31));
04223       SDValue t1 = DAG.getNode(ISD::FSUB, dl, MVT::f32, t0,
04224                                getF32Constant(DAG, 0x3ea21fb2));
04225       SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t1, X);
04226       SDValue t3 = DAG.getNode(ISD::FADD, dl, MVT::f32, t2,
04227                                getF32Constant(DAG, 0x3f6ae232));
04228       SDValue t4 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t3, X);
04229       Log10ofMantissa = DAG.getNode(ISD::FSUB, dl, MVT::f32, t4,
04230                                     getF32Constant(DAG, 0x3f25f7c3));
04231     } else { // LimitFloatPrecision <= 18
04232       // For floating-point precision of 18:
04233       //
04234       //   Log10ofMantissa =
04235       //     -0.84299375f +
04236       //       (1.5327582f +
04237       //         (-1.0688956f +
04238       //           (0.49102474f +
04239       //             (-0.12539807f + 0.13508273e-1f * x) * x) * x) * x) * x;
04240       //
04241       // error 0.0000037995730, which is better than 18 bits
04242       SDValue t0 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X,
04243                                getF32Constant(DAG, 0x3c5d51ce));
04244       SDValue t1 = DAG.getNode(ISD::FSUB, dl, MVT::f32, t0,
04245                                getF32Constant(DAG, 0x3e00685a));
04246       SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t1, X);
04247       SDValue t3 = DAG.getNode(ISD::FADD, dl, MVT::f32, t2,
04248                                getF32Constant(DAG, 0x3efb6798));
04249       SDValue t4 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t3, X);
04250       SDValue t5 = DAG.getNode(ISD::FSUB, dl, MVT::f32, t4,
04251                                getF32Constant(DAG, 0x3f88d192));
04252       SDValue t6 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t5, X);
04253       SDValue t7 = DAG.getNode(ISD::FADD, dl, MVT::f32, t6,
04254                                getF32Constant(DAG, 0x3fc4316c));
04255       SDValue t8 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t7, X);
04256       Log10ofMantissa = DAG.getNode(ISD::FSUB, dl, MVT::f32, t8,
04257                                     getF32Constant(DAG, 0x3f57ce70));
04258     }
04259 
04260     return DAG.getNode(ISD::FADD, dl, MVT::f32, LogOfExponent, Log10ofMantissa);
04261   }
04262 
04263   // No special expansion.
04264   return DAG.getNode(ISD::FLOG10, dl, Op.getValueType(), Op);
04265 }
04266 
04267 /// expandExp2 - Lower an exp2 intrinsic. Handles the special sequences for
04268 /// limited-precision mode.
04269 static SDValue expandExp2(SDLoc dl, SDValue Op, SelectionDAG &DAG,
04270                           const TargetLowering &TLI) {
04271   if (Op.getValueType() == MVT::f32 &&
04272       LimitFloatPrecision > 0 && LimitFloatPrecision <= 18) {
04273     SDValue IntegerPartOfX = DAG.getNode(ISD::FP_TO_SINT, dl, MVT::i32, Op);
04274 
04275     //   FractionalPartOfX = x - (float)IntegerPartOfX;
04276     SDValue t1 = DAG.getNode(ISD::SINT_TO_FP, dl, MVT::f32, IntegerPartOfX);
04277     SDValue X = DAG.getNode(ISD::FSUB, dl, MVT::f32, Op, t1);
04278 
04279     //   IntegerPartOfX <<= 23;
04280     IntegerPartOfX = DAG.getNode(ISD::SHL, dl, MVT::i32, IntegerPartOfX,
04281                                  DAG.getConstant(23, TLI.getPointerTy()));
04282 
04283     SDValue TwoToFractionalPartOfX;
04284     if (LimitFloatPrecision <= 6) {
04285       // For floating-point precision of 6:
04286       //
04287       //   TwoToFractionalPartOfX =
04288       //     0.997535578f +
04289       //       (0.735607626f + 0.252464424f * x) * x;
04290       //
04291       // error 0.0144103317, which is 6 bits
04292       SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X,
04293                                getF32Constant(DAG, 0x3e814304));
04294       SDValue t3 = DAG.getNode(ISD::FADD, dl, MVT::f32, t2,
04295                                getF32Constant(DAG, 0x3f3c50c8));
04296       SDValue t4 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t3, X);
04297       TwoToFractionalPartOfX = DAG.getNode(ISD::FADD, dl, MVT::f32, t4,
04298                                            getF32Constant(DAG, 0x3f7f5e7e));
04299     } else if (LimitFloatPrecision <= 12) {
04300       // For floating-point precision of 12:
04301       //
04302       //   TwoToFractionalPartOfX =
04303       //     0.999892986f +
04304       //       (0.696457318f +
04305       //         (0.224338339f + 0.792043434e-1f * x) * x) * x;
04306       //
04307       // error 0.000107046256, which is 13 to 14 bits
04308       SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X,
04309                                getF32Constant(DAG, 0x3da235e3));
04310       SDValue t3 = DAG.getNode(ISD::FADD, dl, MVT::f32, t2,
04311                                getF32Constant(DAG, 0x3e65b8f3));
04312       SDValue t4 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t3, X);
04313       SDValue t5 = DAG.getNode(ISD::FADD, dl, MVT::f32, t4,
04314                                getF32Constant(DAG, 0x3f324b07));
04315       SDValue t6 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t5, X);
04316       TwoToFractionalPartOfX = DAG.getNode(ISD::FADD, dl, MVT::f32, t6,
04317                                            getF32Constant(DAG, 0x3f7ff8fd));
04318     } else { // LimitFloatPrecision <= 18
04319       // For floating-point precision of 18:
04320       //
04321       //   TwoToFractionalPartOfX =
04322       //     0.999999982f +
04323       //       (0.693148872f +
04324       //         (0.240227044f +
04325       //           (0.554906021e-1f +
04326       //             (0.961591928e-2f +
04327       //               (0.136028312e-2f + 0.157059148e-3f *x)*x)*x)*x)*x)*x;
04328       // error 2.47208000*10^(-7), which is better than 18 bits
04329       SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X,
04330                                getF32Constant(DAG, 0x3924b03e));
04331       SDValue t3 = DAG.getNode(ISD::FADD, dl, MVT::f32, t2,
04332                                getF32Constant(DAG, 0x3ab24b87));
04333       SDValue t4 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t3, X);
04334       SDValue t5 = DAG.getNode(ISD::FADD, dl, MVT::f32, t4,
04335                                getF32Constant(DAG, 0x3c1d8c17));
04336       SDValue t6 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t5, X);
04337       SDValue t7 = DAG.getNode(ISD::FADD, dl, MVT::f32, t6,
04338                                getF32Constant(DAG, 0x3d634a1d));
04339       SDValue t8 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t7, X);
04340       SDValue t9 = DAG.getNode(ISD::FADD, dl, MVT::f32, t8,
04341                                getF32Constant(DAG, 0x3e75fe14));
04342       SDValue t10 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t9, X);
04343       SDValue t11 = DAG.getNode(ISD::FADD, dl, MVT::f32, t10,
04344                                 getF32Constant(DAG, 0x3f317234));
04345       SDValue t12 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t11, X);
04346       TwoToFractionalPartOfX = DAG.getNode(ISD::FADD, dl, MVT::f32, t12,
04347                                            getF32Constant(DAG, 0x3f800000));
04348     }
04349 
04350     // Add the exponent into the result in integer domain.
04351     SDValue t13 = DAG.getNode(ISD::BITCAST, dl, MVT::i32,
04352                               TwoToFractionalPartOfX);
04353     return DAG.getNode(ISD::BITCAST, dl, MVT::f32,
04354                        DAG.getNode(ISD::ADD, dl, MVT::i32,
04355                                    t13, IntegerPartOfX));
04356   }
04357 
04358   // No special expansion.
04359   return DAG.getNode(ISD::FEXP2, dl, Op.getValueType(), Op);
04360 }
04361 
04362 /// visitPow - Lower a pow intrinsic. Handles the special sequences for
04363 /// limited-precision mode with x == 10.0f.
04364 static SDValue expandPow(SDLoc dl, SDValue LHS, SDValue RHS,
04365                          SelectionDAG &DAG, const TargetLowering &TLI) {
04366   bool IsExp10 = false;
04367   if (LHS.getValueType() == MVT::f32 && RHS.getValueType() == MVT::f32 &&
04368       LimitFloatPrecision > 0 && LimitFloatPrecision <= 18) {
04369     if (ConstantFPSDNode *LHSC = dyn_cast<ConstantFPSDNode>(LHS)) {
04370       APFloat Ten(10.0f);
04371       IsExp10 = LHSC->isExactlyValue(Ten);
04372     }
04373   }
04374 
04375   if (IsExp10) {
04376     // Put the exponent in the right bit position for later addition to the
04377     // final result:
04378     //
04379     //   #define LOG2OF10 3.3219281f
04380     //   IntegerPartOfX = (int32_t)(x * LOG2OF10);
04381     SDValue t0 = DAG.getNode(ISD::FMUL, dl, MVT::f32, RHS,
04382                              getF32Constant(DAG, 0x40549a78));
04383     SDValue IntegerPartOfX = DAG.getNode(ISD::FP_TO_SINT, dl, MVT::i32, t0);
04384 
04385     //   FractionalPartOfX = x - (float)IntegerPartOfX;
04386     SDValue t1 = DAG.getNode(ISD::SINT_TO_FP, dl, MVT::f32, IntegerPartOfX);
04387     SDValue X = DAG.getNode(ISD::FSUB, dl, MVT::f32, t0, t1);
04388 
04389     //   IntegerPartOfX <<= 23;
04390     IntegerPartOfX = DAG.getNode(ISD::SHL, dl, MVT::i32, IntegerPartOfX,
04391                                  DAG.getConstant(23, TLI.getPointerTy()));
04392 
04393     SDValue TwoToFractionalPartOfX;
04394     if (LimitFloatPrecision <= 6) {
04395       // For floating-point precision of 6:
04396       //
04397       //   twoToFractionalPartOfX =
04398       //     0.997535578f +
04399       //       (0.735607626f + 0.252464424f * x) * x;
04400       //
04401       // error 0.0144103317, which is 6 bits
04402       SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X,
04403                                getF32Constant(DAG, 0x3e814304));
04404       SDValue t3 = DAG.getNode(ISD::FADD, dl, MVT::f32, t2,
04405                                getF32Constant(DAG, 0x3f3c50c8));
04406       SDValue t4 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t3, X);
04407       TwoToFractionalPartOfX = DAG.getNode(ISD::FADD, dl, MVT::f32, t4,
04408                                            getF32Constant(DAG, 0x3f7f5e7e));
04409     } else if (LimitFloatPrecision <= 12) {
04410       // For floating-point precision of 12:
04411       //
04412       //   TwoToFractionalPartOfX =
04413       //     0.999892986f +
04414       //       (0.696457318f +
04415       //         (0.224338339f + 0.792043434e-1f * x) * x) * x;
04416       //
04417       // error 0.000107046256, which is 13 to 14 bits
04418       SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X,
04419                                getF32Constant(DAG, 0x3da235e3));
04420       SDValue t3 = DAG.getNode(ISD::FADD, dl, MVT::f32, t2,
04421                                getF32Constant(DAG, 0x3e65b8f3));
04422       SDValue t4 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t3, X);
04423       SDValue t5 = DAG.getNode(ISD::FADD, dl, MVT::f32, t4,
04424                                getF32Constant(DAG, 0x3f324b07));
04425       SDValue t6 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t5, X);
04426       TwoToFractionalPartOfX = DAG.getNode(ISD::FADD, dl, MVT::f32, t6,
04427                                            getF32Constant(DAG, 0x3f7ff8fd));
04428     } else { // LimitFloatPrecision <= 18
04429       // For floating-point precision of 18:
04430       //
04431       //   TwoToFractionalPartOfX =
04432       //     0.999999982f +
04433       //       (0.693148872f +
04434       //         (0.240227044f +
04435       //           (0.554906021e-1f +
04436       //             (0.961591928e-2f +
04437       //               (0.136028312e-2f + 0.157059148e-3f *x)*x)*x)*x)*x)*x;
04438       // error 2.47208000*10^(-7), which is better than 18 bits
04439       SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X,
04440                                getF32Constant(DAG, 0x3924b03e));
04441       SDValue t3 = DAG.getNode(ISD::FADD, dl, MVT::f32, t2,
04442                                getF32Constant(DAG, 0x3ab24b87));
04443       SDValue t4 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t3, X);
04444       SDValue t5 = DAG.getNode(ISD::FADD, dl, MVT::f32, t4,
04445                                getF32Constant(DAG, 0x3c1d8c17));
04446       SDValue t6 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t5, X);
04447       SDValue t7 = DAG.getNode(ISD::FADD, dl, MVT::f32, t6,
04448                                getF32Constant(DAG, 0x3d634a1d));
04449       SDValue t8 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t7, X);
04450       SDValue t9 = DAG.getNode(ISD::FADD, dl, MVT::f32, t8,
04451                                getF32Constant(DAG, 0x3e75fe14));
04452       SDValue t10 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t9, X);
04453       SDValue t11 = DAG.getNode(ISD::FADD, dl, MVT::f32, t10,
04454                                 getF32Constant(DAG, 0x3f317234));
04455       SDValue t12 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t11, X);
04456       TwoToFractionalPartOfX = DAG.getNode(ISD::FADD, dl, MVT::f32, t12,
04457                                            getF32Constant(DAG, 0x3f800000));
04458     }
04459 
04460     SDValue t13 = DAG.getNode(ISD::BITCAST, dl,MVT::i32,TwoToFractionalPartOfX);
04461     return DAG.getNode(ISD::BITCAST, dl, MVT::f32,
04462                        DAG.getNode(ISD::ADD, dl, MVT::i32,
04463                                    t13, IntegerPartOfX));
04464   }
04465 
04466   // No special expansion.
04467   return DAG.getNode(ISD::FPOW, dl, LHS.getValueType(), LHS, RHS);
04468 }
04469 
04470 
04471 /// ExpandPowI - Expand a llvm.powi intrinsic.
04472 static SDValue ExpandPowI(SDLoc DL, SDValue LHS, SDValue RHS,
04473                           SelectionDAG &DAG) {
04474   // If RHS is a constant, we can expand this out to a multiplication tree,
04475   // otherwise we end up lowering to a call to __powidf2 (for example).  When
04476   // optimizing for size, we only want to do this if the expansion would produce
04477   // a small number of multiplies, otherwise we do the full expansion.
04478   if (ConstantSDNode *RHSC = dyn_cast<ConstantSDNode>(RHS)) {
04479     // Get the exponent as a positive value.
04480     unsigned Val = RHSC->getSExtValue();
04481     if ((int)Val < 0) Val = -Val;
04482 
04483     // powi(x, 0) -> 1.0
04484     if (Val == 0)
04485       return DAG.getConstantFP(1.0, LHS.getValueType());
04486 
04487     const Function *F = DAG.getMachineFunction().getFunction();
04488     if (!F->getAttributes().hasAttribute(AttributeSet::FunctionIndex,
04489                                          Attribute::OptimizeForSize) ||
04490         // If optimizing for size, don't insert too many multiplies.  This
04491         // inserts up to 5 multiplies.
04492         CountPopulation_32(Val)+Log2_32(Val) < 7) {
04493       // We use the simple binary decomposition method to generate the multiply
04494       // sequence.  There are more optimal ways to do this (for example,
04495       // powi(x,15) generates one more multiply than it should), but this has
04496       // the benefit of being both really simple and much better than a libcall.
04497       SDValue Res;  // Logically starts equal to 1.0
04498       SDValue CurSquare = LHS;
04499       while (Val) {
04500         if (Val & 1) {
04501           if (Res.getNode())
04502             Res = DAG.getNode(ISD::FMUL, DL,Res.getValueType(), Res, CurSquare);
04503           else
04504             Res = CurSquare;  // 1.0*CurSquare.
04505         }
04506 
04507         CurSquare = DAG.getNode(ISD::FMUL, DL, CurSquare.getValueType(),
04508                                 CurSquare, CurSquare);
04509         Val >>= 1;
04510       }
04511 
04512       // If the original was negative, invert the result, producing 1/(x*x*x).
04513       if (RHSC->getSExtValue() < 0)
04514         Res = DAG.getNode(ISD::FDIV, DL, LHS.getValueType(),
04515                           DAG.getConstantFP(1.0, LHS.getValueType()), Res);
04516       return Res;
04517     }
04518   }
04519 
04520   // Otherwise, expand to a libcall.
04521   return DAG.getNode(ISD::FPOWI, DL, LHS.getValueType(), LHS, RHS);
04522 }
04523 
04524 // getTruncatedArgReg - Find underlying register used for an truncated
04525 // argument.
04526 static unsigned getTruncatedArgReg(const SDValue &N) {
04527   if (N.getOpcode() != ISD::TRUNCATE)
04528     return 0;
04529 
04530   const SDValue &Ext = N.getOperand(0);
04531   if (Ext.getOpcode() == ISD::AssertZext ||
04532       Ext.getOpcode() == ISD::AssertSext) {
04533     const SDValue &CFR = Ext.getOperand(0);
04534     if (CFR.getOpcode() == ISD::CopyFromReg)
04535       return cast<RegisterSDNode>(CFR.getOperand(1))->getReg();
04536     if (CFR.getOpcode() == ISD::TRUNCATE)
04537       return getTruncatedArgReg(CFR);
04538   }
04539   return 0;
04540 }
04541 
04542 /// EmitFuncArgumentDbgValue - If the DbgValueInst is a dbg_value of a function
04543 /// argument, create the corresponding DBG_VALUE machine instruction for it now.
04544 /// At the end of instruction selection, they will be inserted to the entry BB.
04545 bool
04546 SelectionDAGBuilder::EmitFuncArgumentDbgValue(const Value *V, MDNode *Variable,
04547                                               int64_t Offset,
04548                                               const SDValue &N) {
04549   const Argument *Arg = dyn_cast<Argument>(V);
04550   if (!Arg)
04551     return false;
04552 
04553   MachineFunction &MF = DAG.getMachineFunction();
04554   const TargetInstrInfo *TII = DAG.getTarget().getInstrInfo();
04555 
04556   // Ignore inlined function arguments here.
04557   DIVariable DV(Variable);
04558   if (DV.isInlinedFnArgument(MF.getFunction()))
04559     return false;
04560 
04561   Optional<MachineOperand> Op;
04562   // Some arguments' frame index is recorded during argument lowering.
04563   if (int FI = FuncInfo.getArgumentFrameIndex(Arg))
04564     Op = MachineOperand::CreateFI(FI);
04565 
04566   if (!Op && N.getNode()) {
04567     unsigned Reg;
04568     if (N.getOpcode() == ISD::CopyFromReg)
04569       Reg = cast<RegisterSDNode>(N.getOperand(1))->getReg();
04570     else
04571       Reg = getTruncatedArgReg(N);
04572     if (Reg && TargetRegisterInfo::isVirtualRegister(Reg)) {
04573       MachineRegisterInfo &RegInfo = MF.getRegInfo();
04574       unsigned PR = RegInfo.getLiveInPhysReg(Reg);
04575       if (PR)
04576         Reg = PR;
04577     }
04578     if (Reg)
04579       Op = MachineOperand::CreateReg(Reg, false);
04580   }
04581 
04582   if (!Op) {
04583     // Check if ValueMap has reg number.
04584     DenseMap<const Value *, unsigned>::iterator VMI = FuncInfo.ValueMap.find(V);
04585     if (VMI != FuncInfo.ValueMap.end())
04586       Op = MachineOperand::CreateReg(VMI->second, false);
04587   }
04588 
04589   if (!Op && N.getNode())
04590     // Check if frame index is available.
04591     if (LoadSDNode *LNode = dyn_cast<LoadSDNode>(N.getNode()))
04592       if (FrameIndexSDNode *FINode =
04593           dyn_cast<FrameIndexSDNode>(LNode->getBasePtr().getNode()))
04594         Op = MachineOperand::CreateFI(FINode->getIndex());
04595 
04596   if (!Op)
04597     return false;
04598 
04599   // FIXME: This does not handle register-indirect values at offset 0.
04600   bool IsIndirect = Offset != 0;
04601   if (Op->isReg())
04602     FuncInfo.ArgDbgValues.push_back(BuildMI(MF, getCurDebugLoc(),
04603                                             TII->get(TargetOpcode::DBG_VALUE),
04604                                             IsIndirect,
04605                                             Op->getReg(), Offset, Variable));
04606   else
04607     FuncInfo.ArgDbgValues.push_back(
04608       BuildMI(MF, getCurDebugLoc(), TII->get(TargetOpcode::DBG_VALUE))
04609           .addOperand(*Op).addImm(Offset).addMetadata(Variable));
04610 
04611   return true;
04612 }
04613 
04614 // VisualStudio defines setjmp as _setjmp
04615 #if defined(_MSC_VER) && defined(setjmp) && \
04616                          !defined(setjmp_undefined_for_msvc)
04617 #  pragma push_macro("setjmp")
04618 #  undef setjmp
04619 #  define setjmp_undefined_for_msvc
04620 #endif
04621 
04622 /// visitIntrinsicCall - Lower the call to the specified intrinsic function.  If
04623 /// we want to emit this as a call to a named external function, return the name
04624 /// otherwise lower it and return null.
04625 const char *
04626 SelectionDAGBuilder::visitIntrinsicCall(const CallInst &I, unsigned Intrinsic) {
04627   const TargetLowering *TLI = TM.getTargetLowering();
04628   SDLoc sdl = getCurSDLoc();
04629   DebugLoc dl = getCurDebugLoc();
04630   SDValue Res;
04631 
04632   switch (Intrinsic) {
04633   default:
04634     // By default, turn this into a target intrinsic node.
04635     visitTargetIntrinsic(I, Intrinsic);
04636     return nullptr;
04637   case Intrinsic::vastart:  visitVAStart(I); return nullptr;
04638   case Intrinsic::vaend:    visitVAEnd(I); return nullptr;
04639   case Intrinsic::vacopy:   visitVACopy(I); return nullptr;
04640   case Intrinsic::returnaddress:
04641     setValue(&I, DAG.getNode(ISD::RETURNADDR, sdl, TLI->getPointerTy(),
04642                              getValue(I.getArgOperand(0))));
04643     return nullptr;
04644   case Intrinsic::frameaddress:
04645     setValue(&I, DAG.getNode(ISD::FRAMEADDR, sdl, TLI->getPointerTy(),
04646                              getValue(I.getArgOperand(0))));
04647     return nullptr;
04648   case Intrinsic::setjmp:
04649     return &"_setjmp"[!TLI->usesUnderscoreSetJmp()];
04650   case Intrinsic::longjmp:
04651     return &"_longjmp"[!TLI->usesUnderscoreLongJmp()];
04652   case Intrinsic::memcpy: {
04653     // Assert for address < 256 since we support only user defined address
04654     // spaces.
04655     assert(cast<PointerType>(I.getArgOperand(0)->getType())->getAddressSpace()
04656            < 256 &&
04657            cast<PointerType>(I.getArgOperand(1)->getType())->getAddressSpace()
04658            < 256 &&
04659            "Unknown address space");
04660     SDValue Op1 = getValue(I.getArgOperand(0));
04661     SDValue Op2 = getValue(I.getArgOperand(1));
04662     SDValue Op3 = getValue(I.getArgOperand(2));
04663     unsigned Align = cast<ConstantInt>(I.getArgOperand(3))->getZExtValue();
04664     if (!Align)
04665       Align = 1; // @llvm.memcpy defines 0 and 1 to both mean no alignment.
04666     bool isVol = cast<ConstantInt>(I.getArgOperand(4))->getZExtValue();
04667     DAG.setRoot(DAG.getMemcpy(getRoot(), sdl, Op1, Op2, Op3, Align, isVol, false,
04668                               MachinePointerInfo(I.getArgOperand(0)),
04669                               MachinePointerInfo(I.getArgOperand(1))));
04670     return nullptr;
04671   }
04672   case Intrinsic::memset: {
04673     // Assert for address < 256 since we support only user defined address
04674     // spaces.
04675     assert(cast<PointerType>(I.getArgOperand(0)->getType())->getAddressSpace()
04676            < 256 &&
04677            "Unknown address space");
04678     SDValue Op1 = getValue(I.getArgOperand(0));
04679     SDValue Op2 = getValue(I.getArgOperand(1));
04680     SDValue Op3 = getValue(I.getArgOperand(2));
04681     unsigned Align = cast<ConstantInt>(I.getArgOperand(3))->getZExtValue();
04682     if (!Align)
04683       Align = 1; // @llvm.memset defines 0 and 1 to both mean no alignment.
04684     bool isVol = cast<ConstantInt>(I.getArgOperand(4))->getZExtValue();
04685     DAG.setRoot(DAG.getMemset(getRoot(), sdl, Op1, Op2, Op3, Align, isVol,
04686                               MachinePointerInfo(I.getArgOperand(0))));
04687     return nullptr;
04688   }
04689   case Intrinsic::memmove: {
04690     // Assert for address < 256 since we support only user defined address
04691     // spaces.
04692     assert(cast<PointerType>(I.getArgOperand(0)->getType())->getAddressSpace()
04693            < 256 &&
04694            cast<PointerType>(I.getArgOperand(1)->getType())->getAddressSpace()
04695            < 256 &&
04696            "Unknown address space");
04697     SDValue Op1 = getValue(I.getArgOperand(0));
04698     SDValue Op2 = getValue(I.getArgOperand(1));
04699     SDValue Op3 = getValue(I.getArgOperand(2));
04700     unsigned Align = cast<ConstantInt>(I.getArgOperand(3))->getZExtValue();
04701     if (!Align)
04702       Align = 1; // @llvm.memmove defines 0 and 1 to both mean no alignment.
04703     bool isVol = cast<ConstantInt>(I.getArgOperand(4))->getZExtValue();
04704     DAG.setRoot(DAG.getMemmove(getRoot(), sdl, Op1, Op2, Op3, Align, isVol,
04705                                MachinePointerInfo(I.getArgOperand(0)),
04706                                MachinePointerInfo(I.getArgOperand(1))));
04707     return nullptr;
04708   }
04709   case Intrinsic::dbg_declare: {
04710     const DbgDeclareInst &DI = cast<DbgDeclareInst>(I);
04711     MDNode *Variable = DI.getVariable();
04712     const Value *Address = DI.getAddress();
04713     DIVariable DIVar(Variable);
04714     assert((!DIVar || DIVar.isVariable()) &&
04715       "Variable in DbgDeclareInst should be either null or a DIVariable.");
04716     if (!Address || !DIVar) {
04717       DEBUG(dbgs() << "Dropping debug info for " << DI << "\n");
04718       return nullptr;
04719     }
04720 
04721     // Check if address has undef value.
04722     if (isa<UndefValue>(Address) ||
04723         (Address->use_empty() && !isa<Argument>(Address))) {
04724       DEBUG(dbgs() << "Dropping debug info for " << DI << "\n");
04725       return nullptr;
04726     }
04727 
04728     SDValue &N = NodeMap[Address];
04729     if (!N.getNode() && isa<Argument>(Address))
04730       // Check unused arguments map.
04731       N = UnusedArgNodeMap[Address];
04732     SDDbgValue *SDV;
04733     if (N.getNode()) {
04734       if (const BitCastInst *BCI = dyn_cast<BitCastInst>(Address))
04735         Address = BCI->getOperand(0);
04736       // Parameters are handled specially.
04737       bool isParameter =
04738         (DIVariable(Variable).getTag() == dwarf::DW_TAG_arg_variable ||
04739          isa<Argument>(Address));
04740 
04741       const AllocaInst *AI = dyn_cast<AllocaInst>(Address);
04742 
04743       if (isParameter && !AI) {
04744         FrameIndexSDNode *FINode = dyn_cast<FrameIndexSDNode>(N.getNode());
04745         if (FINode)
04746           // Byval parameter.  We have a frame index at this point.
04747           SDV = DAG.getDbgValue(Variable, FINode->getIndex(),
04748                                 0, dl, SDNodeOrder);
04749         else {
04750           // Address is an argument, so try to emit its dbg value using
04751           // virtual register info from the FuncInfo.ValueMap.
04752           EmitFuncArgumentDbgValue(Address, Variable, 0, N);
04753           return nullptr;
04754         }
04755       } else if (AI)
04756         SDV = DAG.getDbgValue(Variable, N.getNode(), N.getResNo(),
04757                               0, dl, SDNodeOrder);
04758       else {
04759         // Can't do anything with other non-AI cases yet.
04760         DEBUG(dbgs() << "Dropping debug info for " << DI << "\n");
04761         DEBUG(dbgs() << "non-AllocaInst issue for Address: \n\t");
04762         DEBUG(Address->dump());
04763         return nullptr;
04764       }
04765       DAG.AddDbgValue(SDV, N.getNode(), isParameter);
04766     } else {
04767       // If Address is an argument then try to emit its dbg value using
04768       // virtual register info from the FuncInfo.ValueMap.
04769       if (!EmitFuncArgumentDbgValue(Address, Variable, 0, N)) {
04770         // If variable is pinned by a alloca in dominating bb then
04771         // use StaticAllocaMap.
04772         if (const AllocaInst *AI = dyn_cast<AllocaInst>(Address)) {
04773           if (AI->getParent() != DI.getParent()) {
04774             DenseMap<const AllocaInst*, int>::iterator SI =
04775               FuncInfo.StaticAllocaMap.find(AI);
04776             if (SI != FuncInfo.StaticAllocaMap.end()) {
04777               SDV = DAG.getDbgValue(Variable, SI->second,
04778                                     0, dl, SDNodeOrder);
04779               DAG.AddDbgValue(SDV, nullptr, false);
04780               return nullptr;
04781             }
04782           }
04783         }
04784         DEBUG(dbgs() << "Dropping debug info for " << DI << "\n");
04785       }
04786     }
04787     return nullptr;
04788   }
04789   case Intrinsic::dbg_value: {
04790     const DbgValueInst &DI = cast<DbgValueInst>(I);
04791     DIVariable DIVar(DI.getVariable());
04792     assert((!DIVar || DIVar.isVariable()) &&
04793       "Variable in DbgValueInst should be either null or a DIVariable.");
04794     if (!DIVar)
04795       return nullptr;
04796 
04797     MDNode *Variable = DI.getVariable();
04798     uint64_t Offset = DI.getOffset();
04799     const Value *V = DI.getValue();
04800     if (!V)
04801       return nullptr;
04802 
04803     SDDbgValue *SDV;
04804     if (isa<ConstantInt>(V) || isa<ConstantFP>(V) || isa<UndefValue>(V)) {
04805       SDV = DAG.getDbgValue(Variable, V, Offset, dl, SDNodeOrder);
04806       DAG.AddDbgValue(SDV, nullptr, false);
04807     } else {
04808       // Do not use getValue() in here; we don't want to generate code at
04809       // this point if it hasn't been done yet.
04810       SDValue N = NodeMap[V];
04811       if (!N.getNode() && isa<Argument>(V))
04812         // Check unused arguments map.
04813         N = UnusedArgNodeMap[V];
04814       if (N.getNode()) {
04815         if (!EmitFuncArgumentDbgValue(V, Variable, Offset, N)) {
04816           SDV = DAG.getDbgValue(Variable, N.getNode(),
04817                                 N.getResNo(), Offset, dl, SDNodeOrder);
04818           DAG.AddDbgValue(SDV, N.getNode(), false);
04819         }
04820       } else if (!V->use_empty() ) {
04821         // Do not call getValue(V) yet, as we don't want to generate code.
04822         // Remember it for later.
04823         DanglingDebugInfo DDI(&DI, dl, SDNodeOrder);
04824         DanglingDebugInfoMap[V] = DDI;
04825       } else {
04826         // We may expand this to cover more cases.  One case where we have no
04827         // data available is an unreferenced parameter.
04828         DEBUG(dbgs() << "Dropping debug info for " << DI << "\n");
04829       }
04830     }
04831 
04832     // Build a debug info table entry.
04833     if (const BitCastInst *BCI = dyn_cast<BitCastInst>(V))
04834       V = BCI->getOperand(0);
04835     const AllocaInst *AI = dyn_cast<AllocaInst>(V);
04836     // Don't handle byval struct arguments or VLAs, for example.
04837     if (!AI) {
04838       DEBUG(dbgs() << "Dropping debug location info for:\n  " << DI << "\n");
04839       DEBUG(dbgs() << "  Last seen at:\n    " << *V << "\n");
04840       return nullptr;
04841     }
04842     DenseMap<const AllocaInst*, int>::iterator SI =
04843       FuncInfo.StaticAllocaMap.find(AI);
04844     if (SI == FuncInfo.StaticAllocaMap.end())
04845       return nullptr; // VLAs.
04846     int FI = SI->second;
04847 
04848     MachineModuleInfo &MMI = DAG.getMachineFunction().getMMI();
04849     if (!DI.getDebugLoc().isUnknown() && MMI.hasDebugInfo())
04850       MMI.setVariableDbgInfo(Variable, FI, DI.getDebugLoc());
04851     return nullptr;
04852   }
04853 
04854   case Intrinsic::eh_typeid_for: {
04855     // Find the type id for the given typeinfo.
04856     GlobalVariable *GV = ExtractTypeInfo(I.getArgOperand(0));
04857     unsigned TypeID = DAG.getMachineFunction().getMMI().getTypeIDFor(GV);
04858     Res = DAG.getConstant(TypeID, MVT::i32);
04859     setValue(&I, Res);
04860     return nullptr;
04861   }
04862 
04863   case Intrinsic::eh_return_i32:
04864   case Intrinsic::eh_return_i64:
04865     DAG.getMachineFunction().getMMI().setCallsEHReturn(true);
04866     DAG.setRoot(DAG.getNode(ISD::EH_RETURN, sdl,
04867                             MVT::Other,
04868                             getControlRoot(),
04869                             getValue(I.getArgOperand(0)),
04870                             getValue(I.getArgOperand(1))));
04871     return nullptr;
04872   case Intrinsic::eh_unwind_init:
04873     DAG.getMachineFunction().getMMI().setCallsUnwindInit(true);
04874     return nullptr;
04875   case Intrinsic::eh_dwarf_cfa: {
04876     SDValue CfaArg = DAG.getSExtOrTrunc(getValue(I.getArgOperand(0)), sdl,
04877                                         TLI->getPointerTy());
04878     SDValue Offset = DAG.getNode(ISD::ADD, sdl,
04879                                  CfaArg.getValueType(),
04880                                  DAG.getNode(ISD::FRAME_TO_ARGS_OFFSET, sdl,
04881                                              CfaArg.getValueType()),
04882                                  CfaArg);
04883     SDValue FA = DAG.getNode(ISD::FRAMEADDR, sdl,
04884                              TLI->getPointerTy(),
04885                              DAG.getConstant(0, TLI->getPointerTy()));
04886     setValue(&I, DAG.getNode(ISD::ADD, sdl, FA.getValueType(),
04887                              FA, Offset));
04888     return nullptr;
04889   }
04890   case Intrinsic::eh_sjlj_callsite: {
04891     MachineModuleInfo &MMI = DAG.getMachineFunction().getMMI();
04892     ConstantInt *CI = dyn_cast<ConstantInt>(I.getArgOperand(0));
04893     assert(CI && "Non-constant call site value in eh.sjlj.callsite!");
04894     assert(MMI.getCurrentCallSite() == 0 && "Overlapping call sites!");
04895 
04896     MMI.setCurrentCallSite(CI->getZExtValue());
04897     return nullptr;
04898   }
04899   case Intrinsic::eh_sjlj_functioncontext: {
04900     // Get and store the index of the function context.
04901     MachineFrameInfo *MFI = DAG.getMachineFunction().getFrameInfo();
04902     AllocaInst *FnCtx =
04903       cast<AllocaInst>(I.getArgOperand(0)->stripPointerCasts());
04904     int FI = FuncInfo.StaticAllocaMap[FnCtx];
04905     MFI->setFunctionContextIndex(FI);
04906     return nullptr;
04907   }
04908   case Intrinsic::eh_sjlj_setjmp: {
04909     SDValue Ops[2];
04910     Ops[0] = getRoot();
04911     Ops[1] = getValue(I.getArgOperand(0));
04912     SDValue Op = DAG.getNode(ISD::EH_SJLJ_SETJMP, sdl,
04913                              DAG.getVTList(MVT::i32, MVT::Other),
04914                              Ops, 2);
04915     setValue(&I, Op.getValue(0));
04916     DAG.setRoot(Op.getValue(1));
04917     return nullptr;
04918   }
04919   case Intrinsic::eh_sjlj_longjmp: {
04920     DAG.setRoot(DAG.getNode(ISD::EH_SJLJ_LONGJMP, sdl, MVT::Other,
04921                             getRoot(), getValue(I.getArgOperand(0))));
04922     return nullptr;
04923   }
04924 
04925   case Intrinsic::x86_mmx_pslli_w:
04926   case Intrinsic::x86_mmx_pslli_d:
04927   case Intrinsic::x86_mmx_pslli_q:
04928   case Intrinsic::x86_mmx_psrli_w:
04929   case Intrinsic::x86_mmx_psrli_d:
04930   case Intrinsic::x86_mmx_psrli_q:
04931   case Intrinsic::x86_mmx_psrai_w:
04932   case Intrinsic::x86_mmx_psrai_d: {
04933     SDValue ShAmt = getValue(I.getArgOperand(1));
04934     if (isa<ConstantSDNode>(ShAmt)) {
04935       visitTargetIntrinsic(I, Intrinsic);
04936       return nullptr;
04937     }
04938     unsigned NewIntrinsic = 0;
04939     EVT ShAmtVT = MVT::v2i32;
04940     switch (Intrinsic) {
04941     case Intrinsic::x86_mmx_pslli_w:
04942       NewIntrinsic = Intrinsic::x86_mmx_psll_w;
04943       break;
04944     case Intrinsic::x86_mmx_pslli_d:
04945       NewIntrinsic = Intrinsic::x86_mmx_psll_d;
04946       break;
04947     case Intrinsic::x86_mmx_pslli_q:
04948       NewIntrinsic = Intrinsic::x86_mmx_psll_q;
04949       break;
04950     case Intrinsic::x86_mmx_psrli_w:
04951       NewIntrinsic = Intrinsic::x86_mmx_psrl_w;
04952       break;
04953     case Intrinsic::x86_mmx_psrli_d:
04954       NewIntrinsic = Intrinsic::x86_mmx_psrl_d;
04955       break;
04956     case Intrinsic::x86_mmx_psrli_q:
04957       NewIntrinsic = Intrinsic::x86_mmx_psrl_q;
04958       break;
04959     case Intrinsic::x86_mmx_psrai_w:
04960       NewIntrinsic = Intrinsic::x86_mmx_psra_w;
04961       break;
04962     case Intrinsic::x86_mmx_psrai_d:
04963       NewIntrinsic = Intrinsic::x86_mmx_psra_d;
04964       break;
04965     default: llvm_unreachable("Impossible intrinsic");  // Can't reach here.
04966     }
04967 
04968     // The vector shift intrinsics with scalars uses 32b shift amounts but
04969     // the sse2/mmx shift instructions reads 64 bits. Set the upper 32 bits
04970     // to be zero.
04971     // We must do this early because v2i32 is not a legal type.
04972     SDValue ShOps[2];
04973     ShOps[0] = ShAmt;
04974     ShOps[1] = DAG.getConstant(0, MVT::i32);
04975     ShAmt =  DAG.getNode(ISD::BUILD_VECTOR, sdl, ShAmtVT, &ShOps[0], 2);
04976     EVT DestVT = TLI->getValueType(I.getType());
04977     ShAmt = DAG.getNode(ISD::BITCAST, sdl, DestVT, ShAmt);
04978     Res = DAG.getNode(ISD::INTRINSIC_WO_CHAIN, sdl, DestVT,
04979                        DAG.getConstant(NewIntrinsic, MVT::i32),
04980                        getValue(I.getArgOperand(0)), ShAmt);
04981     setValue(&I, Res);
04982     return nullptr;
04983   }
04984   case Intrinsic::x86_avx_vinsertf128_pd_256:
04985   case Intrinsic::x86_avx_vinsertf128_ps_256:
04986   case Intrinsic::x86_avx_vinsertf128_si_256:
04987   case Intrinsic::x86_avx2_vinserti128: {
04988     EVT DestVT = TLI->getValueType(I.getType());
04989     EVT ElVT = TLI->getValueType(I.getArgOperand(1)->getType());
04990     uint64_t Idx = (cast<ConstantInt>(I.getArgOperand(2))->getZExtValue() & 1) *
04991                    ElVT.getVectorNumElements();
04992     Res = DAG.getNode(ISD::INSERT_SUBVECTOR, sdl, DestVT,
04993                       getValue(I.getArgOperand(0)),
04994                       getValue(I.getArgOperand(1)),
04995                       DAG.getConstant(Idx, TLI->getVectorIdxTy()));
04996     setValue(&I, Res);
04997     return nullptr;
04998   }
04999   case Intrinsic::x86_avx_vextractf128_pd_256:
05000   case Intrinsic::x86_avx_vextractf128_ps_256:
05001   case Intrinsic::x86_avx_vextractf128_si_256:
05002   case Intrinsic::x86_avx2_vextracti128: {
05003     EVT DestVT = TLI->getValueType(I.getType());
05004     uint64_t Idx = (cast<ConstantInt>(I.getArgOperand(1))->getZExtValue() & 1) *
05005                    DestVT.getVectorNumElements();
05006     Res = DAG.getNode(ISD::EXTRACT_SUBVECTOR, sdl, DestVT,
05007                       getValue(I.getArgOperand(0)),
05008                       DAG.getConstant(Idx, TLI->getVectorIdxTy()));
05009     setValue(&I, Res);
05010     return nullptr;
05011   }
05012   case Intrinsic::convertff:
05013   case Intrinsic::convertfsi:
05014   case Intrinsic::convertfui:
05015   case Intrinsic::convertsif:
05016   case Intrinsic::convertuif:
05017   case Intrinsic::convertss:
05018   case Intrinsic::convertsu:
05019   case Intrinsic::convertus:
05020   case Intrinsic::convertuu: {
05021     ISD::CvtCode Code = ISD::CVT_INVALID;
05022     switch (Intrinsic) {
05023     default: llvm_unreachable("Impossible intrinsic");  // Can't reach here.
05024     case Intrinsic::convertff:  Code = ISD::CVT_FF; break;
05025     case Intrinsic::convertfsi: Code = ISD::CVT_FS; break;
05026     case Intrinsic::convertfui: Code = ISD::CVT_FU; break;
05027     case Intrinsic::convertsif: Code = ISD::CVT_SF; break;
05028     case Intrinsic::convertuif: Code = ISD::CVT_UF; break;
05029     case Intrinsic::convertss:  Code = ISD::CVT_SS; break;
05030     case Intrinsic::convertsu:  Code = ISD::CVT_SU; break;
05031     case Intrinsic::convertus:  Code = ISD::CVT_US; break;
05032     case Intrinsic::convertuu:  Code = ISD::CVT_UU; break;
05033     }
05034     EVT DestVT = TLI->getValueType(I.getType());
05035     const Value *Op1 = I.getArgOperand(0);
05036     Res = DAG.getConvertRndSat(DestVT, sdl, getValue(Op1),
05037                                DAG.getValueType(DestVT),
05038                                DAG.getValueType(getValue(Op1).getValueType()),
05039                                getValue(I.getArgOperand(1)),
05040                                getValue(I.getArgOperand(2)),
05041                                Code);
05042     setValue(&I, Res);
05043     return nullptr;
05044   }
05045   case Intrinsic::powi:
05046     setValue(&I, ExpandPowI(sdl, getValue(I.getArgOperand(0)),
05047                             getValue(I.getArgOperand(1)), DAG));
05048     return nullptr;
05049   case Intrinsic::log:
05050     setValue(&I, expandLog(sdl, getValue(I.getArgOperand(0)), DAG, *TLI));
05051     return nullptr;
05052   case Intrinsic::log2:
05053     setValue(&I, expandLog2(sdl, getValue(I.getArgOperand(0)), DAG, *TLI));
05054     return nullptr;
05055   case Intrinsic::log10:
05056     setValue(&I, expandLog10(sdl, getValue(I.getArgOperand(0)), DAG, *TLI));
05057     return nullptr;
05058   case Intrinsic::exp:
05059     setValue(&I, expandExp(sdl, getValue(I.getArgOperand(0)), DAG, *TLI));
05060     return nullptr;
05061   case Intrinsic::exp2:
05062     setValue(&I, expandExp2(sdl, getValue(I.getArgOperand(0)), DAG, *TLI));
05063     return nullptr;
05064   case Intrinsic::pow:
05065     setValue(&I, expandPow(sdl, getValue(I.getArgOperand(0)),
05066                            getValue(I.getArgOperand(1)), DAG, *TLI));
05067     return nullptr;
05068   case Intrinsic::sqrt:
05069   case Intrinsic::fabs:
05070   case Intrinsic::sin:
05071   case Intrinsic::cos:
05072   case Intrinsic::floor:
05073   case Intrinsic::ceil:
05074   case Intrinsic::trunc:
05075   case Intrinsic::rint:
05076   case Intrinsic::nearbyint:
05077   case Intrinsic::round: {
05078     unsigned Opcode;
05079     switch (Intrinsic) {
05080     default: llvm_unreachable("Impossible intrinsic");  // Can't reach here.
05081     case Intrinsic::sqrt:      Opcode = ISD::FSQRT;      break;
05082     case Intrinsic::fabs:      Opcode = ISD::FABS;       break;
05083     case Intrinsic::sin:       Opcode = ISD::FSIN;       break;
05084     case Intrinsic::cos:       Opcode = ISD::FCOS;       break;
05085     case Intrinsic::floor:     Opcode = ISD::FFLOOR;     break;
05086     case Intrinsic::ceil:      Opcode = ISD::FCEIL;      break;
05087     case Intrinsic::trunc:     Opcode = ISD::FTRUNC;     break;
05088     case Intrinsic::rint:      Opcode = ISD::FRINT;      break;
05089     case Intrinsic::nearbyint: Opcode = ISD::FNEARBYINT; break;
05090     case Intrinsic::round:     Opcode = ISD::FROUND;     break;
05091     }
05092 
05093     setValue(&I, DAG.getNode(Opcode, sdl,
05094                              getValue(I.getArgOperand(0)).getValueType(),
05095                              getValue(I.getArgOperand(0))));
05096     return nullptr;
05097   }
05098   case Intrinsic::copysign:
05099     setValue(&I, DAG.getNode(ISD::FCOPYSIGN, sdl,
05100                              getValue(I.getArgOperand(0)).getValueType(),
05101                              getValue(I.getArgOperand(0)),
05102                              getValue(I.getArgOperand(1))));
05103     return nullptr;
05104   case Intrinsic::fma:
05105     setValue(&I, DAG.getNode(ISD::FMA, sdl,
05106                              getValue(I.getArgOperand(0)).getValueType(),
05107                              getValue(I.getArgOperand(0)),
05108                              getValue(I.getArgOperand(1)),
05109                              getValue(I.getArgOperand(2))));
05110     return nullptr;
05111   case Intrinsic::fmuladd: {
05112     EVT VT = TLI->getValueType(I.getType());
05113     if (TM.Options.AllowFPOpFusion != FPOpFusion::Strict &&
05114         TLI->isFMAFasterThanFMulAndFAdd(VT)) {
05115       setValue(&I, DAG.getNode(ISD::FMA, sdl,
05116                                getValue(I.getArgOperand(0)).getValueType(),
05117                                getValue(I.getArgOperand(0)),
05118                                getValue(I.getArgOperand(1)),
05119                                getValue(I.getArgOperand(2))));
05120     } else {
05121       SDValue Mul = DAG.getNode(ISD::FMUL, sdl,
05122                                 getValue(I.getArgOperand(0)).getValueType(),
05123                                 getValue(I.getArgOperand(0)),
05124                                 getValue(I.getArgOperand(1)));
05125       SDValue Add = DAG.getNode(ISD::FADD, sdl,
05126                                 getValue(I.getArgOperand(0)).getValueType(),
05127                                 Mul,
05128                                 getValue(I.getArgOperand(2)));
05129       setValue(&I, Add);
05130     }
05131     return nullptr;
05132   }
05133   case Intrinsic::convert_to_fp16:
05134     setValue(&I, DAG.getNode(ISD::FP32_TO_FP16, sdl,
05135                              MVT::i16, getValue(I.getArgOperand(0))));
05136     return nullptr;
05137   case Intrinsic::convert_from_fp16:
05138     setValue(&I, DAG.getNode(ISD::FP16_TO_FP32, sdl,
05139                              MVT::f32, getValue(I.getArgOperand(0))));
05140     return nullptr;
05141   case Intrinsic::pcmarker: {
05142     SDValue Tmp = getValue(I.getArgOperand(0));
05143     DAG.setRoot(DAG.getNode(ISD::PCMARKER, sdl, MVT::Other, getRoot(), Tmp));
05144     return nullptr;
05145   }
05146   case Intrinsic::readcyclecounter: {
05147     SDValue Op = getRoot();
05148     Res = DAG.getNode(ISD::READCYCLECOUNTER, sdl,
05149                       DAG.getVTList(MVT::i64, MVT::Other),
05150                       &Op, 1);
05151     setValue(&I, Res);
05152     DAG.setRoot(Res.getValue(1));
05153     return nullptr;
05154   }
05155   case Intrinsic::bswap:
05156     setValue(&I, DAG.getNode(ISD::BSWAP, sdl,
05157                              getValue(I.getArgOperand(0)).getValueType(),
05158                              getValue(I.getArgOperand(0))));
05159     return nullptr;
05160   case Intrinsic::cttz: {
05161     SDValue Arg = getValue(I.getArgOperand(0));
05162     ConstantInt *CI = cast<ConstantInt>(I.getArgOperand(1));
05163     EVT Ty = Arg.getValueType();
05164     setValue(&I, DAG.getNode(CI->isZero() ? ISD::CTTZ : ISD::CTTZ_ZERO_UNDEF,
05165                              sdl, Ty, Arg));
05166     return nullptr;
05167   }
05168   case Intrinsic::ctlz: {
05169     SDValue Arg = getValue(I.getArgOperand(0));
05170     ConstantInt *CI = cast<ConstantInt>(I.getArgOperand(1));
05171     EVT Ty = Arg.getValueType();
05172     setValue(&I, DAG.getNode(CI->isZero() ? ISD::CTLZ : ISD::CTLZ_ZERO_UNDEF,
05173                              sdl, Ty, Arg));
05174     return nullptr;
05175   }
05176   case Intrinsic::ctpop: {
05177     SDValue Arg = getValue(I.getArgOperand(0));
05178     EVT Ty = Arg.getValueType();
05179     setValue(&I, DAG.getNode(ISD::CTPOP, sdl, Ty, Arg));
05180     return nullptr;
05181   }
05182   case Intrinsic::stacksave: {
05183     SDValue Op = getRoot();
05184     Res = DAG.getNode(ISD::STACKSAVE, sdl,
05185                       DAG.getVTList(TLI->getPointerTy(), MVT::Other), &Op, 1);
05186     setValue(&I, Res);
05187     DAG.setRoot(Res.getValue(1));
05188     return nullptr;
05189   }
05190   case Intrinsic::stackrestore: {
05191     Res = getValue(I.getArgOperand(0));
05192     DAG.setRoot(DAG.getNode(ISD::STACKRESTORE, sdl, MVT::Other, getRoot(), Res));
05193     return nullptr;
05194   }
05195   case Intrinsic::stackprotector: {
05196     // Emit code into the DAG to store the stack guard onto the stack.
05197     MachineFunction &MF = DAG.getMachineFunction();
05198     MachineFrameInfo *MFI = MF.getFrameInfo();
05199     EVT PtrTy = TLI->getPointerTy();
05200 
05201     SDValue Src = getValue(I.getArgOperand(0));   // The guard's value.
05202     AllocaInst *Slot = cast<AllocaInst>(I.getArgOperand(1));
05203 
05204     int FI = FuncInfo.StaticAllocaMap[Slot];
05205     MFI->setStackProtectorIndex(FI);
05206 
05207     SDValue FIN = DAG.getFrameIndex(FI, PtrTy);
05208 
05209     // Store the stack protector onto the stack.
05210     Res = DAG.getStore(getRoot(), sdl, Src, FIN,
05211                        MachinePointerInfo::getFixedStack(FI),
05212                        true, false, 0);
05213     setValue(&I, Res);
05214     DAG.setRoot(Res);
05215     return nullptr;
05216   }
05217   case Intrinsic::objectsize: {
05218     // If we don't know by now, we're never going to know.
05219     ConstantInt *CI = dyn_cast<ConstantInt>(I.getArgOperand(1));
05220 
05221     assert(CI && "Non-constant type in __builtin_object_size?");
05222 
05223     SDValue Arg = getValue(I.getCalledValue());
05224     EVT Ty = Arg.getValueType();
05225 
05226     if (CI->isZero())
05227       Res = DAG.getConstant(-1ULL, Ty);
05228     else
05229       Res = DAG.getConstant(0, Ty);
05230 
05231     setValue(&I, Res);
05232     return nullptr;
05233   }
05234   case Intrinsic::annotation:
05235   case Intrinsic::ptr_annotation:
05236     // Drop the intrinsic, but forward the value
05237     setValue(&I, getValue(I.getOperand(0)));
05238     return nullptr;
05239   case Intrinsic::var_annotation:
05240     // Discard annotate attributes
05241     return nullptr;
05242 
05243   case Intrinsic::init_trampoline: {
05244     const Function *F = cast<Function>(I.getArgOperand(1)->stripPointerCasts());
05245 
05246     SDValue Ops[6];
05247     Ops[0] = getRoot();
05248     Ops[1] = getValue(I.getArgOperand(0));
05249     Ops[2] = getValue(I.getArgOperand(1));
05250     Ops[3] = getValue(I.getArgOperand(2));
05251     Ops[4] = DAG.getSrcValue(I.getArgOperand(0));
05252     Ops[5] = DAG.getSrcValue(F);
05253 
05254     Res = DAG.getNode(ISD::INIT_TRAMPOLINE, sdl, MVT::Other, Ops, 6);
05255 
05256     DAG.setRoot(Res);
05257     return nullptr;
05258   }
05259   case Intrinsic::adjust_trampoline: {
05260     setValue(&I, DAG.getNode(ISD::ADJUST_TRAMPOLINE, sdl,
05261                              TLI->getPointerTy(),
05262                              getValue(I.getArgOperand(0))));
05263     return nullptr;
05264   }
05265   case Intrinsic::gcroot:
05266     if (GFI) {
05267       const Value *Alloca = I.getArgOperand(0)->stripPointerCasts();
05268       const Constant *TypeMap = cast<Constant>(I.getArgOperand(1));
05269 
05270       FrameIndexSDNode *FI = cast<FrameIndexSDNode>(getValue(Alloca).getNode());
05271       GFI->addStackRoot(FI->getIndex(), TypeMap);
05272     }
05273     return nullptr;
05274   case Intrinsic::gcread:
05275   case Intrinsic::gcwrite:
05276     llvm_unreachable("GC failed to lower gcread/gcwrite intrinsics!");
05277   case Intrinsic::flt_rounds:
05278     setValue(&I, DAG.getNode(ISD::FLT_ROUNDS_, sdl, MVT::i32));
05279     return nullptr;
05280 
05281   case Intrinsic::expect: {
05282     // Just replace __builtin_expect(exp, c) with EXP.
05283     setValue(&I, getValue(I.getArgOperand(0)));
05284     return nullptr;
05285   }
05286 
05287   case Intrinsic::debugtrap:
05288   case Intrinsic::trap: {
05289     StringRef TrapFuncName = TM.Options.getTrapFunctionName();
05290     if (TrapFuncName.empty()) {
05291       ISD::NodeType Op = (Intrinsic == Intrinsic::trap) ?
05292         ISD::TRAP : ISD::DEBUGTRAP;
05293       DAG.setRoot(DAG.getNode(Op, sdl,MVT::Other, getRoot()));
05294       return nullptr;
05295     }
05296     TargetLowering::ArgListTy Args;
05297     TargetLowering::
05298     CallLoweringInfo CLI(getRoot(), I.getType(),
05299                  false, false, false, false, 0, CallingConv::C,
05300                  /*isTailCall=*/false,
05301                  /*doesNotRet=*/false, /*isReturnValueUsed=*/true,
05302                  DAG.getExternalSymbol(TrapFuncName.data(),
05303                                        TLI->getPointerTy()),
05304                  Args, DAG, sdl);
05305     std::pair<SDValue, SDValue> Result = TLI->LowerCallTo(CLI);
05306     DAG.setRoot(Result.second);
05307     return nullptr;
05308   }
05309 
05310   case Intrinsic::uadd_with_overflow:
05311   case Intrinsic::sadd_with_overflow:
05312   case Intrinsic::usub_with_overflow:
05313   case Intrinsic::ssub_with_overflow:
05314   case Intrinsic::umul_with_overflow:
05315   case Intrinsic::smul_with_overflow: {
05316     ISD::NodeType Op;
05317     switch (Intrinsic) {
05318     default: llvm_unreachable("Impossible intrinsic");  // Can't reach here.
05319     case Intrinsic::uadd_with_overflow: Op = ISD::UADDO; break;
05320     case Intrinsic::sadd_with_overflow: Op = ISD::SADDO; break;
05321     case Intrinsic::usub_with_overflow: Op = ISD::USUBO; break;
05322     case Intrinsic::ssub_with_overflow: Op = ISD::SSUBO; break;
05323     case Intrinsic::umul_with_overflow: Op = ISD::UMULO; break;
05324     case Intrinsic::smul_with_overflow: Op = ISD::SMULO; break;
05325     }
05326     SDValue Op1 = getValue(I.getArgOperand(0));
05327     SDValue Op2 = getValue(I.getArgOperand(1));
05328 
05329     SDVTList VTs = DAG.getVTList(Op1.getValueType(), MVT::i1);
05330     setValue(&I, DAG.getNode(Op, sdl, VTs, Op1, Op2));
05331     return nullptr;
05332   }
05333   case Intrinsic::prefetch: {
05334     SDValue Ops[5];
05335     unsigned rw = cast<ConstantInt>(I.getArgOperand(1))->getZExtValue();
05336     Ops[0] = getRoot();
05337     Ops[1] = getValue(I.getArgOperand(0));
05338     Ops[2] = getValue(I.getArgOperand(1));
05339     Ops[3] = getValue(I.getArgOperand(2));
05340     Ops[4] = getValue(I.getArgOperand(3));
05341     DAG.setRoot(DAG.getMemIntrinsicNode(ISD::PREFETCH, sdl,
05342                                         DAG.getVTList(MVT::Other),
05343                                         &Ops[0], 5,
05344                                         EVT::getIntegerVT(*Context, 8),
05345                                         MachinePointerInfo(I.getArgOperand(0)),
05346                                         0, /* align */
05347                                         false, /* volatile */
05348                                         rw==0, /* read */
05349                                         rw==1)); /* write */
05350     return nullptr;
05351   }
05352   case Intrinsic::lifetime_start:
05353   case Intrinsic::lifetime_end: {
05354     bool IsStart = (Intrinsic == Intrinsic::lifetime_start);
05355     // Stack coloring is not enabled in O0, discard region information.
05356     if (TM.getOptLevel() == CodeGenOpt::None)
05357       return nullptr;
05358 
05359     SmallVector<Value *, 4> Allocas;
05360     GetUnderlyingObjects(I.getArgOperand(1), Allocas, DL);
05361 
05362     for (SmallVectorImpl<Value*>::iterator Object = Allocas.begin(),
05363            E = Allocas.end(); Object != E; ++Object) {
05364       AllocaInst *LifetimeObject = dyn_cast_or_null<AllocaInst>(*Object);
05365 
05366       // Could not find an Alloca.
05367       if (!LifetimeObject)
05368         continue;
05369 
05370       int FI = FuncInfo.StaticAllocaMap[LifetimeObject];
05371 
05372       SDValue Ops[2];
05373       Ops[0] = getRoot();
05374       Ops[1] = DAG.getFrameIndex(FI, TLI->getPointerTy(), true);
05375       unsigned Opcode = (IsStart ? ISD::LIFETIME_START : ISD::LIFETIME_END);
05376 
05377       Res = DAG.getNode(Opcode, sdl, MVT::Other, Ops, 2);
05378       DAG.setRoot(Res);
05379     }
05380     return nullptr;
05381   }
05382   case Intrinsic::invariant_start:
05383     // Discard region information.
05384     setValue(&I, DAG.getUNDEF(TLI->getPointerTy()));
05385     return nullptr;
05386   case Intrinsic::invariant_end:
05387     // Discard region information.
05388     return nullptr;
05389   case Intrinsic::stackprotectorcheck: {
05390     // Do not actually emit anything for this basic block. Instead we initialize
05391     // the stack protector descriptor and export the guard variable so we can
05392     // access it in FinishBasicBlock.
05393     const BasicBlock *BB = I.getParent();
05394     SPDescriptor.initialize(BB, FuncInfo.MBBMap[BB], I);
05395     ExportFromCurrentBlock(SPDescriptor.getGuard());
05396 
05397     // Flush our exports since we are going to process a terminator.
05398     (void)getControlRoot();
05399     return nullptr;
05400   }
05401   case Intrinsic::clear_cache:
05402     return TLI->getClearCacheBuiltinName();
05403   case Intrinsic::donothing:
05404     // ignore
05405     return nullptr;
05406   case Intrinsic::experimental_stackmap: {
05407     visitStackmap(I);
05408     return nullptr;
05409   }
05410   case Intrinsic::experimental_patchpoint_void:
05411   case Intrinsic::experimental_patchpoint_i64: {
05412     visitPatchpoint(I);
05413     return nullptr;
05414   }
05415   }
05416 }
05417 
05418 void SelectionDAGBuilder::LowerCallTo(ImmutableCallSite CS, SDValue Callee,
05419                                       bool isTailCall,
05420                                       MachineBasicBlock *LandingPad) {
05421   PointerType *PT = cast<PointerType>(CS.getCalledValue()->getType());
05422   FunctionType *FTy = cast<FunctionType>(PT->getElementType());
05423   Type *RetTy = FTy->getReturnType();
05424   MachineModuleInfo &MMI = DAG.getMachineFunction().getMMI();
05425   MCSymbol *BeginLabel = nullptr;
05426 
05427   TargetLowering::ArgListTy Args;
05428   TargetLowering::ArgListEntry Entry;
05429   Args.reserve(CS.arg_size());
05430 
05431   // Check whether the function can return without sret-demotion.
05432   SmallVector<ISD::OutputArg, 4> Outs;
05433   const TargetLowering *TLI = TM.getTargetLowering();
05434   GetReturnInfo(RetTy, CS.getAttributes(), Outs, *TLI);
05435 
05436   bool CanLowerReturn = TLI->CanLowerReturn(CS.getCallingConv(),
05437                                             DAG.getMachineFunction(),
05438                                             FTy->isVarArg(), Outs,
05439                                             FTy->getContext());
05440 
05441   SDValue DemoteStackSlot;
05442   int DemoteStackIdx = -100;
05443 
05444   if (!CanLowerReturn) {
05445     assert(!CS.hasInAllocaArgument() &&
05446            "sret demotion is incompatible with inalloca");
05447     uint64_t TySize = TLI->getDataLayout()->getTypeAllocSize(
05448                       FTy->getReturnType());
05449     unsigned Align  = TLI->getDataLayout()->getPrefTypeAlignment(
05450                       FTy->getReturnType());
05451     MachineFunction &MF = DAG.getMachineFunction();
05452     DemoteStackIdx = MF.getFrameInfo()->CreateStackObject(TySize, Align, false);
05453     Type *StackSlotPtrType = PointerType::getUnqual(FTy->getReturnType());
05454 
05455     DemoteStackSlot = DAG.getFrameIndex(DemoteStackIdx, TLI->getPointerTy());
05456     Entry.Node = DemoteStackSlot;
05457     Entry.Ty = StackSlotPtrType;
05458     Entry.isSExt = false;
05459     Entry.isZExt = false;
05460     Entry.isInReg = false;
05461     Entry.isSRet = true;
05462     Entry.isNest = false;
05463     Entry.isByVal = false;
05464     Entry.isReturned = false;
05465     Entry.Alignment = Align;
05466     Args.push_back(Entry);
05467     RetTy = Type::getVoidTy(FTy->getContext());
05468   }
05469 
05470   for (ImmutableCallSite::arg_iterator i = CS.arg_begin(), e = CS.arg_end();
05471        i != e; ++i) {
05472     const Value *V = *i;
05473 
05474     // Skip empty types
05475     if (V->getType()->isEmptyTy())
05476       continue;
05477 
05478     SDValue ArgNode = getValue(V);
05479     Entry.Node = ArgNode; Entry.Ty = V->getType();
05480 
05481     // Skip the first return-type Attribute to get to params.
05482     Entry.setAttributes(&CS, i - CS.arg_begin() + 1);
05483     Args.push_back(Entry);
05484   }
05485 
05486   if (LandingPad) {
05487     // Insert a label before the invoke call to mark the try range.  This can be
05488     // used to detect deletion of the invoke via the MachineModuleInfo.
05489     BeginLabel = MMI.getContext().CreateTempSymbol();
05490 
05491     // For SjLj, keep track of which landing pads go with which invokes
05492     // so as to maintain the ordering of pads in the LSDA.
05493     unsigned CallSiteIndex = MMI.getCurrentCallSite();
05494     if (CallSiteIndex) {
05495       MMI.setCallSiteBeginLabel(BeginLabel, CallSiteIndex);
05496       LPadToCallSiteMap[LandingPad].push_back(CallSiteIndex);
05497 
05498       // Now that the call site is handled, stop tracking it.
05499       MMI.setCurrentCallSite(0);
05500     }
05501 
05502     // Both PendingLoads and PendingExports must be flushed here;
05503     // this call might not return.
05504     (void)getRoot();
05505     DAG.setRoot(DAG.getEHLabel(getCurSDLoc(), getControlRoot(), BeginLabel));
05506   }
05507 
05508   // Check if target-independent constraints permit a tail call here.
05509   // Target-dependent constraints are checked within TLI->LowerCallTo.
05510   if (isTailCall && !isInTailCallPosition(CS, *TLI))
05511     isTailCall = false;
05512 
05513   TargetLowering::
05514   CallLoweringInfo CLI(getRoot(), RetTy, FTy, isTailCall, Callee, Args, DAG,
05515                        getCurSDLoc(), CS);
05516   std::pair<SDValue,SDValue> Result = TLI->LowerCallTo(CLI);
05517   assert((isTailCall || Result.second.getNode()) &&
05518          "Non-null chain expected with non-tail call!");
05519   assert((Result.second.getNode() || !Result.first.getNode()) &&
05520          "Null value expected with tail call!");
05521   if (Result.first.getNode()) {
05522     setValue(CS.getInstruction(), Result.first);
05523   } else if (!CanLowerReturn && Result.second.getNode()) {
05524     // The instruction result is the result of loading from the
05525     // hidden sret parameter.
05526     SmallVector<EVT, 1> PVTs;
05527     Type *PtrRetTy = PointerType::getUnqual(FTy->getReturnType());
05528 
05529     ComputeValueVTs(*TLI, PtrRetTy, PVTs);
05530     assert(PVTs.size() == 1 && "Pointers should fit in one register");
05531     EVT PtrVT = PVTs[0];
05532 
05533     SmallVector<EVT, 4> RetTys;
05534     SmallVector<uint64_t, 4> Offsets;
05535     RetTy = FTy->getReturnType();
05536     ComputeValueVTs(*TLI, RetTy, RetTys, &Offsets);
05537 
05538     unsigned NumValues = RetTys.size();
05539     SmallVector<SDValue, 4> Values(NumValues);
05540     SmallVector<SDValue, 4> Chains(NumValues);
05541 
05542     for (unsigned i = 0; i < NumValues; ++i) {
05543       SDValue Add = DAG.getNode(ISD::ADD, getCurSDLoc(), PtrVT,
05544                                 DemoteStackSlot,
05545                                 DAG.getConstant(Offsets[i], PtrVT));
05546       SDValue L = DAG.getLoad(RetTys[i], getCurSDLoc(), Result.second, Add,
05547                   MachinePointerInfo::getFixedStack(DemoteStackIdx, Offsets[i]),
05548                               false, false, false, 1);
05549       Values[i] = L;
05550       Chains[i] = L.getValue(1);
05551     }
05552 
05553     SDValue Chain = DAG.getNode(ISD::TokenFactor, getCurSDLoc(),
05554                                 MVT::Other, &Chains[0], NumValues);
05555     PendingLoads.push_back(Chain);
05556 
05557     setValue(CS.getInstruction(),
05558              DAG.getNode(ISD::MERGE_VALUES, getCurSDLoc(),
05559                          DAG.getVTList(RetTys),
05560                          &Values[0], Values.size()));
05561   }
05562 
05563   if (!Result.second.getNode()) {
05564     // As a special case, a null chain means that a tail call has been emitted
05565     // and the DAG root is already updated.
05566     HasTailCall = true;
05567 
05568     // Since there's no actual continuation from this block, nothing can be
05569     // relying on us setting vregs for them.
05570     PendingExports.clear();
05571   } else {
05572     DAG.setRoot(Result.second);
05573   }
05574 
05575   if (LandingPad) {
05576     // Insert a label at the end of the invoke call to mark the try range.  This
05577     // can be used to detect deletion of the invoke via the MachineModuleInfo.
05578     MCSymbol *EndLabel = MMI.getContext().CreateTempSymbol();
05579     DAG.setRoot(DAG.getEHLabel(getCurSDLoc(), getRoot(), EndLabel));
05580 
05581     // Inform MachineModuleInfo of range.
05582     MMI.addInvoke(LandingPad, BeginLabel, EndLabel);
05583   }
05584 }
05585 
05586 /// IsOnlyUsedInZeroEqualityComparison - Return true if it only matters that the
05587 /// value is equal or not-equal to zero.
05588 static bool IsOnlyUsedInZeroEqualityComparison(const Value *V) {
05589   for (const User *U : V->users()) {
05590     if (const ICmpInst *IC = dyn_cast<ICmpInst>(U))
05591       if (IC->isEquality())
05592         if (const Constant *C = dyn_cast<Constant>(IC->getOperand(1)))
05593           if (C->isNullValue())
05594             continue;
05595     // Unknown instruction.
05596     return false;
05597   }
05598   return true;
05599 }
05600 
05601 static SDValue getMemCmpLoad(const Value *PtrVal, MVT LoadVT,
05602                              Type *LoadTy,
05603                              SelectionDAGBuilder &Builder) {
05604 
05605   // Check to see if this load can be trivially constant folded, e.g. if the
05606   // input is from a string literal.
05607   if (const Constant *LoadInput = dyn_cast<Constant>(PtrVal)) {
05608     // Cast pointer to the type we really want to load.
05609     LoadInput = ConstantExpr::getBitCast(const_cast<Constant *>(LoadInput),
05610                                          PointerType::getUnqual(LoadTy));
05611 
05612     if (const Constant *LoadCst =
05613           ConstantFoldLoadFromConstPtr(const_cast<Constant *>(LoadInput),
05614                                        Builder.DL))
05615       return Builder.getValue(LoadCst);
05616   }
05617 
05618   // Otherwise, we have to emit the load.  If the pointer is to unfoldable but
05619   // still constant memory, the input chain can be the entry node.
05620   SDValue Root;
05621   bool ConstantMemory = false;
05622 
05623   // Do not serialize (non-volatile) loads of constant memory with anything.
05624   if (Builder.AA->pointsToConstantMemory(PtrVal)) {
05625     Root = Builder.DAG.getEntryNode();
05626     ConstantMemory = true;
05627   } else {
05628     // Do not serialize non-volatile loads against each other.
05629     Root = Builder.DAG.getRoot();
05630   }
05631 
05632   SDValue Ptr = Builder.getValue(PtrVal);
05633   SDValue LoadVal = Builder.DAG.getLoad(LoadVT, Builder.getCurSDLoc(), Root,
05634                                         Ptr, MachinePointerInfo(PtrVal),
05635                                         false /*volatile*/,
05636                                         false /*nontemporal*/,
05637                                         false /*isinvariant*/, 1 /* align=1 */);
05638 
05639   if (!ConstantMemory)
05640     Builder.PendingLoads.push_back(LoadVal.getValue(1));
05641   return LoadVal;
05642 }
05643 
05644 /// processIntegerCallValue - Record the value for an instruction that
05645 /// produces an integer result, converting the type where necessary.
05646 void SelectionDAGBuilder::processIntegerCallValue(const Instruction &I,
05647                                                   SDValue Value,
05648                                                   bool IsSigned) {
05649   EVT VT = TM.getTargetLowering()->getValueType(I.getType(), true);
05650   if (IsSigned)
05651     Value = DAG.getSExtOrTrunc(Value, getCurSDLoc(), VT);
05652   else
05653     Value = DAG.getZExtOrTrunc(Value, getCurSDLoc(), VT);
05654   setValue(&I, Value);
05655 }
05656 
05657 /// visitMemCmpCall - See if we can lower a call to memcmp in an optimized form.
05658 /// If so, return true and lower it, otherwise return false and it will be
05659 /// lowered like a normal call.
05660 bool SelectionDAGBuilder::visitMemCmpCall(const CallInst &I) {
05661   // Verify that the prototype makes sense.  int memcmp(void*,void*,size_t)
05662   if (I.getNumArgOperands() != 3)
05663     return false;
05664 
05665   const Value *LHS = I.getArgOperand(0), *RHS = I.getArgOperand(1);
05666   if (!LHS->getType()->isPointerTy() || !RHS->getType()->isPointerTy() ||
05667       !I.getArgOperand(2)->getType()->isIntegerTy() ||
05668       !I.getType()->isIntegerTy())
05669     return false;
05670 
05671   const Value *Size = I.getArgOperand(2);
05672   const ConstantInt *CSize = dyn_cast<ConstantInt>(Size);
05673   if (CSize && CSize->getZExtValue() == 0) {
05674     EVT CallVT = TM.getTargetLowering()->getValueType(I.getType(), true);
05675     setValue(&I, DAG.getConstant(0, CallVT));
05676     return true;
05677   }
05678 
05679   const TargetSelectionDAGInfo &TSI = DAG.getSelectionDAGInfo();
05680   std::pair<SDValue, SDValue> Res =
05681     TSI.EmitTargetCodeForMemcmp(DAG, getCurSDLoc(), DAG.getRoot(),
05682                                 getValue(LHS), getValue(RHS), getValue(Size),
05683                                 MachinePointerInfo(LHS),
05684                                 MachinePointerInfo(RHS));
05685   if (Res.first.getNode()) {
05686     processIntegerCallValue(I, Res.first, true);
05687     PendingLoads.push_back(Res.second);
05688     return true;
05689   }
05690 
05691   // memcmp(S1,S2,2) != 0 -> (*(short*)LHS != *(short*)RHS)  != 0
05692   // memcmp(S1,S2,4) != 0 -> (*(int*)LHS != *(int*)RHS)  != 0
05693   if (CSize && IsOnlyUsedInZeroEqualityComparison(&I)) {
05694     bool ActuallyDoIt = true;
05695     MVT LoadVT;
05696     Type *LoadTy;
05697     switch (CSize->getZExtValue()) {
05698     default:
05699       LoadVT = MVT::Other;
05700       LoadTy = nullptr;
05701       ActuallyDoIt = false;
05702       break;
05703     case 2:
05704       LoadVT = MVT::i16;
05705       LoadTy = Type::getInt16Ty(CSize->getContext());
05706       break;
05707     case 4:
05708       LoadVT = MVT::i32;
05709       LoadTy = Type::getInt32Ty(CSize->getContext());
05710       break;
05711     case 8:
05712       LoadVT = MVT::i64;
05713       LoadTy = Type::getInt64Ty(CSize->getContext());
05714       break;
05715         /*
05716     case 16:
05717       LoadVT = MVT::v4i32;
05718       LoadTy = Type::getInt32Ty(CSize->getContext());
05719       LoadTy = VectorType::get(LoadTy, 4);
05720       break;
05721          */
05722     }
05723 
05724     // This turns into unaligned loads.  We only do this if the target natively
05725     // supports the MVT we'll be loading or if it is small enough (<= 4) that
05726     // we'll only produce a small number of byte loads.
05727 
05728     // Require that we can find a legal MVT, and only do this if the target
05729     // supports unaligned loads of that type.  Expanding into byte loads would
05730     // bloat the code.
05731     const TargetLowering *TLI = TM.getTargetLowering();
05732     if (ActuallyDoIt && CSize->getZExtValue() > 4) {
05733       unsigned DstAS = LHS->getType()->getPointerAddressSpace();
05734       unsigned SrcAS = RHS->getType()->getPointerAddressSpace();
05735       // TODO: Handle 5 byte compare as 4-byte + 1 byte.
05736       // TODO: Handle 8 byte compare on x86-32 as two 32-bit loads.
05737       if (!TLI->isTypeLegal(LoadVT) ||
05738           !TLI->allowsUnalignedMemoryAccesses(LoadVT, SrcAS) ||
05739           !TLI->allowsUnalignedMemoryAccesses(LoadVT, DstAS))
05740         ActuallyDoIt = false;
05741     }
05742 
05743     if (ActuallyDoIt) {
05744       SDValue LHSVal = getMemCmpLoad(LHS, LoadVT, LoadTy, *this);
05745       SDValue RHSVal = getMemCmpLoad(RHS, LoadVT, LoadTy, *this);
05746 
05747       SDValue Res = DAG.getSetCC(getCurSDLoc(), MVT::i1, LHSVal, RHSVal,
05748                                  ISD::SETNE);
05749       processIntegerCallValue(I, Res, false);
05750       return true;
05751     }
05752   }
05753 
05754 
05755   return false;
05756 }
05757 
05758 /// visitMemChrCall -- See if we can lower a memchr call into an optimized
05759 /// form.  If so, return true and lower it, otherwise return false and it
05760 /// will be lowered like a normal call.
05761 bool SelectionDAGBuilder::visitMemChrCall(const CallInst &I) {
05762   // Verify that the prototype makes sense.  void *memchr(void *, int, size_t)
05763   if (I.getNumArgOperands() != 3)
05764     return false;
05765 
05766   const Value *Src = I.getArgOperand(0);
05767   const Value *Char = I.getArgOperand(1);
05768   const Value *Length = I.getArgOperand(2);
05769   if (!Src->getType()->isPointerTy() ||
05770       !Char->getType()->isIntegerTy() ||
05771       !Length->getType()->isIntegerTy() ||
05772       !I.getType()->isPointerTy())
05773     return false;
05774 
05775   const TargetSelectionDAGInfo &TSI = DAG.getSelectionDAGInfo();
05776   std::pair<SDValue, SDValue> Res =
05777     TSI.EmitTargetCodeForMemchr(DAG, getCurSDLoc(), DAG.getRoot(),
05778                                 getValue(Src), getValue(Char), getValue(Length),
05779                                 MachinePointerInfo(Src));
05780   if (Res.first.getNode()) {
05781     setValue(&I, Res.first);
05782     PendingLoads.push_back(Res.second);
05783     return true;
05784   }
05785 
05786   return false;
05787 }
05788 
05789 /// visitStrCpyCall -- See if we can lower a strcpy or stpcpy call into an
05790 /// optimized form.  If so, return true and lower it, otherwise return false
05791 /// and it will be lowered like a normal call.
05792 bool SelectionDAGBuilder::visitStrCpyCall(const CallInst &I, bool isStpcpy) {
05793   // Verify that the prototype makes sense.  char *strcpy(char *, char *)
05794   if (I.getNumArgOperands() != 2)
05795     return false;
05796 
05797   const Value *Arg0 = I.getArgOperand(0), *Arg1 = I.getArgOperand(1);
05798   if (!Arg0->getType()->isPointerTy() ||
05799       !Arg1->getType()->isPointerTy() ||
05800       !I.getType()->isPointerTy())
05801     return false;
05802 
05803   const TargetSelectionDAGInfo &TSI = DAG.getSelectionDAGInfo();
05804   std::pair<SDValue, SDValue> Res =
05805     TSI.EmitTargetCodeForStrcpy(DAG, getCurSDLoc(), getRoot(),
05806                                 getValue(Arg0), getValue(Arg1),
05807                                 MachinePointerInfo(Arg0),
05808                                 MachinePointerInfo(Arg1), isStpcpy);
05809   if (Res.first.getNode()) {
05810     setValue(&I, Res.first);
05811     DAG.setRoot(Res.second);
05812     return true;
05813   }
05814 
05815   return false;
05816 }
05817 
05818 /// visitStrCmpCall - See if we can lower a call to strcmp in an optimized form.
05819 /// If so, return true and lower it, otherwise return false and it will be
05820 /// lowered like a normal call.
05821 bool SelectionDAGBuilder::visitStrCmpCall(const CallInst &I) {
05822   // Verify that the prototype makes sense.  int strcmp(void*,void*)
05823   if (I.getNumArgOperands() != 2)
05824     return false;
05825 
05826   const Value *Arg0 = I.getArgOperand(0), *Arg1 = I.getArgOperand(1);
05827   if (!Arg0->getType()->isPointerTy() ||
05828       !Arg1->getType()->isPointerTy() ||
05829       !I.getType()->isIntegerTy())
05830     return false;
05831 
05832   const TargetSelectionDAGInfo &TSI = DAG.getSelectionDAGInfo();
05833   std::pair<SDValue, SDValue> Res =
05834     TSI.EmitTargetCodeForStrcmp(DAG, getCurSDLoc(), DAG.getRoot(),
05835                                 getValue(Arg0), getValue(Arg1),
05836                                 MachinePointerInfo(Arg0),
05837                                 MachinePointerInfo(Arg1));
05838   if (Res.first.getNode()) {
05839     processIntegerCallValue(I, Res.first, true);
05840     PendingLoads.push_back(Res.second);
05841     return true;
05842   }
05843 
05844   return false;
05845 }
05846 
05847 /// visitStrLenCall -- See if we can lower a strlen call into an optimized
05848 /// form.  If so, return true and lower it, otherwise return false and it
05849 /// will be lowered like a normal call.
05850 bool SelectionDAGBuilder::visitStrLenCall(const CallInst &I) {
05851   // Verify that the prototype makes sense.  size_t strlen(char *)
05852   if (I.getNumArgOperands() != 1)
05853     return false;
05854 
05855   const Value *Arg0 = I.getArgOperand(0);
05856   if (!Arg0->getType()->isPointerTy() || !I.getType()->isIntegerTy())
05857     return false;
05858 
05859   const TargetSelectionDAGInfo &TSI = DAG.getSelectionDAGInfo();
05860   std::pair<SDValue, SDValue> Res =
05861     TSI.EmitTargetCodeForStrlen(DAG, getCurSDLoc(), DAG.getRoot(),
05862                                 getValue(Arg0), MachinePointerInfo(Arg0));
05863   if (Res.first.getNode()) {
05864     processIntegerCallValue(I, Res.first, false);
05865     PendingLoads.push_back(Res.second);
05866     return true;
05867   }
05868 
05869   return false;
05870 }
05871 
05872 /// visitStrNLenCall -- See if we can lower a strnlen call into an optimized
05873 /// form.  If so, return true and lower it, otherwise return false and it
05874 /// will be lowered like a normal call.
05875 bool SelectionDAGBuilder::visitStrNLenCall(const CallInst &I) {
05876   // Verify that the prototype makes sense.  size_t strnlen(char *, size_t)
05877   if (I.getNumArgOperands() != 2)
05878     return false;
05879 
05880   const Value *Arg0 = I.getArgOperand(0), *Arg1 = I.getArgOperand(1);
05881   if (!Arg0->getType()->isPointerTy() ||
05882       !Arg1->getType()->isIntegerTy() ||
05883       !I.getType()->isIntegerTy())
05884     return false;
05885 
05886   const TargetSelectionDAGInfo &TSI = DAG.getSelectionDAGInfo();
05887   std::pair<SDValue, SDValue> Res =
05888     TSI.EmitTargetCodeForStrnlen(DAG, getCurSDLoc(), DAG.getRoot(),
05889                                  getValue(Arg0), getValue(Arg1),
05890                                  MachinePointerInfo(Arg0));
05891   if (Res.first.getNode()) {
05892     processIntegerCallValue(I, Res.first, false);
05893     PendingLoads.push_back(Res.second);
05894     return true;
05895   }
05896 
05897   return false;
05898 }
05899 
05900 /// visitUnaryFloatCall - If a call instruction is a unary floating-point
05901 /// operation (as expected), translate it to an SDNode with the specified opcode
05902 /// and return true.
05903 bool SelectionDAGBuilder::visitUnaryFloatCall(const CallInst &I,
05904                                               unsigned Opcode) {
05905   // Sanity check that it really is a unary floating-point call.
05906   if (I.getNumArgOperands() != 1 ||
05907       !I.getArgOperand(0)->getType()->isFloatingPointTy() ||
05908       I.getType() != I.getArgOperand(0)->getType() ||
05909       !I.onlyReadsMemory())
05910     return false;
05911 
05912   SDValue Tmp = getValue(I.getArgOperand(0));
05913   setValue(&I, DAG.getNode(Opcode, getCurSDLoc(), Tmp.getValueType(), Tmp));
05914   return true;
05915 }
05916 
05917 void SelectionDAGBuilder::visitCall(const CallInst &I) {
05918   // Handle inline assembly differently.
05919   if (isa<InlineAsm>(I.getCalledValue())) {
05920     visitInlineAsm(&I);
05921     return;
05922   }
05923 
05924   MachineModuleInfo &MMI = DAG.getMachineFunction().getMMI();
05925   ComputeUsesVAFloatArgument(I, &MMI);
05926 
05927   const char *RenameFn = nullptr;
05928   if (Function *F = I.getCalledFunction()) {
05929     if (F->isDeclaration()) {
05930       if (const TargetIntrinsicInfo *II = TM.getIntrinsicInfo()) {
05931         if (unsigned IID = II->getIntrinsicID(F)) {
05932           RenameFn = visitIntrinsicCall(I, IID);
05933           if (!RenameFn)
05934             return;
05935         }
05936       }
05937       if (unsigned IID = F->getIntrinsicID()) {
05938         RenameFn = visitIntrinsicCall(I, IID);
05939         if (!RenameFn)
05940           return;
05941       }
05942     }
05943 
05944     // Check for well-known libc/libm calls.  If the function is internal, it
05945     // can't be a library call.
05946     LibFunc::Func Func;
05947     if (!F->hasLocalLinkage() && F->hasName() &&
05948         LibInfo->getLibFunc(F->getName(), Func) &&
05949         LibInfo->hasOptimizedCodeGen(Func)) {
05950       switch (Func) {
05951       default: break;
05952       case LibFunc::copysign:
05953       case LibFunc::copysignf:
05954       case LibFunc::copysignl:
05955         if (I.getNumArgOperands() == 2 &&   // Basic sanity checks.
05956             I.getArgOperand(0)->getType()->isFloatingPointTy() &&
05957             I.getType() == I.getArgOperand(0)->getType() &&
05958             I.getType() == I.getArgOperand(1)->getType() &&
05959             I.onlyReadsMemory()) {
05960           SDValue LHS = getValue(I.getArgOperand(0));
05961           SDValue RHS = getValue(I.getArgOperand(1));
05962           setValue(&I, DAG.getNode(ISD::FCOPYSIGN, getCurSDLoc(),
05963                                    LHS.getValueType(), LHS, RHS));
05964           return;
05965         }
05966         break;
05967       case LibFunc::fabs:
05968       case LibFunc::fabsf:
05969       case LibFunc::fabsl:
05970         if (visitUnaryFloatCall(I, ISD::FABS))
05971           return;
05972         break;
05973       case LibFunc::sin:
05974       case LibFunc::sinf:
05975       case LibFunc::sinl:
05976         if (visitUnaryFloatCall(I, ISD::FSIN))
05977           return;
05978         break;
05979       case LibFunc::cos:
05980       case LibFunc::cosf:
05981       case LibFunc::cosl:
05982         if (visitUnaryFloatCall(I, ISD::FCOS))
05983           return;
05984         break;
05985       case LibFunc::sqrt:
05986       case LibFunc::sqrtf:
05987       case LibFunc::sqrtl:
05988       case LibFunc::sqrt_finite:
05989       case LibFunc::sqrtf_finite:
05990       case LibFunc::sqrtl_finite:
05991         if (visitUnaryFloatCall(I, ISD::FSQRT))
05992           return;
05993         break;
05994       case LibFunc::floor:
05995       case LibFunc::floorf:
05996       case LibFunc::floorl:
05997         if (visitUnaryFloatCall(I, ISD::FFLOOR))
05998           return;
05999         break;
06000       case LibFunc::nearbyint:
06001       case LibFunc::nearbyintf:
06002       case LibFunc::nearbyintl:
06003         if (visitUnaryFloatCall(I, ISD::FNEARBYINT))
06004           return;
06005         break;
06006       case LibFunc::ceil:
06007       case LibFunc::ceilf:
06008       case LibFunc::ceill:
06009         if (visitUnaryFloatCall(I, ISD::FCEIL))
06010           return;
06011         break;
06012       case LibFunc::rint:
06013       case LibFunc::rintf:
06014       case LibFunc::rintl:
06015         if (visitUnaryFloatCall(I, ISD::FRINT))
06016           return;
06017         break;
06018       case LibFunc::round:
06019       case LibFunc::roundf:
06020       case LibFunc::roundl:
06021         if (visitUnaryFloatCall(I, ISD::FROUND))
06022           return;
06023         break;
06024       case LibFunc::trunc:
06025       case LibFunc::truncf:
06026       case LibFunc::truncl:
06027         if (visitUnaryFloatCall(I, ISD::FTRUNC))
06028           return;
06029         break;
06030       case LibFunc::log2:
06031       case LibFunc::log2f:
06032       case LibFunc::log2l:
06033         if (visitUnaryFloatCall(I, ISD::FLOG2))
06034           return;
06035         break;
06036       case LibFunc::exp2:
06037       case LibFunc::exp2f:
06038       case LibFunc::exp2l:
06039         if (visitUnaryFloatCall(I, ISD::FEXP2))
06040           return;
06041         break;
06042       case LibFunc::memcmp:
06043         if (visitMemCmpCall(I))
06044           return;
06045         break;
06046       case LibFunc::memchr:
06047         if (visitMemChrCall(I))
06048           return;
06049         break;
06050       case LibFunc::strcpy:
06051         if (visitStrCpyCall(I, false))
06052           return;
06053         break;
06054       case LibFunc::stpcpy:
06055         if (visitStrCpyCall(I, true))
06056           return;
06057         break;
06058       case LibFunc::strcmp:
06059         if (visitStrCmpCall(I))
06060           return;
06061         break;
06062       case LibFunc::strlen:
06063         if (visitStrLenCall(I))
06064           return;
06065         break;
06066       case LibFunc::strnlen:
06067         if (visitStrNLenCall(I))
06068           return;
06069         break;
06070       }
06071     }
06072   }
06073 
06074   SDValue Callee;
06075   if (!RenameFn)
06076     Callee = getValue(I.getCalledValue());
06077   else
06078     Callee = DAG.getExternalSymbol(RenameFn,
06079                                    TM.getTargetLowering()->getPointerTy());
06080 
06081   // Check if we can potentially perform a tail call. More detailed checking is
06082   // be done within LowerCallTo, after more information about the call is known.
06083   LowerCallTo(&I, Callee, I.isTailCall());
06084 }
06085 
06086 namespace {
06087 
06088 /// AsmOperandInfo - This contains information for each constraint that we are
06089 /// lowering.
06090 class SDISelAsmOperandInfo : public TargetLowering::AsmOperandInfo {
06091 public:
06092   /// CallOperand - If this is the result output operand or a clobber
06093   /// this is null, otherwise it is the incoming operand to the CallInst.
06094   /// This gets modified as the asm is processed.
06095   SDValue CallOperand;
06096 
06097   /// AssignedRegs - If this is a register or register class operand, this
06098   /// contains the set of register corresponding to the operand.
06099   RegsForValue AssignedRegs;
06100 
06101   explicit SDISelAsmOperandInfo(const TargetLowering::AsmOperandInfo &info)
06102     : TargetLowering::AsmOperandInfo(info), CallOperand(nullptr,0) {
06103   }
06104 
06105   /// getCallOperandValEVT - Return the EVT of the Value* that this operand
06106   /// corresponds to.  If there is no Value* for this operand, it returns
06107   /// MVT::Other.
06108   EVT getCallOperandValEVT(LLVMContext &Context,
06109                            const TargetLowering &TLI,
06110                            const DataLayout *DL) const {
06111     if (!CallOperandVal) return MVT::Other;
06112 
06113     if (isa<BasicBlock>(CallOperandVal))
06114       return TLI.getPointerTy();
06115 
06116     llvm::Type *OpTy = CallOperandVal->getType();
06117 
06118     // FIXME: code duplicated from TargetLowering::ParseConstraints().
06119     // If this is an indirect operand, the operand is a pointer to the
06120     // accessed type.
06121     if (isIndirect) {
06122       llvm::PointerType *PtrTy = dyn_cast<PointerType>(OpTy);
06123       if (!PtrTy)
06124         report_fatal_error("Indirect operand for inline asm not a pointer!");
06125       OpTy = PtrTy->getElementType();
06126     }
06127 
06128     // Look for vector wrapped in a struct. e.g. { <16 x i8> }.
06129     if (StructType *STy = dyn_cast<StructType>(OpTy))
06130       if (STy->getNumElements() == 1)
06131         OpTy = STy->getElementType(0);
06132 
06133     // If OpTy is not a single value, it may be a struct/union that we
06134     // can tile with integers.
06135     if (!OpTy->isSingleValueType() && OpTy->isSized()) {
06136       unsigned BitSize = DL->getTypeSizeInBits(OpTy);
06137       switch (BitSize) {
06138       default: break;
06139       case 1:
06140       case 8:
06141       case 16:
06142       case 32:
06143       case 64:
06144       case 128:
06145         OpTy = IntegerType::get(Context, BitSize);
06146         break;
06147       }
06148     }
06149 
06150     return TLI.getValueType(OpTy, true);
06151   }
06152 };
06153 
06154 typedef SmallVector<SDISelAsmOperandInfo,16> SDISelAsmOperandInfoVector;
06155 
06156 } // end anonymous namespace
06157 
06158 /// GetRegistersForValue - Assign registers (virtual or physical) for the
06159 /// specified operand.  We prefer to assign virtual registers, to allow the
06160 /// register allocator to handle the assignment process.  However, if the asm
06161 /// uses features that we can't model on machineinstrs, we have SDISel do the
06162 /// allocation.  This produces generally horrible, but correct, code.
06163 ///
06164 ///   OpInfo describes the operand.
06165 ///
06166 static void GetRegistersForValue(SelectionDAG &DAG,
06167                                  const TargetLowering &TLI,
06168                                  SDLoc DL,
06169                                  SDISelAsmOperandInfo &OpInfo) {
06170   LLVMContext &Context = *DAG.getContext();
06171 
06172   MachineFunction &MF = DAG.getMachineFunction();
06173   SmallVector<unsigned, 4> Regs;
06174 
06175   // If this is a constraint for a single physreg, or a constraint for a
06176   // register class, find it.
06177   std::pair<unsigned, const TargetRegisterClass*> PhysReg =
06178     TLI.getRegForInlineAsmConstraint(OpInfo.ConstraintCode,
06179                                      OpInfo.ConstraintVT);
06180 
06181   unsigned NumRegs = 1;
06182   if (OpInfo.ConstraintVT != MVT::Other) {
06183     // If this is a FP input in an integer register (or visa versa) insert a bit
06184     // cast of the input value.  More generally, handle any case where the input
06185     // value disagrees with the register class we plan to stick this in.
06186     if (OpInfo.Type == InlineAsm::isInput &&
06187         PhysReg.second && !PhysReg.second->hasType(OpInfo.ConstraintVT)) {
06188       // Try to convert to the first EVT that the reg class contains.  If the
06189       // types are identical size, use a bitcast to convert (e.g. two differing
06190       // vector types).
06191       MVT RegVT = *PhysReg.second->vt_begin();
06192       if (RegVT.getSizeInBits() == OpInfo.CallOperand.getValueSizeInBits()) {
06193         OpInfo.CallOperand = DAG.getNode(ISD::BITCAST, DL,
06194                                          RegVT, OpInfo.CallOperand);
06195         OpInfo.ConstraintVT = RegVT;
06196       } else if (RegVT.isInteger() && OpInfo.ConstraintVT.isFloatingPoint()) {
06197         // If the input is a FP value and we want it in FP registers, do a
06198         // bitcast to the corresponding integer type.  This turns an f64 value
06199         // into i64, which can be passed with two i32 values on a 32-bit
06200         // machine.
06201         RegVT = MVT::getIntegerVT(OpInfo.ConstraintVT.getSizeInBits());
06202         OpInfo.CallOperand = DAG.getNode(ISD::BITCAST, DL,
06203                                          RegVT, OpInfo.CallOperand);
06204         OpInfo.ConstraintVT = RegVT;
06205       }
06206     }
06207 
06208     NumRegs = TLI.getNumRegisters(Context, OpInfo.ConstraintVT);
06209   }
06210 
06211   MVT RegVT;
06212   EVT ValueVT = OpInfo.ConstraintVT;
06213 
06214   // If this is a constraint for a specific physical register, like {r17},
06215   // assign it now.
06216   if (unsigned AssignedReg = PhysReg.first) {
06217     const TargetRegisterClass *RC = PhysReg.second;
06218     if (OpInfo.ConstraintVT == MVT::Other)
06219       ValueVT = *RC->vt_begin();
06220 
06221     // Get the actual register value type.  This is important, because the user
06222     // may have asked for (e.g.) the AX register in i32 type.  We need to
06223     // remember that AX is actually i16 to get the right extension.
06224     RegVT = *RC->vt_begin();
06225 
06226     // This is a explicit reference to a physical register.
06227     Regs.push_back(AssignedReg);
06228 
06229     // If this is an expanded reference, add the rest of the regs to Regs.
06230     if (NumRegs != 1) {
06231       TargetRegisterClass::iterator I = RC->begin();
06232       for (; *I != AssignedReg; ++I)
06233         assert(I != RC->end() && "Didn't find reg!");
06234 
06235       // Already added the first reg.
06236       --NumRegs; ++I;
06237