LLVM API Documentation

SelectionDAGBuilder.cpp
Go to the documentation of this file.
00001 //===-- SelectionDAGBuilder.cpp - Selection-DAG building ------------------===//
00002 //
00003 //                     The LLVM Compiler Infrastructure
00004 //
00005 // This file is distributed under the University of Illinois Open Source
00006 // License. See LICENSE.TXT for details.
00007 //
00008 //===----------------------------------------------------------------------===//
00009 //
00010 // This implements routines for translating from LLVM IR into SelectionDAG IR.
00011 //
00012 //===----------------------------------------------------------------------===//
00013 
00014 #include "SelectionDAGBuilder.h"
00015 #include "SDNodeDbgValue.h"
00016 #include "llvm/ADT/BitVector.h"
00017 #include "llvm/ADT/Optional.h"
00018 #include "llvm/ADT/SmallSet.h"
00019 #include "llvm/Analysis/AliasAnalysis.h"
00020 #include "llvm/Analysis/BranchProbabilityInfo.h"
00021 #include "llvm/Analysis/ConstantFolding.h"
00022 #include "llvm/Analysis/ValueTracking.h"
00023 #include "llvm/CodeGen/Analysis.h"
00024 #include "llvm/CodeGen/FastISel.h"
00025 #include "llvm/CodeGen/FunctionLoweringInfo.h"
00026 #include "llvm/CodeGen/GCMetadata.h"
00027 #include "llvm/CodeGen/GCStrategy.h"
00028 #include "llvm/CodeGen/MachineFrameInfo.h"
00029 #include "llvm/CodeGen/MachineFunction.h"
00030 #include "llvm/CodeGen/MachineInstrBuilder.h"
00031 #include "llvm/CodeGen/MachineJumpTableInfo.h"
00032 #include "llvm/CodeGen/MachineModuleInfo.h"
00033 #include "llvm/CodeGen/MachineRegisterInfo.h"
00034 #include "llvm/CodeGen/SelectionDAG.h"
00035 #include "llvm/CodeGen/StackMaps.h"
00036 #include "llvm/IR/CallingConv.h"
00037 #include "llvm/IR/Constants.h"
00038 #include "llvm/IR/DataLayout.h"
00039 #include "llvm/IR/DebugInfo.h"
00040 #include "llvm/IR/DerivedTypes.h"
00041 #include "llvm/IR/Function.h"
00042 #include "llvm/IR/GlobalVariable.h"
00043 #include "llvm/IR/InlineAsm.h"
00044 #include "llvm/IR/Instructions.h"
00045 #include "llvm/IR/IntrinsicInst.h"
00046 #include "llvm/IR/Intrinsics.h"
00047 #include "llvm/IR/LLVMContext.h"
00048 #include "llvm/IR/Module.h"
00049 #include "llvm/Support/CommandLine.h"
00050 #include "llvm/Support/Debug.h"
00051 #include "llvm/Support/ErrorHandling.h"
00052 #include "llvm/Support/MathExtras.h"
00053 #include "llvm/Support/raw_ostream.h"
00054 #include "llvm/Target/TargetFrameLowering.h"
00055 #include "llvm/Target/TargetInstrInfo.h"
00056 #include "llvm/Target/TargetIntrinsicInfo.h"
00057 #include "llvm/Target/TargetLibraryInfo.h"
00058 #include "llvm/Target/TargetLowering.h"
00059 #include "llvm/Target/TargetOptions.h"
00060 #include "llvm/Target/TargetSelectionDAGInfo.h"
00061 #include "llvm/Target/TargetSubtargetInfo.h"
00062 #include <algorithm>
00063 using namespace llvm;
00064 
00065 #define DEBUG_TYPE "isel"
00066 
00067 /// LimitFloatPrecision - Generate low-precision inline sequences for
00068 /// some float libcalls (6, 8 or 12 bits).
00069 static unsigned LimitFloatPrecision;
00070 
00071 static cl::opt<unsigned, true>
00072 LimitFPPrecision("limit-float-precision",
00073                  cl::desc("Generate low-precision inline sequences "
00074                           "for some float libcalls"),
00075                  cl::location(LimitFloatPrecision),
00076                  cl::init(0));
00077 
00078 // Limit the width of DAG chains. This is important in general to prevent
00079 // prevent DAG-based analysis from blowing up. For example, alias analysis and
00080 // load clustering may not complete in reasonable time. It is difficult to
00081 // recognize and avoid this situation within each individual analysis, and
00082 // future analyses are likely to have the same behavior. Limiting DAG width is
00083 // the safe approach, and will be especially important with global DAGs.
00084 //
00085 // MaxParallelChains default is arbitrarily high to avoid affecting
00086 // optimization, but could be lowered to improve compile time. Any ld-ld-st-st
00087 // sequence over this should have been converted to llvm.memcpy by the
00088 // frontend. It easy to induce this behavior with .ll code such as:
00089 // %buffer = alloca [4096 x i8]
00090 // %data = load [4096 x i8]* %argPtr
00091 // store [4096 x i8] %data, [4096 x i8]* %buffer
00092 static const unsigned MaxParallelChains = 64;
00093 
00094 static SDValue getCopyFromPartsVector(SelectionDAG &DAG, SDLoc DL,
00095                                       const SDValue *Parts, unsigned NumParts,
00096                                       MVT PartVT, EVT ValueVT, const Value *V);
00097 
00098 /// getCopyFromParts - Create a value that contains the specified legal parts
00099 /// combined into the value they represent.  If the parts combine to a type
00100 /// larger then ValueVT then AssertOp can be used to specify whether the extra
00101 /// bits are known to be zero (ISD::AssertZext) or sign extended from ValueVT
00102 /// (ISD::AssertSext).
00103 static SDValue getCopyFromParts(SelectionDAG &DAG, SDLoc DL,
00104                                 const SDValue *Parts,
00105                                 unsigned NumParts, MVT PartVT, EVT ValueVT,
00106                                 const Value *V,
00107                                 ISD::NodeType AssertOp = ISD::DELETED_NODE) {
00108   if (ValueVT.isVector())
00109     return getCopyFromPartsVector(DAG, DL, Parts, NumParts,
00110                                   PartVT, ValueVT, V);
00111 
00112   assert(NumParts > 0 && "No parts to assemble!");
00113   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
00114   SDValue Val = Parts[0];
00115 
00116   if (NumParts > 1) {
00117     // Assemble the value from multiple parts.
00118     if (ValueVT.isInteger()) {
00119       unsigned PartBits = PartVT.getSizeInBits();
00120       unsigned ValueBits = ValueVT.getSizeInBits();
00121 
00122       // Assemble the power of 2 part.
00123       unsigned RoundParts = NumParts & (NumParts - 1) ?
00124         1 << Log2_32(NumParts) : NumParts;
00125       unsigned RoundBits = PartBits * RoundParts;
00126       EVT RoundVT = RoundBits == ValueBits ?
00127         ValueVT : EVT::getIntegerVT(*DAG.getContext(), RoundBits);
00128       SDValue Lo, Hi;
00129 
00130       EVT HalfVT = EVT::getIntegerVT(*DAG.getContext(), RoundBits/2);
00131 
00132       if (RoundParts > 2) {
00133         Lo = getCopyFromParts(DAG, DL, Parts, RoundParts / 2,
00134                               PartVT, HalfVT, V);
00135         Hi = getCopyFromParts(DAG, DL, Parts + RoundParts / 2,
00136                               RoundParts / 2, PartVT, HalfVT, V);
00137       } else {
00138         Lo = DAG.getNode(ISD::BITCAST, DL, HalfVT, Parts[0]);
00139         Hi = DAG.getNode(ISD::BITCAST, DL, HalfVT, Parts[1]);
00140       }
00141 
00142       if (TLI.isBigEndian())
00143         std::swap(Lo, Hi);
00144 
00145       Val = DAG.getNode(ISD::BUILD_PAIR, DL, RoundVT, Lo, Hi);
00146 
00147       if (RoundParts < NumParts) {
00148         // Assemble the trailing non-power-of-2 part.
00149         unsigned OddParts = NumParts - RoundParts;
00150         EVT OddVT = EVT::getIntegerVT(*DAG.getContext(), OddParts * PartBits);
00151         Hi = getCopyFromParts(DAG, DL,
00152                               Parts + RoundParts, OddParts, PartVT, OddVT, V);
00153 
00154         // Combine the round and odd parts.
00155         Lo = Val;
00156         if (TLI.isBigEndian())
00157           std::swap(Lo, Hi);
00158         EVT TotalVT = EVT::getIntegerVT(*DAG.getContext(), NumParts * PartBits);
00159         Hi = DAG.getNode(ISD::ANY_EXTEND, DL, TotalVT, Hi);
00160         Hi = DAG.getNode(ISD::SHL, DL, TotalVT, Hi,
00161                          DAG.getConstant(Lo.getValueType().getSizeInBits(),
00162                                          TLI.getPointerTy()));
00163         Lo = DAG.getNode(ISD::ZERO_EXTEND, DL, TotalVT, Lo);
00164         Val = DAG.getNode(ISD::OR, DL, TotalVT, Lo, Hi);
00165       }
00166     } else if (PartVT.isFloatingPoint()) {
00167       // FP split into multiple FP parts (for ppcf128)
00168       assert(ValueVT == EVT(MVT::ppcf128) && PartVT == MVT::f64 &&
00169              "Unexpected split");
00170       SDValue Lo, Hi;
00171       Lo = DAG.getNode(ISD::BITCAST, DL, EVT(MVT::f64), Parts[0]);
00172       Hi = DAG.getNode(ISD::BITCAST, DL, EVT(MVT::f64), Parts[1]);
00173       if (TLI.hasBigEndianPartOrdering(ValueVT))
00174         std::swap(Lo, Hi);
00175       Val = DAG.getNode(ISD::BUILD_PAIR, DL, ValueVT, Lo, Hi);
00176     } else {
00177       // FP split into integer parts (soft fp)
00178       assert(ValueVT.isFloatingPoint() && PartVT.isInteger() &&
00179              !PartVT.isVector() && "Unexpected split");
00180       EVT IntVT = EVT::getIntegerVT(*DAG.getContext(), ValueVT.getSizeInBits());
00181       Val = getCopyFromParts(DAG, DL, Parts, NumParts, PartVT, IntVT, V);
00182     }
00183   }
00184 
00185   // There is now one part, held in Val.  Correct it to match ValueVT.
00186   EVT PartEVT = Val.getValueType();
00187 
00188   if (PartEVT == ValueVT)
00189     return Val;
00190 
00191   if (PartEVT.isInteger() && ValueVT.isInteger()) {
00192     if (ValueVT.bitsLT(PartEVT)) {
00193       // For a truncate, see if we have any information to
00194       // indicate whether the truncated bits will always be
00195       // zero or sign-extension.
00196       if (AssertOp != ISD::DELETED_NODE)
00197         Val = DAG.getNode(AssertOp, DL, PartEVT, Val,
00198                           DAG.getValueType(ValueVT));
00199       return DAG.getNode(ISD::TRUNCATE, DL, ValueVT, Val);
00200     }
00201     return DAG.getNode(ISD::ANY_EXTEND, DL, ValueVT, Val);
00202   }
00203 
00204   if (PartEVT.isFloatingPoint() && ValueVT.isFloatingPoint()) {
00205     // FP_ROUND's are always exact here.
00206     if (ValueVT.bitsLT(Val.getValueType()))
00207       return DAG.getNode(ISD::FP_ROUND, DL, ValueVT, Val,
00208                          DAG.getTargetConstant(1, TLI.getPointerTy()));
00209 
00210     return DAG.getNode(ISD::FP_EXTEND, DL, ValueVT, Val);
00211   }
00212 
00213   if (PartEVT.getSizeInBits() == ValueVT.getSizeInBits())
00214     return DAG.getNode(ISD::BITCAST, DL, ValueVT, Val);
00215 
00216   llvm_unreachable("Unknown mismatch!");
00217 }
00218 
00219 static void diagnosePossiblyInvalidConstraint(LLVMContext &Ctx, const Value *V,
00220                                               const Twine &ErrMsg) {
00221   const Instruction *I = dyn_cast_or_null<Instruction>(V);
00222   if (!V)
00223     return Ctx.emitError(ErrMsg);
00224 
00225   const char *AsmError = ", possible invalid constraint for vector type";
00226   if (const CallInst *CI = dyn_cast<CallInst>(I))
00227     if (isa<InlineAsm>(CI->getCalledValue()))
00228       return Ctx.emitError(I, ErrMsg + AsmError);
00229 
00230   return Ctx.emitError(I, ErrMsg);
00231 }
00232 
00233 /// getCopyFromPartsVector - Create a value that contains the specified legal
00234 /// parts combined into the value they represent.  If the parts combine to a
00235 /// type larger then ValueVT then AssertOp can be used to specify whether the
00236 /// extra bits are known to be zero (ISD::AssertZext) or sign extended from
00237 /// ValueVT (ISD::AssertSext).
00238 static SDValue getCopyFromPartsVector(SelectionDAG &DAG, SDLoc DL,
00239                                       const SDValue *Parts, unsigned NumParts,
00240                                       MVT PartVT, EVT ValueVT, const Value *V) {
00241   assert(ValueVT.isVector() && "Not a vector value");
00242   assert(NumParts > 0 && "No parts to assemble!");
00243   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
00244   SDValue Val = Parts[0];
00245 
00246   // Handle a multi-element vector.
00247   if (NumParts > 1) {
00248     EVT IntermediateVT;
00249     MVT RegisterVT;
00250     unsigned NumIntermediates;
00251     unsigned NumRegs =
00252     TLI.getVectorTypeBreakdown(*DAG.getContext(), ValueVT, IntermediateVT,
00253                                NumIntermediates, RegisterVT);
00254     assert(NumRegs == NumParts && "Part count doesn't match vector breakdown!");
00255     NumParts = NumRegs; // Silence a compiler warning.
00256     assert(RegisterVT == PartVT && "Part type doesn't match vector breakdown!");
00257     assert(RegisterVT == Parts[0].getSimpleValueType() &&
00258            "Part type doesn't match part!");
00259 
00260     // Assemble the parts into intermediate operands.
00261     SmallVector<SDValue, 8> Ops(NumIntermediates);
00262     if (NumIntermediates == NumParts) {
00263       // If the register was not expanded, truncate or copy the value,
00264       // as appropriate.
00265       for (unsigned i = 0; i != NumParts; ++i)
00266         Ops[i] = getCopyFromParts(DAG, DL, &Parts[i], 1,
00267                                   PartVT, IntermediateVT, V);
00268     } else if (NumParts > 0) {
00269       // If the intermediate type was expanded, build the intermediate
00270       // operands from the parts.
00271       assert(NumParts % NumIntermediates == 0 &&
00272              "Must expand into a divisible number of parts!");
00273       unsigned Factor = NumParts / NumIntermediates;
00274       for (unsigned i = 0; i != NumIntermediates; ++i)
00275         Ops[i] = getCopyFromParts(DAG, DL, &Parts[i * Factor], Factor,
00276                                   PartVT, IntermediateVT, V);
00277     }
00278 
00279     // Build a vector with BUILD_VECTOR or CONCAT_VECTORS from the
00280     // intermediate operands.
00281     Val = DAG.getNode(IntermediateVT.isVector() ? ISD::CONCAT_VECTORS
00282                                                 : ISD::BUILD_VECTOR,
00283                       DL, ValueVT, Ops);
00284   }
00285 
00286   // There is now one part, held in Val.  Correct it to match ValueVT.
00287   EVT PartEVT = Val.getValueType();
00288 
00289   if (PartEVT == ValueVT)
00290     return Val;
00291 
00292   if (PartEVT.isVector()) {
00293     // If the element type of the source/dest vectors are the same, but the
00294     // parts vector has more elements than the value vector, then we have a
00295     // vector widening case (e.g. <2 x float> -> <4 x float>).  Extract the
00296     // elements we want.
00297     if (PartEVT.getVectorElementType() == ValueVT.getVectorElementType()) {
00298       assert(PartEVT.getVectorNumElements() > ValueVT.getVectorNumElements() &&
00299              "Cannot narrow, it would be a lossy transformation");
00300       return DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, ValueVT, Val,
00301                          DAG.getConstant(0, TLI.getVectorIdxTy()));
00302     }
00303 
00304     // Vector/Vector bitcast.
00305     if (ValueVT.getSizeInBits() == PartEVT.getSizeInBits())
00306       return DAG.getNode(ISD::BITCAST, DL, ValueVT, Val);
00307 
00308     assert(PartEVT.getVectorNumElements() == ValueVT.getVectorNumElements() &&
00309       "Cannot handle this kind of promotion");
00310     // Promoted vector extract
00311     bool Smaller = ValueVT.bitsLE(PartEVT);
00312     return DAG.getNode((Smaller ? ISD::TRUNCATE : ISD::ANY_EXTEND),
00313                        DL, ValueVT, Val);
00314 
00315   }
00316 
00317   // Trivial bitcast if the types are the same size and the destination
00318   // vector type is legal.
00319   if (PartEVT.getSizeInBits() == ValueVT.getSizeInBits() &&
00320       TLI.isTypeLegal(ValueVT))
00321     return DAG.getNode(ISD::BITCAST, DL, ValueVT, Val);
00322 
00323   // Handle cases such as i8 -> <1 x i1>
00324   if (ValueVT.getVectorNumElements() != 1) {
00325     diagnosePossiblyInvalidConstraint(*DAG.getContext(), V,
00326                                       "non-trivial scalar-to-vector conversion");
00327     return DAG.getUNDEF(ValueVT);
00328   }
00329 
00330   if (ValueVT.getVectorNumElements() == 1 &&
00331       ValueVT.getVectorElementType() != PartEVT) {
00332     bool Smaller = ValueVT.bitsLE(PartEVT);
00333     Val = DAG.getNode((Smaller ? ISD::TRUNCATE : ISD::ANY_EXTEND),
00334                        DL, ValueVT.getScalarType(), Val);
00335   }
00336 
00337   return DAG.getNode(ISD::BUILD_VECTOR, DL, ValueVT, Val);
00338 }
00339 
00340 static void getCopyToPartsVector(SelectionDAG &DAG, SDLoc dl,
00341                                  SDValue Val, SDValue *Parts, unsigned NumParts,
00342                                  MVT PartVT, const Value *V);
00343 
00344 /// getCopyToParts - Create a series of nodes that contain the specified value
00345 /// split into legal parts.  If the parts contain more bits than Val, then, for
00346 /// integers, ExtendKind can be used to specify how to generate the extra bits.
00347 static void getCopyToParts(SelectionDAG &DAG, SDLoc DL,
00348                            SDValue Val, SDValue *Parts, unsigned NumParts,
00349                            MVT PartVT, const Value *V,
00350                            ISD::NodeType ExtendKind = ISD::ANY_EXTEND) {
00351   EVT ValueVT = Val.getValueType();
00352 
00353   // Handle the vector case separately.
00354   if (ValueVT.isVector())
00355     return getCopyToPartsVector(DAG, DL, Val, Parts, NumParts, PartVT, V);
00356 
00357   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
00358   unsigned PartBits = PartVT.getSizeInBits();
00359   unsigned OrigNumParts = NumParts;
00360   assert(TLI.isTypeLegal(PartVT) && "Copying to an illegal type!");
00361 
00362   if (NumParts == 0)
00363     return;
00364 
00365   assert(!ValueVT.isVector() && "Vector case handled elsewhere");
00366   EVT PartEVT = PartVT;
00367   if (PartEVT == ValueVT) {
00368     assert(NumParts == 1 && "No-op copy with multiple parts!");
00369     Parts[0] = Val;
00370     return;
00371   }
00372 
00373   if (NumParts * PartBits > ValueVT.getSizeInBits()) {
00374     // If the parts cover more bits than the value has, promote the value.
00375     if (PartVT.isFloatingPoint() && ValueVT.isFloatingPoint()) {
00376       assert(NumParts == 1 && "Do not know what to promote to!");
00377       Val = DAG.getNode(ISD::FP_EXTEND, DL, PartVT, Val);
00378     } else {
00379       assert((PartVT.isInteger() || PartVT == MVT::x86mmx) &&
00380              ValueVT.isInteger() &&
00381              "Unknown mismatch!");
00382       ValueVT = EVT::getIntegerVT(*DAG.getContext(), NumParts * PartBits);
00383       Val = DAG.getNode(ExtendKind, DL, ValueVT, Val);
00384       if (PartVT == MVT::x86mmx)
00385         Val = DAG.getNode(ISD::BITCAST, DL, PartVT, Val);
00386     }
00387   } else if (PartBits == ValueVT.getSizeInBits()) {
00388     // Different types of the same size.
00389     assert(NumParts == 1 && PartEVT != ValueVT);
00390     Val = DAG.getNode(ISD::BITCAST, DL, PartVT, Val);
00391   } else if (NumParts * PartBits < ValueVT.getSizeInBits()) {
00392     // If the parts cover less bits than value has, truncate the value.
00393     assert((PartVT.isInteger() || PartVT == MVT::x86mmx) &&
00394            ValueVT.isInteger() &&
00395            "Unknown mismatch!");
00396     ValueVT = EVT::getIntegerVT(*DAG.getContext(), NumParts * PartBits);
00397     Val = DAG.getNode(ISD::TRUNCATE, DL, ValueVT, Val);
00398     if (PartVT == MVT::x86mmx)
00399       Val = DAG.getNode(ISD::BITCAST, DL, PartVT, Val);
00400   }
00401 
00402   // The value may have changed - recompute ValueVT.
00403   ValueVT = Val.getValueType();
00404   assert(NumParts * PartBits == ValueVT.getSizeInBits() &&
00405          "Failed to tile the value with PartVT!");
00406 
00407   if (NumParts == 1) {
00408     if (PartEVT != ValueVT)
00409       diagnosePossiblyInvalidConstraint(*DAG.getContext(), V,
00410                                         "scalar-to-vector conversion failed");
00411 
00412     Parts[0] = Val;
00413     return;
00414   }
00415 
00416   // Expand the value into multiple parts.
00417   if (NumParts & (NumParts - 1)) {
00418     // The number of parts is not a power of 2.  Split off and copy the tail.
00419     assert(PartVT.isInteger() && ValueVT.isInteger() &&
00420            "Do not know what to expand to!");
00421     unsigned RoundParts = 1 << Log2_32(NumParts);
00422     unsigned RoundBits = RoundParts * PartBits;
00423     unsigned OddParts = NumParts - RoundParts;
00424     SDValue OddVal = DAG.getNode(ISD::SRL, DL, ValueVT, Val,
00425                                  DAG.getIntPtrConstant(RoundBits));
00426     getCopyToParts(DAG, DL, OddVal, Parts + RoundParts, OddParts, PartVT, V);
00427 
00428     if (TLI.isBigEndian())
00429       // The odd parts were reversed by getCopyToParts - unreverse them.
00430       std::reverse(Parts + RoundParts, Parts + NumParts);
00431 
00432     NumParts = RoundParts;
00433     ValueVT = EVT::getIntegerVT(*DAG.getContext(), NumParts * PartBits);
00434     Val = DAG.getNode(ISD::TRUNCATE, DL, ValueVT, Val);
00435   }
00436 
00437   // The number of parts is a power of 2.  Repeatedly bisect the value using
00438   // EXTRACT_ELEMENT.
00439   Parts[0] = DAG.getNode(ISD::BITCAST, DL,
00440                          EVT::getIntegerVT(*DAG.getContext(),
00441                                            ValueVT.getSizeInBits()),
00442                          Val);
00443 
00444   for (unsigned StepSize = NumParts; StepSize > 1; StepSize /= 2) {
00445     for (unsigned i = 0; i < NumParts; i += StepSize) {
00446       unsigned ThisBits = StepSize * PartBits / 2;
00447       EVT ThisVT = EVT::getIntegerVT(*DAG.getContext(), ThisBits);
00448       SDValue &Part0 = Parts[i];
00449       SDValue &Part1 = Parts[i+StepSize/2];
00450 
00451       Part1 = DAG.getNode(ISD::EXTRACT_ELEMENT, DL,
00452                           ThisVT, Part0, DAG.getIntPtrConstant(1));
00453       Part0 = DAG.getNode(ISD::EXTRACT_ELEMENT, DL,
00454                           ThisVT, Part0, DAG.getIntPtrConstant(0));
00455 
00456       if (ThisBits == PartBits && ThisVT != PartVT) {
00457         Part0 = DAG.getNode(ISD::BITCAST, DL, PartVT, Part0);
00458         Part1 = DAG.getNode(ISD::BITCAST, DL, PartVT, Part1);
00459       }
00460     }
00461   }
00462 
00463   if (TLI.isBigEndian())
00464     std::reverse(Parts, Parts + OrigNumParts);
00465 }
00466 
00467 
00468 /// getCopyToPartsVector - Create a series of nodes that contain the specified
00469 /// value split into legal parts.
00470 static void getCopyToPartsVector(SelectionDAG &DAG, SDLoc DL,
00471                                  SDValue Val, SDValue *Parts, unsigned NumParts,
00472                                  MVT PartVT, const Value *V) {
00473   EVT ValueVT = Val.getValueType();
00474   assert(ValueVT.isVector() && "Not a vector");
00475   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
00476 
00477   if (NumParts == 1) {
00478     EVT PartEVT = PartVT;
00479     if (PartEVT == ValueVT) {
00480       // Nothing to do.
00481     } else if (PartVT.getSizeInBits() == ValueVT.getSizeInBits()) {
00482       // Bitconvert vector->vector case.
00483       Val = DAG.getNode(ISD::BITCAST, DL, PartVT, Val);
00484     } else if (PartVT.isVector() &&
00485                PartEVT.getVectorElementType() == ValueVT.getVectorElementType() &&
00486                PartEVT.getVectorNumElements() > ValueVT.getVectorNumElements()) {
00487       EVT ElementVT = PartVT.getVectorElementType();
00488       // Vector widening case, e.g. <2 x float> -> <4 x float>.  Shuffle in
00489       // undef elements.
00490       SmallVector<SDValue, 16> Ops;
00491       for (unsigned i = 0, e = ValueVT.getVectorNumElements(); i != e; ++i)
00492         Ops.push_back(DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL,
00493                                   ElementVT, Val, DAG.getConstant(i,
00494                                                   TLI.getVectorIdxTy())));
00495 
00496       for (unsigned i = ValueVT.getVectorNumElements(),
00497            e = PartVT.getVectorNumElements(); i != e; ++i)
00498         Ops.push_back(DAG.getUNDEF(ElementVT));
00499 
00500       Val = DAG.getNode(ISD::BUILD_VECTOR, DL, PartVT, Ops);
00501 
00502       // FIXME: Use CONCAT for 2x -> 4x.
00503 
00504       //SDValue UndefElts = DAG.getUNDEF(VectorTy);
00505       //Val = DAG.getNode(ISD::CONCAT_VECTORS, DL, PartVT, Val, UndefElts);
00506     } else if (PartVT.isVector() &&
00507                PartEVT.getVectorElementType().bitsGE(
00508                  ValueVT.getVectorElementType()) &&
00509                PartEVT.getVectorNumElements() == ValueVT.getVectorNumElements()) {
00510 
00511       // Promoted vector extract
00512       bool Smaller = PartEVT.bitsLE(ValueVT);
00513       Val = DAG.getNode((Smaller ? ISD::TRUNCATE : ISD::ANY_EXTEND),
00514                         DL, PartVT, Val);
00515     } else{
00516       // Vector -> scalar conversion.
00517       assert(ValueVT.getVectorNumElements() == 1 &&
00518              "Only trivial vector-to-scalar conversions should get here!");
00519       Val = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL,
00520                         PartVT, Val, DAG.getConstant(0, TLI.getVectorIdxTy()));
00521 
00522       bool Smaller = ValueVT.bitsLE(PartVT);
00523       Val = DAG.getNode((Smaller ? ISD::TRUNCATE : ISD::ANY_EXTEND),
00524                          DL, PartVT, Val);
00525     }
00526 
00527     Parts[0] = Val;
00528     return;
00529   }
00530 
00531   // Handle a multi-element vector.
00532   EVT IntermediateVT;
00533   MVT RegisterVT;
00534   unsigned NumIntermediates;
00535   unsigned NumRegs = TLI.getVectorTypeBreakdown(*DAG.getContext(), ValueVT,
00536                                                 IntermediateVT,
00537                                                 NumIntermediates, RegisterVT);
00538   unsigned NumElements = ValueVT.getVectorNumElements();
00539 
00540   assert(NumRegs == NumParts && "Part count doesn't match vector breakdown!");
00541   NumParts = NumRegs; // Silence a compiler warning.
00542   assert(RegisterVT == PartVT && "Part type doesn't match vector breakdown!");
00543 
00544   // Split the vector into intermediate operands.
00545   SmallVector<SDValue, 8> Ops(NumIntermediates);
00546   for (unsigned i = 0; i != NumIntermediates; ++i) {
00547     if (IntermediateVT.isVector())
00548       Ops[i] = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL,
00549                            IntermediateVT, Val,
00550                    DAG.getConstant(i * (NumElements / NumIntermediates),
00551                                    TLI.getVectorIdxTy()));
00552     else
00553       Ops[i] = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL,
00554                            IntermediateVT, Val,
00555                            DAG.getConstant(i, TLI.getVectorIdxTy()));
00556   }
00557 
00558   // Split the intermediate operands into legal parts.
00559   if (NumParts == NumIntermediates) {
00560     // If the register was not expanded, promote or copy the value,
00561     // as appropriate.
00562     for (unsigned i = 0; i != NumParts; ++i)
00563       getCopyToParts(DAG, DL, Ops[i], &Parts[i], 1, PartVT, V);
00564   } else if (NumParts > 0) {
00565     // If the intermediate type was expanded, split each the value into
00566     // legal parts.
00567     assert(NumParts % NumIntermediates == 0 &&
00568            "Must expand into a divisible number of parts!");
00569     unsigned Factor = NumParts / NumIntermediates;
00570     for (unsigned i = 0; i != NumIntermediates; ++i)
00571       getCopyToParts(DAG, DL, Ops[i], &Parts[i*Factor], Factor, PartVT, V);
00572   }
00573 }
00574 
00575 namespace {
00576   /// RegsForValue - This struct represents the registers (physical or virtual)
00577   /// that a particular set of values is assigned, and the type information
00578   /// about the value. The most common situation is to represent one value at a
00579   /// time, but struct or array values are handled element-wise as multiple
00580   /// values.  The splitting of aggregates is performed recursively, so that we
00581   /// never have aggregate-typed registers. The values at this point do not
00582   /// necessarily have legal types, so each value may require one or more
00583   /// registers of some legal type.
00584   ///
00585   struct RegsForValue {
00586     /// ValueVTs - The value types of the values, which may not be legal, and
00587     /// may need be promoted or synthesized from one or more registers.
00588     ///
00589     SmallVector<EVT, 4> ValueVTs;
00590 
00591     /// RegVTs - The value types of the registers. This is the same size as
00592     /// ValueVTs and it records, for each value, what the type of the assigned
00593     /// register or registers are. (Individual values are never synthesized
00594     /// from more than one type of register.)
00595     ///
00596     /// With virtual registers, the contents of RegVTs is redundant with TLI's
00597     /// getRegisterType member function, however when with physical registers
00598     /// it is necessary to have a separate record of the types.
00599     ///
00600     SmallVector<MVT, 4> RegVTs;
00601 
00602     /// Regs - This list holds the registers assigned to the values.
00603     /// Each legal or promoted value requires one register, and each
00604     /// expanded value requires multiple registers.
00605     ///
00606     SmallVector<unsigned, 4> Regs;
00607 
00608     RegsForValue() {}
00609 
00610     RegsForValue(const SmallVector<unsigned, 4> &regs,
00611                  MVT regvt, EVT valuevt)
00612       : ValueVTs(1, valuevt), RegVTs(1, regvt), Regs(regs) {}
00613 
00614     RegsForValue(LLVMContext &Context, const TargetLowering &tli,
00615                  unsigned Reg, Type *Ty) {
00616       ComputeValueVTs(tli, Ty, ValueVTs);
00617 
00618       for (unsigned Value = 0, e = ValueVTs.size(); Value != e; ++Value) {
00619         EVT ValueVT = ValueVTs[Value];
00620         unsigned NumRegs = tli.getNumRegisters(Context, ValueVT);
00621         MVT RegisterVT = tli.getRegisterType(Context, ValueVT);
00622         for (unsigned i = 0; i != NumRegs; ++i)
00623           Regs.push_back(Reg + i);
00624         RegVTs.push_back(RegisterVT);
00625         Reg += NumRegs;
00626       }
00627     }
00628 
00629     /// append - Add the specified values to this one.
00630     void append(const RegsForValue &RHS) {
00631       ValueVTs.append(RHS.ValueVTs.begin(), RHS.ValueVTs.end());
00632       RegVTs.append(RHS.RegVTs.begin(), RHS.RegVTs.end());
00633       Regs.append(RHS.Regs.begin(), RHS.Regs.end());
00634     }
00635 
00636     /// getCopyFromRegs - Emit a series of CopyFromReg nodes that copies from
00637     /// this value and returns the result as a ValueVTs value.  This uses
00638     /// Chain/Flag as the input and updates them for the output Chain/Flag.
00639     /// If the Flag pointer is NULL, no flag is used.
00640     SDValue getCopyFromRegs(SelectionDAG &DAG, FunctionLoweringInfo &FuncInfo,
00641                             SDLoc dl,
00642                             SDValue &Chain, SDValue *Flag,
00643                             const Value *V = nullptr) const;
00644 
00645     /// getCopyToRegs - Emit a series of CopyToReg nodes that copies the
00646     /// specified value into the registers specified by this object.  This uses
00647     /// Chain/Flag as the input and updates them for the output Chain/Flag.
00648     /// If the Flag pointer is NULL, no flag is used.
00649     void getCopyToRegs(SDValue Val, SelectionDAG &DAG, SDLoc dl,
00650                        SDValue &Chain, SDValue *Flag, const Value *V) const;
00651 
00652     /// AddInlineAsmOperands - Add this value to the specified inlineasm node
00653     /// operand list.  This adds the code marker, matching input operand index
00654     /// (if applicable), and includes the number of values added into it.
00655     void AddInlineAsmOperands(unsigned Kind,
00656                               bool HasMatching, unsigned MatchingIdx,
00657                               SelectionDAG &DAG,
00658                               std::vector<SDValue> &Ops) const;
00659   };
00660 }
00661 
00662 /// getCopyFromRegs - Emit a series of CopyFromReg nodes that copies from
00663 /// this value and returns the result as a ValueVT value.  This uses
00664 /// Chain/Flag as the input and updates them for the output Chain/Flag.
00665 /// If the Flag pointer is NULL, no flag is used.
00666 SDValue RegsForValue::getCopyFromRegs(SelectionDAG &DAG,
00667                                       FunctionLoweringInfo &FuncInfo,
00668                                       SDLoc dl,
00669                                       SDValue &Chain, SDValue *Flag,
00670                                       const Value *V) const {
00671   // A Value with type {} or [0 x %t] needs no registers.
00672   if (ValueVTs.empty())
00673     return SDValue();
00674 
00675   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
00676 
00677   // Assemble the legal parts into the final values.
00678   SmallVector<SDValue, 4> Values(ValueVTs.size());
00679   SmallVector<SDValue, 8> Parts;
00680   for (unsigned Value = 0, Part = 0, e = ValueVTs.size(); Value != e; ++Value) {
00681     // Copy the legal parts from the registers.
00682     EVT ValueVT = ValueVTs[Value];
00683     unsigned NumRegs = TLI.getNumRegisters(*DAG.getContext(), ValueVT);
00684     MVT RegisterVT = RegVTs[Value];
00685 
00686     Parts.resize(NumRegs);
00687     for (unsigned i = 0; i != NumRegs; ++i) {
00688       SDValue P;
00689       if (!Flag) {
00690         P = DAG.getCopyFromReg(Chain, dl, Regs[Part+i], RegisterVT);
00691       } else {
00692         P = DAG.getCopyFromReg(Chain, dl, Regs[Part+i], RegisterVT, *Flag);
00693         *Flag = P.getValue(2);
00694       }
00695 
00696       Chain = P.getValue(1);
00697       Parts[i] = P;
00698 
00699       // If the source register was virtual and if we know something about it,
00700       // add an assert node.
00701       if (!TargetRegisterInfo::isVirtualRegister(Regs[Part+i]) ||
00702           !RegisterVT.isInteger() || RegisterVT.isVector())
00703         continue;
00704 
00705       const FunctionLoweringInfo::LiveOutInfo *LOI =
00706         FuncInfo.GetLiveOutRegInfo(Regs[Part+i]);
00707       if (!LOI)
00708         continue;
00709 
00710       unsigned RegSize = RegisterVT.getSizeInBits();
00711       unsigned NumSignBits = LOI->NumSignBits;
00712       unsigned NumZeroBits = LOI->KnownZero.countLeadingOnes();
00713 
00714       if (NumZeroBits == RegSize) {
00715         // The current value is a zero.
00716         // Explicitly express that as it would be easier for
00717         // optimizations to kick in.
00718         Parts[i] = DAG.getConstant(0, RegisterVT);
00719         continue;
00720       }
00721 
00722       // FIXME: We capture more information than the dag can represent.  For
00723       // now, just use the tightest assertzext/assertsext possible.
00724       bool isSExt = true;
00725       EVT FromVT(MVT::Other);
00726       if (NumSignBits == RegSize)
00727         isSExt = true, FromVT = MVT::i1;   // ASSERT SEXT 1
00728       else if (NumZeroBits >= RegSize-1)
00729         isSExt = false, FromVT = MVT::i1;  // ASSERT ZEXT 1
00730       else if (NumSignBits > RegSize-8)
00731         isSExt = true, FromVT = MVT::i8;   // ASSERT SEXT 8
00732       else if (NumZeroBits >= RegSize-8)
00733         isSExt = false, FromVT = MVT::i8;  // ASSERT ZEXT 8
00734       else if (NumSignBits > RegSize-16)
00735         isSExt = true, FromVT = MVT::i16;  // ASSERT SEXT 16
00736       else if (NumZeroBits >= RegSize-16)
00737         isSExt = false, FromVT = MVT::i16; // ASSERT ZEXT 16
00738       else if (NumSignBits > RegSize-32)
00739         isSExt = true, FromVT = MVT::i32;  // ASSERT SEXT 32
00740       else if (NumZeroBits >= RegSize-32)
00741         isSExt = false, FromVT = MVT::i32; // ASSERT ZEXT 32
00742       else
00743         continue;
00744 
00745       // Add an assertion node.
00746       assert(FromVT != MVT::Other);
00747       Parts[i] = DAG.getNode(isSExt ? ISD::AssertSext : ISD::AssertZext, dl,
00748                              RegisterVT, P, DAG.getValueType(FromVT));
00749     }
00750 
00751     Values[Value] = getCopyFromParts(DAG, dl, Parts.begin(),
00752                                      NumRegs, RegisterVT, ValueVT, V);
00753     Part += NumRegs;
00754     Parts.clear();
00755   }
00756 
00757   return DAG.getNode(ISD::MERGE_VALUES, dl, DAG.getVTList(ValueVTs), Values);
00758 }
00759 
00760 /// getCopyToRegs - Emit a series of CopyToReg nodes that copies the
00761 /// specified value into the registers specified by this object.  This uses
00762 /// Chain/Flag as the input and updates them for the output Chain/Flag.
00763 /// If the Flag pointer is NULL, no flag is used.
00764 void RegsForValue::getCopyToRegs(SDValue Val, SelectionDAG &DAG, SDLoc dl,
00765                                  SDValue &Chain, SDValue *Flag,
00766                                  const Value *V) const {
00767   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
00768 
00769   // Get the list of the values's legal parts.
00770   unsigned NumRegs = Regs.size();
00771   SmallVector<SDValue, 8> Parts(NumRegs);
00772   for (unsigned Value = 0, Part = 0, e = ValueVTs.size(); Value != e; ++Value) {
00773     EVT ValueVT = ValueVTs[Value];
00774     unsigned NumParts = TLI.getNumRegisters(*DAG.getContext(), ValueVT);
00775     MVT RegisterVT = RegVTs[Value];
00776     ISD::NodeType ExtendKind =
00777       TLI.isZExtFree(Val, RegisterVT)? ISD::ZERO_EXTEND: ISD::ANY_EXTEND;
00778 
00779     getCopyToParts(DAG, dl, Val.getValue(Val.getResNo() + Value),
00780                    &Parts[Part], NumParts, RegisterVT, V, ExtendKind);
00781     Part += NumParts;
00782   }
00783 
00784   // Copy the parts into the registers.
00785   SmallVector<SDValue, 8> Chains(NumRegs);
00786   for (unsigned i = 0; i != NumRegs; ++i) {
00787     SDValue Part;
00788     if (!Flag) {
00789       Part = DAG.getCopyToReg(Chain, dl, Regs[i], Parts[i]);
00790     } else {
00791       Part = DAG.getCopyToReg(Chain, dl, Regs[i], Parts[i], *Flag);
00792       *Flag = Part.getValue(1);
00793     }
00794 
00795     Chains[i] = Part.getValue(0);
00796   }
00797 
00798   if (NumRegs == 1 || Flag)
00799     // If NumRegs > 1 && Flag is used then the use of the last CopyToReg is
00800     // flagged to it. That is the CopyToReg nodes and the user are considered
00801     // a single scheduling unit. If we create a TokenFactor and return it as
00802     // chain, then the TokenFactor is both a predecessor (operand) of the
00803     // user as well as a successor (the TF operands are flagged to the user).
00804     // c1, f1 = CopyToReg
00805     // c2, f2 = CopyToReg
00806     // c3     = TokenFactor c1, c2
00807     // ...
00808     //        = op c3, ..., f2
00809     Chain = Chains[NumRegs-1];
00810   else
00811     Chain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Chains);
00812 }
00813 
00814 /// AddInlineAsmOperands - Add this value to the specified inlineasm node
00815 /// operand list.  This adds the code marker and includes the number of
00816 /// values added into it.
00817 void RegsForValue::AddInlineAsmOperands(unsigned Code, bool HasMatching,
00818                                         unsigned MatchingIdx,
00819                                         SelectionDAG &DAG,
00820                                         std::vector<SDValue> &Ops) const {
00821   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
00822 
00823   unsigned Flag = InlineAsm::getFlagWord(Code, Regs.size());
00824   if (HasMatching)
00825     Flag = InlineAsm::getFlagWordForMatchingOp(Flag, MatchingIdx);
00826   else if (!Regs.empty() &&
00827            TargetRegisterInfo::isVirtualRegister(Regs.front())) {
00828     // Put the register class of the virtual registers in the flag word.  That
00829     // way, later passes can recompute register class constraints for inline
00830     // assembly as well as normal instructions.
00831     // Don't do this for tied operands that can use the regclass information
00832     // from the def.
00833     const MachineRegisterInfo &MRI = DAG.getMachineFunction().getRegInfo();
00834     const TargetRegisterClass *RC = MRI.getRegClass(Regs.front());
00835     Flag = InlineAsm::getFlagWordForRegClass(Flag, RC->getID());
00836   }
00837 
00838   SDValue Res = DAG.getTargetConstant(Flag, MVT::i32);
00839   Ops.push_back(Res);
00840 
00841   unsigned SP = TLI.getStackPointerRegisterToSaveRestore();
00842   for (unsigned Value = 0, Reg = 0, e = ValueVTs.size(); Value != e; ++Value) {
00843     unsigned NumRegs = TLI.getNumRegisters(*DAG.getContext(), ValueVTs[Value]);
00844     MVT RegisterVT = RegVTs[Value];
00845     for (unsigned i = 0; i != NumRegs; ++i) {
00846       assert(Reg < Regs.size() && "Mismatch in # registers expected");
00847       unsigned TheReg = Regs[Reg++];
00848       Ops.push_back(DAG.getRegister(TheReg, RegisterVT));
00849 
00850       if (TheReg == SP && Code == InlineAsm::Kind_Clobber) {
00851         // If we clobbered the stack pointer, MFI should know about it.
00852         assert(DAG.getMachineFunction().getFrameInfo()->
00853             hasInlineAsmWithSPAdjust());
00854       }
00855     }
00856   }
00857 }
00858 
00859 void SelectionDAGBuilder::init(GCFunctionInfo *gfi, AliasAnalysis &aa,
00860                                const TargetLibraryInfo *li) {
00861   AA = &aa;
00862   GFI = gfi;
00863   LibInfo = li;
00864   DL = DAG.getSubtarget().getDataLayout();
00865   Context = DAG.getContext();
00866   LPadToCallSiteMap.clear();
00867 }
00868 
00869 /// clear - Clear out the current SelectionDAG and the associated
00870 /// state and prepare this SelectionDAGBuilder object to be used
00871 /// for a new block. This doesn't clear out information about
00872 /// additional blocks that are needed to complete switch lowering
00873 /// or PHI node updating; that information is cleared out as it is
00874 /// consumed.
00875 void SelectionDAGBuilder::clear() {
00876   NodeMap.clear();
00877   UnusedArgNodeMap.clear();
00878   PendingLoads.clear();
00879   PendingExports.clear();
00880   CurInst = nullptr;
00881   HasTailCall = false;
00882   SDNodeOrder = LowestSDNodeOrder;
00883 }
00884 
00885 /// clearDanglingDebugInfo - Clear the dangling debug information
00886 /// map. This function is separated from the clear so that debug
00887 /// information that is dangling in a basic block can be properly
00888 /// resolved in a different basic block. This allows the
00889 /// SelectionDAG to resolve dangling debug information attached
00890 /// to PHI nodes.
00891 void SelectionDAGBuilder::clearDanglingDebugInfo() {
00892   DanglingDebugInfoMap.clear();
00893 }
00894 
00895 /// getRoot - Return the current virtual root of the Selection DAG,
00896 /// flushing any PendingLoad items. This must be done before emitting
00897 /// a store or any other node that may need to be ordered after any
00898 /// prior load instructions.
00899 ///
00900 SDValue SelectionDAGBuilder::getRoot() {
00901   if (PendingLoads.empty())
00902     return DAG.getRoot();
00903 
00904   if (PendingLoads.size() == 1) {
00905     SDValue Root = PendingLoads[0];
00906     DAG.setRoot(Root);
00907     PendingLoads.clear();
00908     return Root;
00909   }
00910 
00911   // Otherwise, we have to make a token factor node.
00912   SDValue Root = DAG.getNode(ISD::TokenFactor, getCurSDLoc(), MVT::Other,
00913                              PendingLoads);
00914   PendingLoads.clear();
00915   DAG.setRoot(Root);
00916   return Root;
00917 }
00918 
00919 /// getControlRoot - Similar to getRoot, but instead of flushing all the
00920 /// PendingLoad items, flush all the PendingExports items. It is necessary
00921 /// to do this before emitting a terminator instruction.
00922 ///
00923 SDValue SelectionDAGBuilder::getControlRoot() {
00924   SDValue Root = DAG.getRoot();
00925 
00926   if (PendingExports.empty())
00927     return Root;
00928 
00929   // Turn all of the CopyToReg chains into one factored node.
00930   if (Root.getOpcode() != ISD::EntryToken) {
00931     unsigned i = 0, e = PendingExports.size();
00932     for (; i != e; ++i) {
00933       assert(PendingExports[i].getNode()->getNumOperands() > 1);
00934       if (PendingExports[i].getNode()->getOperand(0) == Root)
00935         break;  // Don't add the root if we already indirectly depend on it.
00936     }
00937 
00938     if (i == e)
00939       PendingExports.push_back(Root);
00940   }
00941 
00942   Root = DAG.getNode(ISD::TokenFactor, getCurSDLoc(), MVT::Other,
00943                      PendingExports);
00944   PendingExports.clear();
00945   DAG.setRoot(Root);
00946   return Root;
00947 }
00948 
00949 void SelectionDAGBuilder::visit(const Instruction &I) {
00950   // Set up outgoing PHI node register values before emitting the terminator.
00951   if (isa<TerminatorInst>(&I))
00952     HandlePHINodesInSuccessorBlocks(I.getParent());
00953 
00954   ++SDNodeOrder;
00955 
00956   CurInst = &I;
00957 
00958   visit(I.getOpcode(), I);
00959 
00960   if (!isa<TerminatorInst>(&I) && !HasTailCall)
00961     CopyToExportRegsIfNeeded(&I);
00962 
00963   CurInst = nullptr;
00964 }
00965 
00966 void SelectionDAGBuilder::visitPHI(const PHINode &) {
00967   llvm_unreachable("SelectionDAGBuilder shouldn't visit PHI nodes!");
00968 }
00969 
00970 void SelectionDAGBuilder::visit(unsigned Opcode, const User &I) {
00971   // Note: this doesn't use InstVisitor, because it has to work with
00972   // ConstantExpr's in addition to instructions.
00973   switch (Opcode) {
00974   default: llvm_unreachable("Unknown instruction type encountered!");
00975     // Build the switch statement using the Instruction.def file.
00976 #define HANDLE_INST(NUM, OPCODE, CLASS) \
00977     case Instruction::OPCODE: visit##OPCODE((const CLASS&)I); break;
00978 #include "llvm/IR/Instruction.def"
00979   }
00980 }
00981 
00982 // resolveDanglingDebugInfo - if we saw an earlier dbg_value referring to V,
00983 // generate the debug data structures now that we've seen its definition.
00984 void SelectionDAGBuilder::resolveDanglingDebugInfo(const Value *V,
00985                                                    SDValue Val) {
00986   DanglingDebugInfo &DDI = DanglingDebugInfoMap[V];
00987   if (DDI.getDI()) {
00988     const DbgValueInst *DI = DDI.getDI();
00989     DebugLoc dl = DDI.getdl();
00990     unsigned DbgSDNodeOrder = DDI.getSDNodeOrder();
00991     MDNode *Variable = DI->getVariable();
00992     uint64_t Offset = DI->getOffset();
00993     // A dbg.value for an alloca is always indirect.
00994     bool IsIndirect = isa<AllocaInst>(V) || Offset != 0;
00995     SDDbgValue *SDV;
00996     if (Val.getNode()) {
00997       if (!EmitFuncArgumentDbgValue(V, Variable, Offset, IsIndirect, Val)) {
00998         SDV = DAG.getDbgValue(Variable, Val.getNode(),
00999                               Val.getResNo(), IsIndirect,
01000             Offset, dl, DbgSDNodeOrder);
01001         DAG.AddDbgValue(SDV, Val.getNode(), false);
01002       }
01003     } else
01004       DEBUG(dbgs() << "Dropping debug info for " << *DI << "\n");
01005     DanglingDebugInfoMap[V] = DanglingDebugInfo();
01006   }
01007 }
01008 
01009 /// getValue - Return an SDValue for the given Value.
01010 SDValue SelectionDAGBuilder::getValue(const Value *V) {
01011   // If we already have an SDValue for this value, use it. It's important
01012   // to do this first, so that we don't create a CopyFromReg if we already
01013   // have a regular SDValue.
01014   SDValue &N = NodeMap[V];
01015   if (N.getNode()) return N;
01016 
01017   // If there's a virtual register allocated and initialized for this
01018   // value, use it.
01019   DenseMap<const Value *, unsigned>::iterator It = FuncInfo.ValueMap.find(V);
01020   if (It != FuncInfo.ValueMap.end()) {
01021     unsigned InReg = It->second;
01022     RegsForValue RFV(*DAG.getContext(),
01023                      *TM.getSubtargetImpl()->getTargetLowering(), InReg,
01024                      V->getType());
01025     SDValue Chain = DAG.getEntryNode();
01026     N = RFV.getCopyFromRegs(DAG, FuncInfo, getCurSDLoc(), Chain, nullptr, V);
01027     resolveDanglingDebugInfo(V, N);
01028     return N;
01029   }
01030 
01031   // Otherwise create a new SDValue and remember it.
01032   SDValue Val = getValueImpl(V);
01033   NodeMap[V] = Val;
01034   resolveDanglingDebugInfo(V, Val);
01035   return Val;
01036 }
01037 
01038 /// getNonRegisterValue - Return an SDValue for the given Value, but
01039 /// don't look in FuncInfo.ValueMap for a virtual register.
01040 SDValue SelectionDAGBuilder::getNonRegisterValue(const Value *V) {
01041   // If we already have an SDValue for this value, use it.
01042   SDValue &N = NodeMap[V];
01043   if (N.getNode()) return N;
01044 
01045   // Otherwise create a new SDValue and remember it.
01046   SDValue Val = getValueImpl(V);
01047   NodeMap[V] = Val;
01048   resolveDanglingDebugInfo(V, Val);
01049   return Val;
01050 }
01051 
01052 /// getValueImpl - Helper function for getValue and getNonRegisterValue.
01053 /// Create an SDValue for the given value.
01054 SDValue SelectionDAGBuilder::getValueImpl(const Value *V) {
01055   const TargetLowering *TLI = TM.getSubtargetImpl()->getTargetLowering();
01056 
01057   if (const Constant *C = dyn_cast<Constant>(V)) {
01058     EVT VT = TLI->getValueType(V->getType(), true);
01059 
01060     if (const ConstantInt *CI = dyn_cast<ConstantInt>(C))
01061       return DAG.getConstant(*CI, VT);
01062 
01063     if (const GlobalValue *GV = dyn_cast<GlobalValue>(C))
01064       return DAG.getGlobalAddress(GV, getCurSDLoc(), VT);
01065 
01066     if (isa<ConstantPointerNull>(C)) {
01067       unsigned AS = V->getType()->getPointerAddressSpace();
01068       return DAG.getConstant(0, TLI->getPointerTy(AS));
01069     }
01070 
01071     if (const ConstantFP *CFP = dyn_cast<ConstantFP>(C))
01072       return DAG.getConstantFP(*CFP, VT);
01073 
01074     if (isa<UndefValue>(C) && !V->getType()->isAggregateType())
01075       return DAG.getUNDEF(VT);
01076 
01077     if (const ConstantExpr *CE = dyn_cast<ConstantExpr>(C)) {
01078       visit(CE->getOpcode(), *CE);
01079       SDValue N1 = NodeMap[V];
01080       assert(N1.getNode() && "visit didn't populate the NodeMap!");
01081       return N1;
01082     }
01083 
01084     if (isa<ConstantStruct>(C) || isa<ConstantArray>(C)) {
01085       SmallVector<SDValue, 4> Constants;
01086       for (User::const_op_iterator OI = C->op_begin(), OE = C->op_end();
01087            OI != OE; ++OI) {
01088         SDNode *Val = getValue(*OI).getNode();
01089         // If the operand is an empty aggregate, there are no values.
01090         if (!Val) continue;
01091         // Add each leaf value from the operand to the Constants list
01092         // to form a flattened list of all the values.
01093         for (unsigned i = 0, e = Val->getNumValues(); i != e; ++i)
01094           Constants.push_back(SDValue(Val, i));
01095       }
01096 
01097       return DAG.getMergeValues(Constants, getCurSDLoc());
01098     }
01099 
01100     if (const ConstantDataSequential *CDS =
01101           dyn_cast<ConstantDataSequential>(C)) {
01102       SmallVector<SDValue, 4> Ops;
01103       for (unsigned i = 0, e = CDS->getNumElements(); i != e; ++i) {
01104         SDNode *Val = getValue(CDS->getElementAsConstant(i)).getNode();
01105         // Add each leaf value from the operand to the Constants list
01106         // to form a flattened list of all the values.
01107         for (unsigned i = 0, e = Val->getNumValues(); i != e; ++i)
01108           Ops.push_back(SDValue(Val, i));
01109       }
01110 
01111       if (isa<ArrayType>(CDS->getType()))
01112         return DAG.getMergeValues(Ops, getCurSDLoc());
01113       return NodeMap[V] = DAG.getNode(ISD::BUILD_VECTOR, getCurSDLoc(),
01114                                       VT, Ops);
01115     }
01116 
01117     if (C->getType()->isStructTy() || C->getType()->isArrayTy()) {
01118       assert((isa<ConstantAggregateZero>(C) || isa<UndefValue>(C)) &&
01119              "Unknown struct or array constant!");
01120 
01121       SmallVector<EVT, 4> ValueVTs;
01122       ComputeValueVTs(*TLI, C->getType(), ValueVTs);
01123       unsigned NumElts = ValueVTs.size();
01124       if (NumElts == 0)
01125         return SDValue(); // empty struct
01126       SmallVector<SDValue, 4> Constants(NumElts);
01127       for (unsigned i = 0; i != NumElts; ++i) {
01128         EVT EltVT = ValueVTs[i];
01129         if (isa<UndefValue>(C))
01130           Constants[i] = DAG.getUNDEF(EltVT);
01131         else if (EltVT.isFloatingPoint())
01132           Constants[i] = DAG.getConstantFP(0, EltVT);
01133         else
01134           Constants[i] = DAG.getConstant(0, EltVT);
01135       }
01136 
01137       return DAG.getMergeValues(Constants, getCurSDLoc());
01138     }
01139 
01140     if (const BlockAddress *BA = dyn_cast<BlockAddress>(C))
01141       return DAG.getBlockAddress(BA, VT);
01142 
01143     VectorType *VecTy = cast<VectorType>(V->getType());
01144     unsigned NumElements = VecTy->getNumElements();
01145 
01146     // Now that we know the number and type of the elements, get that number of
01147     // elements into the Ops array based on what kind of constant it is.
01148     SmallVector<SDValue, 16> Ops;
01149     if (const ConstantVector *CV = dyn_cast<ConstantVector>(C)) {
01150       for (unsigned i = 0; i != NumElements; ++i)
01151         Ops.push_back(getValue(CV->getOperand(i)));
01152     } else {
01153       assert(isa<ConstantAggregateZero>(C) && "Unknown vector constant!");
01154       EVT EltVT = TLI->getValueType(VecTy->getElementType());
01155 
01156       SDValue Op;
01157       if (EltVT.isFloatingPoint())
01158         Op = DAG.getConstantFP(0, EltVT);
01159       else
01160         Op = DAG.getConstant(0, EltVT);
01161       Ops.assign(NumElements, Op);
01162     }
01163 
01164     // Create a BUILD_VECTOR node.
01165     return NodeMap[V] = DAG.getNode(ISD::BUILD_VECTOR, getCurSDLoc(), VT, Ops);
01166   }
01167 
01168   // If this is a static alloca, generate it as the frameindex instead of
01169   // computation.
01170   if (const AllocaInst *AI = dyn_cast<AllocaInst>(V)) {
01171     DenseMap<const AllocaInst*, int>::iterator SI =
01172       FuncInfo.StaticAllocaMap.find(AI);
01173     if (SI != FuncInfo.StaticAllocaMap.end())
01174       return DAG.getFrameIndex(SI->second, TLI->getPointerTy());
01175   }
01176 
01177   // If this is an instruction which fast-isel has deferred, select it now.
01178   if (const Instruction *Inst = dyn_cast<Instruction>(V)) {
01179     unsigned InReg = FuncInfo.InitializeRegForValue(Inst);
01180     RegsForValue RFV(*DAG.getContext(), *TLI, InReg, Inst->getType());
01181     SDValue Chain = DAG.getEntryNode();
01182     return RFV.getCopyFromRegs(DAG, FuncInfo, getCurSDLoc(), Chain, nullptr, V);
01183   }
01184 
01185   llvm_unreachable("Can't get register for value!");
01186 }
01187 
01188 void SelectionDAGBuilder::visitRet(const ReturnInst &I) {
01189   const TargetLowering *TLI = TM.getSubtargetImpl()->getTargetLowering();
01190   SDValue Chain = getControlRoot();
01191   SmallVector<ISD::OutputArg, 8> Outs;
01192   SmallVector<SDValue, 8> OutVals;
01193 
01194   if (!FuncInfo.CanLowerReturn) {
01195     unsigned DemoteReg = FuncInfo.DemoteRegister;
01196     const Function *F = I.getParent()->getParent();
01197 
01198     // Emit a store of the return value through the virtual register.
01199     // Leave Outs empty so that LowerReturn won't try to load return
01200     // registers the usual way.
01201     SmallVector<EVT, 1> PtrValueVTs;
01202     ComputeValueVTs(*TLI, PointerType::getUnqual(F->getReturnType()),
01203                     PtrValueVTs);
01204 
01205     SDValue RetPtr = DAG.getRegister(DemoteReg, PtrValueVTs[0]);
01206     SDValue RetOp = getValue(I.getOperand(0));
01207 
01208     SmallVector<EVT, 4> ValueVTs;
01209     SmallVector<uint64_t, 4> Offsets;
01210     ComputeValueVTs(*TLI, I.getOperand(0)->getType(), ValueVTs, &Offsets);
01211     unsigned NumValues = ValueVTs.size();
01212 
01213     SmallVector<SDValue, 4> Chains(NumValues);
01214     for (unsigned i = 0; i != NumValues; ++i) {
01215       SDValue Add = DAG.getNode(ISD::ADD, getCurSDLoc(),
01216                                 RetPtr.getValueType(), RetPtr,
01217                                 DAG.getIntPtrConstant(Offsets[i]));
01218       Chains[i] =
01219         DAG.getStore(Chain, getCurSDLoc(),
01220                      SDValue(RetOp.getNode(), RetOp.getResNo() + i),
01221                      // FIXME: better loc info would be nice.
01222                      Add, MachinePointerInfo(), false, false, 0);
01223     }
01224 
01225     Chain = DAG.getNode(ISD::TokenFactor, getCurSDLoc(),
01226                         MVT::Other, Chains);
01227   } else if (I.getNumOperands() != 0) {
01228     SmallVector<EVT, 4> ValueVTs;
01229     ComputeValueVTs(*TLI, I.getOperand(0)->getType(), ValueVTs);
01230     unsigned NumValues = ValueVTs.size();
01231     if (NumValues) {
01232       SDValue RetOp = getValue(I.getOperand(0));
01233       for (unsigned j = 0, f = NumValues; j != f; ++j) {
01234         EVT VT = ValueVTs[j];
01235 
01236         ISD::NodeType ExtendKind = ISD::ANY_EXTEND;
01237 
01238         const Function *F = I.getParent()->getParent();
01239         if (F->getAttributes().hasAttribute(AttributeSet::ReturnIndex,
01240                                             Attribute::SExt))
01241           ExtendKind = ISD::SIGN_EXTEND;
01242         else if (F->getAttributes().hasAttribute(AttributeSet::ReturnIndex,
01243                                                  Attribute::ZExt))
01244           ExtendKind = ISD::ZERO_EXTEND;
01245 
01246         if (ExtendKind != ISD::ANY_EXTEND && VT.isInteger())
01247           VT = TLI->getTypeForExtArgOrReturn(*DAG.getContext(), VT, ExtendKind);
01248 
01249         unsigned NumParts = TLI->getNumRegisters(*DAG.getContext(), VT);
01250         MVT PartVT = TLI->getRegisterType(*DAG.getContext(), VT);
01251         SmallVector<SDValue, 4> Parts(NumParts);
01252         getCopyToParts(DAG, getCurSDLoc(),
01253                        SDValue(RetOp.getNode(), RetOp.getResNo() + j),
01254                        &Parts[0], NumParts, PartVT, &I, ExtendKind);
01255 
01256         // 'inreg' on function refers to return value
01257         ISD::ArgFlagsTy Flags = ISD::ArgFlagsTy();
01258         if (F->getAttributes().hasAttribute(AttributeSet::ReturnIndex,
01259                                             Attribute::InReg))
01260           Flags.setInReg();
01261 
01262         // Propagate extension type if any
01263         if (ExtendKind == ISD::SIGN_EXTEND)
01264           Flags.setSExt();
01265         else if (ExtendKind == ISD::ZERO_EXTEND)
01266           Flags.setZExt();
01267 
01268         for (unsigned i = 0; i < NumParts; ++i) {
01269           Outs.push_back(ISD::OutputArg(Flags, Parts[i].getValueType(),
01270                                         VT, /*isfixed=*/true, 0, 0));
01271           OutVals.push_back(Parts[i]);
01272         }
01273       }
01274     }
01275   }
01276 
01277   bool isVarArg = DAG.getMachineFunction().getFunction()->isVarArg();
01278   CallingConv::ID CallConv =
01279     DAG.getMachineFunction().getFunction()->getCallingConv();
01280   Chain = TM.getSubtargetImpl()->getTargetLowering()->LowerReturn(
01281       Chain, CallConv, isVarArg, Outs, OutVals, getCurSDLoc(), DAG);
01282 
01283   // Verify that the target's LowerReturn behaved as expected.
01284   assert(Chain.getNode() && Chain.getValueType() == MVT::Other &&
01285          "LowerReturn didn't return a valid chain!");
01286 
01287   // Update the DAG with the new chain value resulting from return lowering.
01288   DAG.setRoot(Chain);
01289 }
01290 
01291 /// CopyToExportRegsIfNeeded - If the given value has virtual registers
01292 /// created for it, emit nodes to copy the value into the virtual
01293 /// registers.
01294 void SelectionDAGBuilder::CopyToExportRegsIfNeeded(const Value *V) {
01295   // Skip empty types
01296   if (V->getType()->isEmptyTy())
01297     return;
01298 
01299   DenseMap<const Value *, unsigned>::iterator VMI = FuncInfo.ValueMap.find(V);
01300   if (VMI != FuncInfo.ValueMap.end()) {
01301     assert(!V->use_empty() && "Unused value assigned virtual registers!");
01302     CopyValueToVirtualRegister(V, VMI->second);
01303   }
01304 }
01305 
01306 /// ExportFromCurrentBlock - If this condition isn't known to be exported from
01307 /// the current basic block, add it to ValueMap now so that we'll get a
01308 /// CopyTo/FromReg.
01309 void SelectionDAGBuilder::ExportFromCurrentBlock(const Value *V) {
01310   // No need to export constants.
01311   if (!isa<Instruction>(V) && !isa<Argument>(V)) return;
01312 
01313   // Already exported?
01314   if (FuncInfo.isExportedInst(V)) return;
01315 
01316   unsigned Reg = FuncInfo.InitializeRegForValue(V);
01317   CopyValueToVirtualRegister(V, Reg);
01318 }
01319 
01320 bool SelectionDAGBuilder::isExportableFromCurrentBlock(const Value *V,
01321                                                      const BasicBlock *FromBB) {
01322   // The operands of the setcc have to be in this block.  We don't know
01323   // how to export them from some other block.
01324   if (const Instruction *VI = dyn_cast<Instruction>(V)) {
01325     // Can export from current BB.
01326     if (VI->getParent() == FromBB)
01327       return true;
01328 
01329     // Is already exported, noop.
01330     return FuncInfo.isExportedInst(V);
01331   }
01332 
01333   // If this is an argument, we can export it if the BB is the entry block or
01334   // if it is already exported.
01335   if (isa<Argument>(V)) {
01336     if (FromBB == &FromBB->getParent()->getEntryBlock())
01337       return true;
01338 
01339     // Otherwise, can only export this if it is already exported.
01340     return FuncInfo.isExportedInst(V);
01341   }
01342 
01343   // Otherwise, constants can always be exported.
01344   return true;
01345 }
01346 
01347 /// Return branch probability calculated by BranchProbabilityInfo for IR blocks.
01348 uint32_t SelectionDAGBuilder::getEdgeWeight(const MachineBasicBlock *Src,
01349                                             const MachineBasicBlock *Dst) const {
01350   BranchProbabilityInfo *BPI = FuncInfo.BPI;
01351   if (!BPI)
01352     return 0;
01353   const BasicBlock *SrcBB = Src->getBasicBlock();
01354   const BasicBlock *DstBB = Dst->getBasicBlock();
01355   return BPI->getEdgeWeight(SrcBB, DstBB);
01356 }
01357 
01358 void SelectionDAGBuilder::
01359 addSuccessorWithWeight(MachineBasicBlock *Src, MachineBasicBlock *Dst,
01360                        uint32_t Weight /* = 0 */) {
01361   if (!Weight)
01362     Weight = getEdgeWeight(Src, Dst);
01363   Src->addSuccessor(Dst, Weight);
01364 }
01365 
01366 
01367 static bool InBlock(const Value *V, const BasicBlock *BB) {
01368   if (const Instruction *I = dyn_cast<Instruction>(V))
01369     return I->getParent() == BB;
01370   return true;
01371 }
01372 
01373 /// EmitBranchForMergedCondition - Helper method for FindMergedConditions.
01374 /// This function emits a branch and is used at the leaves of an OR or an
01375 /// AND operator tree.
01376 ///
01377 void
01378 SelectionDAGBuilder::EmitBranchForMergedCondition(const Value *Cond,
01379                                                   MachineBasicBlock *TBB,
01380                                                   MachineBasicBlock *FBB,
01381                                                   MachineBasicBlock *CurBB,
01382                                                   MachineBasicBlock *SwitchBB,
01383                                                   uint32_t TWeight,
01384                                                   uint32_t FWeight) {
01385   const BasicBlock *BB = CurBB->getBasicBlock();
01386 
01387   // If the leaf of the tree is a comparison, merge the condition into
01388   // the caseblock.
01389   if (const CmpInst *BOp = dyn_cast<CmpInst>(Cond)) {
01390     // The operands of the cmp have to be in this block.  We don't know
01391     // how to export them from some other block.  If this is the first block
01392     // of the sequence, no exporting is needed.
01393     if (CurBB == SwitchBB ||
01394         (isExportableFromCurrentBlock(BOp->getOperand(0), BB) &&
01395          isExportableFromCurrentBlock(BOp->getOperand(1), BB))) {
01396       ISD::CondCode Condition;
01397       if (const ICmpInst *IC = dyn_cast<ICmpInst>(Cond)) {
01398         Condition = getICmpCondCode(IC->getPredicate());
01399       } else if (const FCmpInst *FC = dyn_cast<FCmpInst>(Cond)) {
01400         Condition = getFCmpCondCode(FC->getPredicate());
01401         if (TM.Options.NoNaNsFPMath)
01402           Condition = getFCmpCodeWithoutNaN(Condition);
01403       } else {
01404         Condition = ISD::SETEQ; // silence warning.
01405         llvm_unreachable("Unknown compare instruction");
01406       }
01407 
01408       CaseBlock CB(Condition, BOp->getOperand(0), BOp->getOperand(1), nullptr,
01409                    TBB, FBB, CurBB, TWeight, FWeight);
01410       SwitchCases.push_back(CB);
01411       return;
01412     }
01413   }
01414 
01415   // Create a CaseBlock record representing this branch.
01416   CaseBlock CB(ISD::SETEQ, Cond, ConstantInt::getTrue(*DAG.getContext()),
01417                nullptr, TBB, FBB, CurBB, TWeight, FWeight);
01418   SwitchCases.push_back(CB);
01419 }
01420 
01421 /// Scale down both weights to fit into uint32_t.
01422 static void ScaleWeights(uint64_t &NewTrue, uint64_t &NewFalse) {
01423   uint64_t NewMax = (NewTrue > NewFalse) ? NewTrue : NewFalse;
01424   uint32_t Scale = (NewMax / UINT32_MAX) + 1;
01425   NewTrue = NewTrue / Scale;
01426   NewFalse = NewFalse / Scale;
01427 }
01428 
01429 /// FindMergedConditions - If Cond is an expression like
01430 void SelectionDAGBuilder::FindMergedConditions(const Value *Cond,
01431                                                MachineBasicBlock *TBB,
01432                                                MachineBasicBlock *FBB,
01433                                                MachineBasicBlock *CurBB,
01434                                                MachineBasicBlock *SwitchBB,
01435                                                unsigned Opc, uint32_t TWeight,
01436                                                uint32_t FWeight) {
01437   // If this node is not part of the or/and tree, emit it as a branch.
01438   const Instruction *BOp = dyn_cast<Instruction>(Cond);
01439   if (!BOp || !(isa<BinaryOperator>(BOp) || isa<CmpInst>(BOp)) ||
01440       (unsigned)BOp->getOpcode() != Opc || !BOp->hasOneUse() ||
01441       BOp->getParent() != CurBB->getBasicBlock() ||
01442       !InBlock(BOp->getOperand(0), CurBB->getBasicBlock()) ||
01443       !InBlock(BOp->getOperand(1), CurBB->getBasicBlock())) {
01444     EmitBranchForMergedCondition(Cond, TBB, FBB, CurBB, SwitchBB,
01445                                  TWeight, FWeight);
01446     return;
01447   }
01448 
01449   //  Create TmpBB after CurBB.
01450   MachineFunction::iterator BBI = CurBB;
01451   MachineFunction &MF = DAG.getMachineFunction();
01452   MachineBasicBlock *TmpBB = MF.CreateMachineBasicBlock(CurBB->getBasicBlock());
01453   CurBB->getParent()->insert(++BBI, TmpBB);
01454 
01455   if (Opc == Instruction::Or) {
01456     // Codegen X | Y as:
01457     // BB1:
01458     //   jmp_if_X TBB
01459     //   jmp TmpBB
01460     // TmpBB:
01461     //   jmp_if_Y TBB
01462     //   jmp FBB
01463     //
01464 
01465     // We have flexibility in setting Prob for BB1 and Prob for TmpBB.
01466     // The requirement is that
01467     //   TrueProb for BB1 + (FalseProb for BB1 * TrueProb for TmpBB)
01468     //     = TrueProb for orignal BB.
01469     // Assuming the orignal weights are A and B, one choice is to set BB1's
01470     // weights to A and A+2B, and set TmpBB's weights to A and 2B. This choice
01471     // assumes that
01472     //   TrueProb for BB1 == FalseProb for BB1 * TrueProb for TmpBB.
01473     // Another choice is to assume TrueProb for BB1 equals to TrueProb for
01474     // TmpBB, but the math is more complicated.
01475 
01476     uint64_t NewTrueWeight = TWeight;
01477     uint64_t NewFalseWeight = (uint64_t)TWeight + 2 * (uint64_t)FWeight;
01478     ScaleWeights(NewTrueWeight, NewFalseWeight);
01479     // Emit the LHS condition.
01480     FindMergedConditions(BOp->getOperand(0), TBB, TmpBB, CurBB, SwitchBB, Opc,
01481                          NewTrueWeight, NewFalseWeight);
01482 
01483     NewTrueWeight = TWeight;
01484     NewFalseWeight = 2 * (uint64_t)FWeight;
01485     ScaleWeights(NewTrueWeight, NewFalseWeight);
01486     // Emit the RHS condition into TmpBB.
01487     FindMergedConditions(BOp->getOperand(1), TBB, FBB, TmpBB, SwitchBB, Opc,
01488                          NewTrueWeight, NewFalseWeight);
01489   } else {
01490     assert(Opc == Instruction::And && "Unknown merge op!");
01491     // Codegen X & Y as:
01492     // BB1:
01493     //   jmp_if_X TmpBB
01494     //   jmp FBB
01495     // TmpBB:
01496     //   jmp_if_Y TBB
01497     //   jmp FBB
01498     //
01499     //  This requires creation of TmpBB after CurBB.
01500 
01501     // We have flexibility in setting Prob for BB1 and Prob for TmpBB.
01502     // The requirement is that
01503     //   FalseProb for BB1 + (TrueProb for BB1 * FalseProb for TmpBB)
01504     //     = FalseProb for orignal BB.
01505     // Assuming the orignal weights are A and B, one choice is to set BB1's
01506     // weights to 2A+B and B, and set TmpBB's weights to 2A and B. This choice
01507     // assumes that
01508     //   FalseProb for BB1 == TrueProb for BB1 * FalseProb for TmpBB.
01509 
01510     uint64_t NewTrueWeight = 2 * (uint64_t)TWeight + (uint64_t)FWeight;
01511     uint64_t NewFalseWeight = FWeight;
01512     ScaleWeights(NewTrueWeight, NewFalseWeight);
01513     // Emit the LHS condition.
01514     FindMergedConditions(BOp->getOperand(0), TmpBB, FBB, CurBB, SwitchBB, Opc,
01515                          NewTrueWeight, NewFalseWeight);
01516 
01517     NewTrueWeight = 2 * (uint64_t)TWeight;
01518     NewFalseWeight = FWeight;
01519     ScaleWeights(NewTrueWeight, NewFalseWeight);
01520     // Emit the RHS condition into TmpBB.
01521     FindMergedConditions(BOp->getOperand(1), TBB, FBB, TmpBB, SwitchBB, Opc,
01522                          NewTrueWeight, NewFalseWeight);
01523   }
01524 }
01525 
01526 /// If the set of cases should be emitted as a series of branches, return true.
01527 /// If we should emit this as a bunch of and/or'd together conditions, return
01528 /// false.
01529 bool
01530 SelectionDAGBuilder::ShouldEmitAsBranches(const std::vector<CaseBlock> &Cases) {
01531   if (Cases.size() != 2) return true;
01532 
01533   // If this is two comparisons of the same values or'd or and'd together, they
01534   // will get folded into a single comparison, so don't emit two blocks.
01535   if ((Cases[0].CmpLHS == Cases[1].CmpLHS &&
01536        Cases[0].CmpRHS == Cases[1].CmpRHS) ||
01537       (Cases[0].CmpRHS == Cases[1].CmpLHS &&
01538        Cases[0].CmpLHS == Cases[1].CmpRHS)) {
01539     return false;
01540   }
01541 
01542   // Handle: (X != null) | (Y != null) --> (X|Y) != 0
01543   // Handle: (X == null) & (Y == null) --> (X|Y) == 0
01544   if (Cases[0].CmpRHS == Cases[1].CmpRHS &&
01545       Cases[0].CC == Cases[1].CC &&
01546       isa<Constant>(Cases[0].CmpRHS) &&
01547       cast<Constant>(Cases[0].CmpRHS)->isNullValue()) {
01548     if (Cases[0].CC == ISD::SETEQ && Cases[0].TrueBB == Cases[1].ThisBB)
01549       return false;
01550     if (Cases[0].CC == ISD::SETNE && Cases[0].FalseBB == Cases[1].ThisBB)
01551       return false;
01552   }
01553 
01554   return true;
01555 }
01556 
01557 void SelectionDAGBuilder::visitBr(const BranchInst &I) {
01558   MachineBasicBlock *BrMBB = FuncInfo.MBB;
01559 
01560   // Update machine-CFG edges.
01561   MachineBasicBlock *Succ0MBB = FuncInfo.MBBMap[I.getSuccessor(0)];
01562 
01563   // Figure out which block is immediately after the current one.
01564   MachineBasicBlock *NextBlock = nullptr;
01565   MachineFunction::iterator BBI = BrMBB;
01566   if (++BBI != FuncInfo.MF->end())
01567     NextBlock = BBI;
01568 
01569   if (I.isUnconditional()) {
01570     // Update machine-CFG edges.
01571     BrMBB->addSuccessor(Succ0MBB);
01572 
01573     // If this is not a fall-through branch or optimizations are switched off,
01574     // emit the branch.
01575     if (Succ0MBB != NextBlock || TM.getOptLevel() == CodeGenOpt::None)
01576       DAG.setRoot(DAG.getNode(ISD::BR, getCurSDLoc(),
01577                               MVT::Other, getControlRoot(),
01578                               DAG.getBasicBlock(Succ0MBB)));
01579 
01580     return;
01581   }
01582 
01583   // If this condition is one of the special cases we handle, do special stuff
01584   // now.
01585   const Value *CondVal = I.getCondition();
01586   MachineBasicBlock *Succ1MBB = FuncInfo.MBBMap[I.getSuccessor(1)];
01587 
01588   // If this is a series of conditions that are or'd or and'd together, emit
01589   // this as a sequence of branches instead of setcc's with and/or operations.
01590   // As long as jumps are not expensive, this should improve performance.
01591   // For example, instead of something like:
01592   //     cmp A, B
01593   //     C = seteq
01594   //     cmp D, E
01595   //     F = setle
01596   //     or C, F
01597   //     jnz foo
01598   // Emit:
01599   //     cmp A, B
01600   //     je foo
01601   //     cmp D, E
01602   //     jle foo
01603   //
01604   if (const BinaryOperator *BOp = dyn_cast<BinaryOperator>(CondVal)) {
01605     if (!TM.getSubtargetImpl()->getTargetLowering()->isJumpExpensive() &&
01606         BOp->hasOneUse() && (BOp->getOpcode() == Instruction::And ||
01607                              BOp->getOpcode() == Instruction::Or)) {
01608       FindMergedConditions(BOp, Succ0MBB, Succ1MBB, BrMBB, BrMBB,
01609                            BOp->getOpcode(), getEdgeWeight(BrMBB, Succ0MBB),
01610                            getEdgeWeight(BrMBB, Succ1MBB));
01611       // If the compares in later blocks need to use values not currently
01612       // exported from this block, export them now.  This block should always
01613       // be the first entry.
01614       assert(SwitchCases[0].ThisBB == BrMBB && "Unexpected lowering!");
01615 
01616       // Allow some cases to be rejected.
01617       if (ShouldEmitAsBranches(SwitchCases)) {
01618         for (unsigned i = 1, e = SwitchCases.size(); i != e; ++i) {
01619           ExportFromCurrentBlock(SwitchCases[i].CmpLHS);
01620           ExportFromCurrentBlock(SwitchCases[i].CmpRHS);
01621         }
01622 
01623         // Emit the branch for this block.
01624         visitSwitchCase(SwitchCases[0], BrMBB);
01625         SwitchCases.erase(SwitchCases.begin());
01626         return;
01627       }
01628 
01629       // Okay, we decided not to do this, remove any inserted MBB's and clear
01630       // SwitchCases.
01631       for (unsigned i = 1, e = SwitchCases.size(); i != e; ++i)
01632         FuncInfo.MF->erase(SwitchCases[i].ThisBB);
01633 
01634       SwitchCases.clear();
01635     }
01636   }
01637 
01638   // Create a CaseBlock record representing this branch.
01639   CaseBlock CB(ISD::SETEQ, CondVal, ConstantInt::getTrue(*DAG.getContext()),
01640                nullptr, Succ0MBB, Succ1MBB, BrMBB);
01641 
01642   // Use visitSwitchCase to actually insert the fast branch sequence for this
01643   // cond branch.
01644   visitSwitchCase(CB, BrMBB);
01645 }
01646 
01647 /// visitSwitchCase - Emits the necessary code to represent a single node in
01648 /// the binary search tree resulting from lowering a switch instruction.
01649 void SelectionDAGBuilder::visitSwitchCase(CaseBlock &CB,
01650                                           MachineBasicBlock *SwitchBB) {
01651   SDValue Cond;
01652   SDValue CondLHS = getValue(CB.CmpLHS);
01653   SDLoc dl = getCurSDLoc();
01654 
01655   // Build the setcc now.
01656   if (!CB.CmpMHS) {
01657     // Fold "(X == true)" to X and "(X == false)" to !X to
01658     // handle common cases produced by branch lowering.
01659     if (CB.CmpRHS == ConstantInt::getTrue(*DAG.getContext()) &&
01660         CB.CC == ISD::SETEQ)
01661       Cond = CondLHS;
01662     else if (CB.CmpRHS == ConstantInt::getFalse(*DAG.getContext()) &&
01663              CB.CC == ISD::SETEQ) {
01664       SDValue True = DAG.getConstant(1, CondLHS.getValueType());
01665       Cond = DAG.getNode(ISD::XOR, dl, CondLHS.getValueType(), CondLHS, True);
01666     } else
01667       Cond = DAG.getSetCC(dl, MVT::i1, CondLHS, getValue(CB.CmpRHS), CB.CC);
01668   } else {
01669     assert(CB.CC == ISD::SETLE && "Can handle only LE ranges now");
01670 
01671     const APInt& Low = cast<ConstantInt>(CB.CmpLHS)->getValue();
01672     const APInt& High  = cast<ConstantInt>(CB.CmpRHS)->getValue();
01673 
01674     SDValue CmpOp = getValue(CB.CmpMHS);
01675     EVT VT = CmpOp.getValueType();
01676 
01677     if (cast<ConstantInt>(CB.CmpLHS)->isMinValue(true)) {
01678       Cond = DAG.getSetCC(dl, MVT::i1, CmpOp, DAG.getConstant(High, VT),
01679                           ISD::SETLE);
01680     } else {
01681       SDValue SUB = DAG.getNode(ISD::SUB, dl,
01682                                 VT, CmpOp, DAG.getConstant(Low, VT));
01683       Cond = DAG.getSetCC(dl, MVT::i1, SUB,
01684                           DAG.getConstant(High-Low, VT), ISD::SETULE);
01685     }
01686   }
01687 
01688   // Update successor info
01689   addSuccessorWithWeight(SwitchBB, CB.TrueBB, CB.TrueWeight);
01690   // TrueBB and FalseBB are always different unless the incoming IR is
01691   // degenerate. This only happens when running llc on weird IR.
01692   if (CB.TrueBB != CB.FalseBB)
01693     addSuccessorWithWeight(SwitchBB, CB.FalseBB, CB.FalseWeight);
01694 
01695   // Set NextBlock to be the MBB immediately after the current one, if any.
01696   // This is used to avoid emitting unnecessary branches to the next block.
01697   MachineBasicBlock *NextBlock = nullptr;
01698   MachineFunction::iterator BBI = SwitchBB;
01699   if (++BBI != FuncInfo.MF->end())
01700     NextBlock = BBI;
01701 
01702   // If the lhs block is the next block, invert the condition so that we can
01703   // fall through to the lhs instead of the rhs block.
01704   if (CB.TrueBB == NextBlock) {
01705     std::swap(CB.TrueBB, CB.FalseBB);
01706     SDValue True = DAG.getConstant(1, Cond.getValueType());
01707     Cond = DAG.getNode(ISD::XOR, dl, Cond.getValueType(), Cond, True);
01708   }
01709 
01710   SDValue BrCond = DAG.getNode(ISD::BRCOND, dl,
01711                                MVT::Other, getControlRoot(), Cond,
01712                                DAG.getBasicBlock(CB.TrueBB));
01713 
01714   // Insert the false branch. Do this even if it's a fall through branch,
01715   // this makes it easier to do DAG optimizations which require inverting
01716   // the branch condition.
01717   BrCond = DAG.getNode(ISD::BR, dl, MVT::Other, BrCond,
01718                        DAG.getBasicBlock(CB.FalseBB));
01719 
01720   DAG.setRoot(BrCond);
01721 }
01722 
01723 /// visitJumpTable - Emit JumpTable node in the current MBB
01724 void SelectionDAGBuilder::visitJumpTable(JumpTable &JT) {
01725   // Emit the code for the jump table
01726   assert(JT.Reg != -1U && "Should lower JT Header first!");
01727   EVT PTy = TM.getSubtargetImpl()->getTargetLowering()->getPointerTy();
01728   SDValue Index = DAG.getCopyFromReg(getControlRoot(), getCurSDLoc(),
01729                                      JT.Reg, PTy);
01730   SDValue Table = DAG.getJumpTable(JT.JTI, PTy);
01731   SDValue BrJumpTable = DAG.getNode(ISD::BR_JT, getCurSDLoc(),
01732                                     MVT::Other, Index.getValue(1),
01733                                     Table, Index);
01734   DAG.setRoot(BrJumpTable);
01735 }
01736 
01737 /// visitJumpTableHeader - This function emits necessary code to produce index
01738 /// in the JumpTable from switch case.
01739 void SelectionDAGBuilder::visitJumpTableHeader(JumpTable &JT,
01740                                                JumpTableHeader &JTH,
01741                                                MachineBasicBlock *SwitchBB) {
01742   // Subtract the lowest switch case value from the value being switched on and
01743   // conditional branch to default mbb if the result is greater than the
01744   // difference between smallest and largest cases.
01745   SDValue SwitchOp = getValue(JTH.SValue);
01746   EVT VT = SwitchOp.getValueType();
01747   SDValue Sub = DAG.getNode(ISD::SUB, getCurSDLoc(), VT, SwitchOp,
01748                             DAG.getConstant(JTH.First, VT));
01749 
01750   // The SDNode we just created, which holds the value being switched on minus
01751   // the smallest case value, needs to be copied to a virtual register so it
01752   // can be used as an index into the jump table in a subsequent basic block.
01753   // This value may be smaller or larger than the target's pointer type, and
01754   // therefore require extension or truncating.
01755   const TargetLowering *TLI = TM.getSubtargetImpl()->getTargetLowering();
01756   SwitchOp = DAG.getZExtOrTrunc(Sub, getCurSDLoc(), TLI->getPointerTy());
01757 
01758   unsigned JumpTableReg = FuncInfo.CreateReg(TLI->getPointerTy());
01759   SDValue CopyTo = DAG.getCopyToReg(getControlRoot(), getCurSDLoc(),
01760                                     JumpTableReg, SwitchOp);
01761   JT.Reg = JumpTableReg;
01762 
01763   // Emit the range check for the jump table, and branch to the default block
01764   // for the switch statement if the value being switched on exceeds the largest
01765   // case in the switch.
01766   SDValue CMP = DAG.getSetCC(getCurSDLoc(),
01767                              TLI->getSetCCResultType(*DAG.getContext(),
01768                                                      Sub.getValueType()),
01769                              Sub,
01770                              DAG.getConstant(JTH.Last - JTH.First,VT),
01771                              ISD::SETUGT);
01772 
01773   // Set NextBlock to be the MBB immediately after the current one, if any.
01774   // This is used to avoid emitting unnecessary branches to the next block.
01775   MachineBasicBlock *NextBlock = nullptr;
01776   MachineFunction::iterator BBI = SwitchBB;
01777 
01778   if (++BBI != FuncInfo.MF->end())
01779     NextBlock = BBI;
01780 
01781   SDValue BrCond = DAG.getNode(ISD::BRCOND, getCurSDLoc(),
01782                                MVT::Other, CopyTo, CMP,
01783                                DAG.getBasicBlock(JT.Default));
01784 
01785   if (JT.MBB != NextBlock)
01786     BrCond = DAG.getNode(ISD::BR, getCurSDLoc(), MVT::Other, BrCond,
01787                          DAG.getBasicBlock(JT.MBB));
01788 
01789   DAG.setRoot(BrCond);
01790 }
01791 
01792 /// Codegen a new tail for a stack protector check ParentMBB which has had its
01793 /// tail spliced into a stack protector check success bb.
01794 ///
01795 /// For a high level explanation of how this fits into the stack protector
01796 /// generation see the comment on the declaration of class
01797 /// StackProtectorDescriptor.
01798 void SelectionDAGBuilder::visitSPDescriptorParent(StackProtectorDescriptor &SPD,
01799                                                   MachineBasicBlock *ParentBB) {
01800 
01801   // First create the loads to the guard/stack slot for the comparison.
01802   const TargetLowering *TLI = TM.getSubtargetImpl()->getTargetLowering();
01803   EVT PtrTy = TLI->getPointerTy();
01804 
01805   MachineFrameInfo *MFI = ParentBB->getParent()->getFrameInfo();
01806   int FI = MFI->getStackProtectorIndex();
01807 
01808   const Value *IRGuard = SPD.getGuard();
01809   SDValue GuardPtr = getValue(IRGuard);
01810   SDValue StackSlotPtr = DAG.getFrameIndex(FI, PtrTy);
01811 
01812   unsigned Align =
01813     TLI->getDataLayout()->getPrefTypeAlignment(IRGuard->getType());
01814 
01815   SDValue Guard;
01816 
01817   // If GuardReg is set and useLoadStackGuardNode returns true, retrieve the
01818   // guard value from the virtual register holding the value. Otherwise, emit a
01819   // volatile load to retrieve the stack guard value.
01820   unsigned GuardReg = SPD.getGuardReg();
01821 
01822   if (GuardReg && TLI->useLoadStackGuardNode())
01823     Guard = DAG.getCopyFromReg(DAG.getEntryNode(), getCurSDLoc(), GuardReg,
01824                                PtrTy);
01825   else
01826     Guard = DAG.getLoad(PtrTy, getCurSDLoc(), DAG.getEntryNode(),
01827                         GuardPtr, MachinePointerInfo(IRGuard, 0),
01828                         true, false, false, Align);
01829 
01830   SDValue StackSlot = DAG.getLoad(PtrTy, getCurSDLoc(), DAG.getEntryNode(),
01831                                   StackSlotPtr,
01832                                   MachinePointerInfo::getFixedStack(FI),
01833                                   true, false, false, Align);
01834 
01835   // Perform the comparison via a subtract/getsetcc.
01836   EVT VT = Guard.getValueType();
01837   SDValue Sub = DAG.getNode(ISD::SUB, getCurSDLoc(), VT, Guard, StackSlot);
01838 
01839   SDValue Cmp = DAG.getSetCC(getCurSDLoc(),
01840                              TLI->getSetCCResultType(*DAG.getContext(),
01841                                                      Sub.getValueType()),
01842                              Sub, DAG.getConstant(0, VT),
01843                              ISD::SETNE);
01844 
01845   // If the sub is not 0, then we know the guard/stackslot do not equal, so
01846   // branch to failure MBB.
01847   SDValue BrCond = DAG.getNode(ISD::BRCOND, getCurSDLoc(),
01848                                MVT::Other, StackSlot.getOperand(0),
01849                                Cmp, DAG.getBasicBlock(SPD.getFailureMBB()));
01850   // Otherwise branch to success MBB.
01851   SDValue Br = DAG.getNode(ISD::BR, getCurSDLoc(),
01852                            MVT::Other, BrCond,
01853                            DAG.getBasicBlock(SPD.getSuccessMBB()));
01854 
01855   DAG.setRoot(Br);
01856 }
01857 
01858 /// Codegen the failure basic block for a stack protector check.
01859 ///
01860 /// A failure stack protector machine basic block consists simply of a call to
01861 /// __stack_chk_fail().
01862 ///
01863 /// For a high level explanation of how this fits into the stack protector
01864 /// generation see the comment on the declaration of class
01865 /// StackProtectorDescriptor.
01866 void
01867 SelectionDAGBuilder::visitSPDescriptorFailure(StackProtectorDescriptor &SPD) {
01868   const TargetLowering *TLI = TM.getSubtargetImpl()->getTargetLowering();
01869   SDValue Chain = TLI->makeLibCall(DAG, RTLIB::STACKPROTECTOR_CHECK_FAIL,
01870                                    MVT::isVoid, nullptr, 0, false,
01871                                    getCurSDLoc(), false, false).second;
01872   DAG.setRoot(Chain);
01873 }
01874 
01875 /// visitBitTestHeader - This function emits necessary code to produce value
01876 /// suitable for "bit tests"
01877 void SelectionDAGBuilder::visitBitTestHeader(BitTestBlock &B,
01878                                              MachineBasicBlock *SwitchBB) {
01879   // Subtract the minimum value
01880   SDValue SwitchOp = getValue(B.SValue);
01881   EVT VT = SwitchOp.getValueType();
01882   SDValue Sub = DAG.getNode(ISD::SUB, getCurSDLoc(), VT, SwitchOp,
01883                             DAG.getConstant(B.First, VT));
01884 
01885   // Check range
01886   const TargetLowering *TLI = TM.getSubtargetImpl()->getTargetLowering();
01887   SDValue RangeCmp = DAG.getSetCC(getCurSDLoc(),
01888                                   TLI->getSetCCResultType(*DAG.getContext(),
01889                                                          Sub.getValueType()),
01890                                   Sub, DAG.getConstant(B.Range, VT),
01891                                   ISD::SETUGT);
01892 
01893   // Determine the type of the test operands.
01894   bool UsePtrType = false;
01895   if (!TLI->isTypeLegal(VT))
01896     UsePtrType = true;
01897   else {
01898     for (unsigned i = 0, e = B.Cases.size(); i != e; ++i)
01899       if (!isUIntN(VT.getSizeInBits(), B.Cases[i].Mask)) {
01900         // Switch table case range are encoded into series of masks.
01901         // Just use pointer type, it's guaranteed to fit.
01902         UsePtrType = true;
01903         break;
01904       }
01905   }
01906   if (UsePtrType) {
01907     VT = TLI->getPointerTy();
01908     Sub = DAG.getZExtOrTrunc(Sub, getCurSDLoc(), VT);
01909   }
01910 
01911   B.RegVT = VT.getSimpleVT();
01912   B.Reg = FuncInfo.CreateReg(B.RegVT);
01913   SDValue CopyTo = DAG.getCopyToReg(getControlRoot(), getCurSDLoc(),
01914                                     B.Reg, Sub);
01915 
01916   // Set NextBlock to be the MBB immediately after the current one, if any.
01917   // This is used to avoid emitting unnecessary branches to the next block.
01918   MachineBasicBlock *NextBlock = nullptr;
01919   MachineFunction::iterator BBI = SwitchBB;
01920   if (++BBI != FuncInfo.MF->end())
01921     NextBlock = BBI;
01922 
01923   MachineBasicBlock* MBB = B.Cases[0].ThisBB;
01924 
01925   addSuccessorWithWeight(SwitchBB, B.Default);
01926   addSuccessorWithWeight(SwitchBB, MBB);
01927 
01928   SDValue BrRange = DAG.getNode(ISD::BRCOND, getCurSDLoc(),
01929                                 MVT::Other, CopyTo, RangeCmp,
01930                                 DAG.getBasicBlock(B.Default));
01931 
01932   if (MBB != NextBlock)
01933     BrRange = DAG.getNode(ISD::BR, getCurSDLoc(), MVT::Other, CopyTo,
01934                           DAG.getBasicBlock(MBB));
01935 
01936   DAG.setRoot(BrRange);
01937 }
01938 
01939 /// visitBitTestCase - this function produces one "bit test"
01940 void SelectionDAGBuilder::visitBitTestCase(BitTestBlock &BB,
01941                                            MachineBasicBlock* NextMBB,
01942                                            uint32_t BranchWeightToNext,
01943                                            unsigned Reg,
01944                                            BitTestCase &B,
01945                                            MachineBasicBlock *SwitchBB) {
01946   MVT VT = BB.RegVT;
01947   SDValue ShiftOp = DAG.getCopyFromReg(getControlRoot(), getCurSDLoc(),
01948                                        Reg, VT);
01949   SDValue Cmp;
01950   unsigned PopCount = CountPopulation_64(B.Mask);
01951   const TargetLowering *TLI = TM.getSubtargetImpl()->getTargetLowering();
01952   if (PopCount == 1) {
01953     // Testing for a single bit; just compare the shift count with what it
01954     // would need to be to shift a 1 bit in that position.
01955     Cmp = DAG.getSetCC(getCurSDLoc(),
01956                        TLI->getSetCCResultType(*DAG.getContext(), VT),
01957                        ShiftOp,
01958                        DAG.getConstant(countTrailingZeros(B.Mask), VT),
01959                        ISD::SETEQ);
01960   } else if (PopCount == BB.Range) {
01961     // There is only one zero bit in the range, test for it directly.
01962     Cmp = DAG.getSetCC(getCurSDLoc(),
01963                        TLI->getSetCCResultType(*DAG.getContext(), VT),
01964                        ShiftOp,
01965                        DAG.getConstant(CountTrailingOnes_64(B.Mask), VT),
01966                        ISD::SETNE);
01967   } else {
01968     // Make desired shift
01969     SDValue SwitchVal = DAG.getNode(ISD::SHL, getCurSDLoc(), VT,
01970                                     DAG.getConstant(1, VT), ShiftOp);
01971 
01972     // Emit bit tests and jumps
01973     SDValue AndOp = DAG.getNode(ISD::AND, getCurSDLoc(),
01974                                 VT, SwitchVal, DAG.getConstant(B.Mask, VT));
01975     Cmp = DAG.getSetCC(getCurSDLoc(),
01976                        TLI->getSetCCResultType(*DAG.getContext(), VT),
01977                        AndOp, DAG.getConstant(0, VT),
01978                        ISD::SETNE);
01979   }
01980 
01981   // The branch weight from SwitchBB to B.TargetBB is B.ExtraWeight.
01982   addSuccessorWithWeight(SwitchBB, B.TargetBB, B.ExtraWeight);
01983   // The branch weight from SwitchBB to NextMBB is BranchWeightToNext.
01984   addSuccessorWithWeight(SwitchBB, NextMBB, BranchWeightToNext);
01985 
01986   SDValue BrAnd = DAG.getNode(ISD::BRCOND, getCurSDLoc(),
01987                               MVT::Other, getControlRoot(),
01988                               Cmp, DAG.getBasicBlock(B.TargetBB));
01989 
01990   // Set NextBlock to be the MBB immediately after the current one, if any.
01991   // This is used to avoid emitting unnecessary branches to the next block.
01992   MachineBasicBlock *NextBlock = nullptr;
01993   MachineFunction::iterator BBI = SwitchBB;
01994   if (++BBI != FuncInfo.MF->end())
01995     NextBlock = BBI;
01996 
01997   if (NextMBB != NextBlock)
01998     BrAnd = DAG.getNode(ISD::BR, getCurSDLoc(), MVT::Other, BrAnd,
01999                         DAG.getBasicBlock(NextMBB));
02000 
02001   DAG.setRoot(BrAnd);
02002 }
02003 
02004 void SelectionDAGBuilder::visitInvoke(const InvokeInst &I) {
02005   MachineBasicBlock *InvokeMBB = FuncInfo.MBB;
02006 
02007   // Retrieve successors.
02008   MachineBasicBlock *Return = FuncInfo.MBBMap[I.getSuccessor(0)];
02009   MachineBasicBlock *LandingPad = FuncInfo.MBBMap[I.getSuccessor(1)];
02010 
02011   const Value *Callee(I.getCalledValue());
02012   const Function *Fn = dyn_cast<Function>(Callee);
02013   if (isa<InlineAsm>(Callee))
02014     visitInlineAsm(&I);
02015   else if (Fn && Fn->isIntrinsic()) {
02016     assert(Fn->getIntrinsicID() == Intrinsic::donothing);
02017     // Ignore invokes to @llvm.donothing: jump directly to the next BB.
02018   } else
02019     LowerCallTo(&I, getValue(Callee), false, LandingPad);
02020 
02021   // If the value of the invoke is used outside of its defining block, make it
02022   // available as a virtual register.
02023   CopyToExportRegsIfNeeded(&I);
02024 
02025   // Update successor info
02026   addSuccessorWithWeight(InvokeMBB, Return);
02027   addSuccessorWithWeight(InvokeMBB, LandingPad);
02028 
02029   // Drop into normal successor.
02030   DAG.setRoot(DAG.getNode(ISD::BR, getCurSDLoc(),
02031                           MVT::Other, getControlRoot(),
02032                           DAG.getBasicBlock(Return)));
02033 }
02034 
02035 void SelectionDAGBuilder::visitResume(const ResumeInst &RI) {
02036   llvm_unreachable("SelectionDAGBuilder shouldn't visit resume instructions!");
02037 }
02038 
02039 void SelectionDAGBuilder::visitLandingPad(const LandingPadInst &LP) {
02040   assert(FuncInfo.MBB->isLandingPad() &&
02041          "Call to landingpad not in landing pad!");
02042 
02043   MachineBasicBlock *MBB = FuncInfo.MBB;
02044   MachineModuleInfo &MMI = DAG.getMachineFunction().getMMI();
02045   AddLandingPadInfo(LP, MMI, MBB);
02046 
02047   // If there aren't registers to copy the values into (e.g., during SjLj
02048   // exceptions), then don't bother to create these DAG nodes.
02049   const TargetLowering *TLI = TM.getSubtargetImpl()->getTargetLowering();
02050   if (TLI->getExceptionPointerRegister() == 0 &&
02051       TLI->getExceptionSelectorRegister() == 0)
02052     return;
02053 
02054   SmallVector<EVT, 2> ValueVTs;
02055   ComputeValueVTs(*TLI, LP.getType(), ValueVTs);
02056   assert(ValueVTs.size() == 2 && "Only two-valued landingpads are supported");
02057 
02058   // Get the two live-in registers as SDValues. The physregs have already been
02059   // copied into virtual registers.
02060   SDValue Ops[2];
02061   Ops[0] = DAG.getZExtOrTrunc(
02062     DAG.getCopyFromReg(DAG.getEntryNode(), getCurSDLoc(),
02063                        FuncInfo.ExceptionPointerVirtReg, TLI->getPointerTy()),
02064     getCurSDLoc(), ValueVTs[0]);
02065   Ops[1] = DAG.getZExtOrTrunc(
02066     DAG.getCopyFromReg(DAG.getEntryNode(), getCurSDLoc(),
02067                        FuncInfo.ExceptionSelectorVirtReg, TLI->getPointerTy()),
02068     getCurSDLoc(), ValueVTs[1]);
02069 
02070   // Merge into one.
02071   SDValue Res = DAG.getNode(ISD::MERGE_VALUES, getCurSDLoc(),
02072                             DAG.getVTList(ValueVTs), Ops);
02073   setValue(&LP, Res);
02074 }
02075 
02076 /// handleSmallSwitchCaseRange - Emit a series of specific tests (suitable for
02077 /// small case ranges).
02078 bool SelectionDAGBuilder::handleSmallSwitchRange(CaseRec& CR,
02079                                                  CaseRecVector& WorkList,
02080                                                  const Value* SV,
02081                                                  MachineBasicBlock *Default,
02082                                                  MachineBasicBlock *SwitchBB) {
02083   // Size is the number of Cases represented by this range.
02084   size_t Size = CR.Range.second - CR.Range.first;
02085   if (Size > 3)
02086     return false;
02087 
02088   // Get the MachineFunction which holds the current MBB.  This is used when
02089   // inserting any additional MBBs necessary to represent the switch.
02090   MachineFunction *CurMF = FuncInfo.MF;
02091 
02092   // Figure out which block is immediately after the current one.
02093   MachineBasicBlock *NextBlock = nullptr;
02094   MachineFunction::iterator BBI = CR.CaseBB;
02095 
02096   if (++BBI != FuncInfo.MF->end())
02097     NextBlock = BBI;
02098 
02099   BranchProbabilityInfo *BPI = FuncInfo.BPI;
02100   // If any two of the cases has the same destination, and if one value
02101   // is the same as the other, but has one bit unset that the other has set,
02102   // use bit manipulation to do two compares at once.  For example:
02103   // "if (X == 6 || X == 4)" -> "if ((X|2) == 6)"
02104   // TODO: This could be extended to merge any 2 cases in switches with 3 cases.
02105   // TODO: Handle cases where CR.CaseBB != SwitchBB.
02106   if (Size == 2 && CR.CaseBB == SwitchBB) {
02107     Case &Small = *CR.Range.first;
02108     Case &Big = *(CR.Range.second-1);
02109 
02110     if (Small.Low == Small.High && Big.Low == Big.High && Small.BB == Big.BB) {
02111       const APInt& SmallValue = cast<ConstantInt>(Small.Low)->getValue();
02112       const APInt& BigValue = cast<ConstantInt>(Big.Low)->getValue();
02113 
02114       // Check that there is only one bit different.
02115       if (BigValue.countPopulation() == SmallValue.countPopulation() + 1 &&
02116           (SmallValue | BigValue) == BigValue) {
02117         // Isolate the common bit.
02118         APInt CommonBit = BigValue & ~SmallValue;
02119         assert((SmallValue | CommonBit) == BigValue &&
02120                CommonBit.countPopulation() == 1 && "Not a common bit?");
02121 
02122         SDValue CondLHS = getValue(SV);
02123         EVT VT = CondLHS.getValueType();
02124         SDLoc DL = getCurSDLoc();
02125 
02126         SDValue Or = DAG.getNode(ISD::OR, DL, VT, CondLHS,
02127                                  DAG.getConstant(CommonBit, VT));
02128         SDValue Cond = DAG.getSetCC(DL, MVT::i1,
02129                                     Or, DAG.getConstant(BigValue, VT),
02130                                     ISD::SETEQ);
02131 
02132         // Update successor info.
02133         // Both Small and Big will jump to Small.BB, so we sum up the weights.
02134         addSuccessorWithWeight(SwitchBB, Small.BB,
02135                                Small.ExtraWeight + Big.ExtraWeight);
02136         addSuccessorWithWeight(SwitchBB, Default,
02137           // The default destination is the first successor in IR.
02138           BPI ? BPI->getEdgeWeight(SwitchBB->getBasicBlock(), (unsigned)0) : 0);
02139 
02140         // Insert the true branch.
02141         SDValue BrCond = DAG.getNode(ISD::BRCOND, DL, MVT::Other,
02142                                      getControlRoot(), Cond,
02143                                      DAG.getBasicBlock(Small.BB));
02144 
02145         // Insert the false branch.
02146         BrCond = DAG.getNode(ISD::BR, DL, MVT::Other, BrCond,
02147                              DAG.getBasicBlock(Default));
02148 
02149         DAG.setRoot(BrCond);
02150         return true;
02151       }
02152     }
02153   }
02154 
02155   // Order cases by weight so the most likely case will be checked first.
02156   uint32_t UnhandledWeights = 0;
02157   if (BPI) {
02158     for (CaseItr I = CR.Range.first, IE = CR.Range.second; I != IE; ++I) {
02159       uint32_t IWeight = I->ExtraWeight;
02160       UnhandledWeights += IWeight;
02161       for (CaseItr J = CR.Range.first; J < I; ++J) {
02162         uint32_t JWeight = J->ExtraWeight;
02163         if (IWeight > JWeight)
02164           std::swap(*I, *J);
02165       }
02166     }
02167   }
02168   // Rearrange the case blocks so that the last one falls through if possible.
02169   Case &BackCase = *(CR.Range.second-1);
02170   if (Size > 1 &&
02171       NextBlock && Default != NextBlock && BackCase.BB != NextBlock) {
02172     // The last case block won't fall through into 'NextBlock' if we emit the
02173     // branches in this order.  See if rearranging a case value would help.
02174     // We start at the bottom as it's the case with the least weight.
02175     for (Case *I = &*(CR.Range.second-2), *E = &*CR.Range.first-1; I != E; --I)
02176       if (I->BB == NextBlock) {
02177         std::swap(*I, BackCase);
02178         break;
02179       }
02180   }
02181 
02182   // Create a CaseBlock record representing a conditional branch to
02183   // the Case's target mbb if the value being switched on SV is equal
02184   // to C.
02185   MachineBasicBlock *CurBlock = CR.CaseBB;
02186   for (CaseItr I = CR.Range.first, E = CR.Range.second; I != E; ++I) {
02187     MachineBasicBlock *FallThrough;
02188     if (I != E-1) {
02189       FallThrough = CurMF->CreateMachineBasicBlock(CurBlock->getBasicBlock());
02190       CurMF->insert(BBI, FallThrough);
02191 
02192       // Put SV in a virtual register to make it available from the new blocks.
02193       ExportFromCurrentBlock(SV);
02194     } else {
02195       // If the last case doesn't match, go to the default block.
02196       FallThrough = Default;
02197     }
02198 
02199     const Value *RHS, *LHS, *MHS;
02200     ISD::CondCode CC;
02201     if (I->High == I->Low) {
02202       // This is just small small case range :) containing exactly 1 case
02203       CC = ISD::SETEQ;
02204       LHS = SV; RHS = I->High; MHS = nullptr;
02205     } else {
02206       CC = ISD::SETLE;
02207       LHS = I->Low; MHS = SV; RHS = I->High;
02208     }
02209 
02210     // The false weight should be sum of all un-handled cases.
02211     UnhandledWeights -= I->ExtraWeight;
02212     CaseBlock CB(CC, LHS, RHS, MHS, /* truebb */ I->BB, /* falsebb */ FallThrough,
02213                  /* me */ CurBlock,
02214                  /* trueweight */ I->ExtraWeight,
02215                  /* falseweight */ UnhandledWeights);
02216 
02217     // If emitting the first comparison, just call visitSwitchCase to emit the
02218     // code into the current block.  Otherwise, push the CaseBlock onto the
02219     // vector to be later processed by SDISel, and insert the node's MBB
02220     // before the next MBB.
02221     if (CurBlock == SwitchBB)
02222       visitSwitchCase(CB, SwitchBB);
02223     else
02224       SwitchCases.push_back(CB);
02225 
02226     CurBlock = FallThrough;
02227   }
02228 
02229   return true;
02230 }
02231 
02232 static inline bool areJTsAllowed(const TargetLowering &TLI) {
02233   return TLI.supportJumpTables() &&
02234           (TLI.isOperationLegalOrCustom(ISD::BR_JT, MVT::Other) ||
02235            TLI.isOperationLegalOrCustom(ISD::BRIND, MVT::Other));
02236 }
02237 
02238 static APInt ComputeRange(const APInt &First, const APInt &Last) {
02239   uint32_t BitWidth = std::max(Last.getBitWidth(), First.getBitWidth()) + 1;
02240   APInt LastExt = Last.sext(BitWidth), FirstExt = First.sext(BitWidth);
02241   return (LastExt - FirstExt + 1ULL);
02242 }
02243 
02244 /// handleJTSwitchCase - Emit jumptable for current switch case range
02245 bool SelectionDAGBuilder::handleJTSwitchCase(CaseRec &CR,
02246                                              CaseRecVector &WorkList,
02247                                              const Value *SV,
02248                                              MachineBasicBlock *Default,
02249                                              MachineBasicBlock *SwitchBB) {
02250   Case& FrontCase = *CR.Range.first;
02251   Case& BackCase  = *(CR.Range.second-1);
02252 
02253   const APInt &First = cast<ConstantInt>(FrontCase.Low)->getValue();
02254   const APInt &Last  = cast<ConstantInt>(BackCase.High)->getValue();
02255 
02256   APInt TSize(First.getBitWidth(), 0);
02257   for (CaseItr I = CR.Range.first, E = CR.Range.second; I != E; ++I)
02258     TSize += I->size();
02259 
02260   const TargetLowering *TLI = TM.getSubtargetImpl()->getTargetLowering();
02261   if (!areJTsAllowed(*TLI) || TSize.ult(TLI->getMinimumJumpTableEntries()))
02262     return false;
02263 
02264   APInt Range = ComputeRange(First, Last);
02265   // The density is TSize / Range. Require at least 40%.
02266   // It should not be possible for IntTSize to saturate for sane code, but make
02267   // sure we handle Range saturation correctly.
02268   uint64_t IntRange = Range.getLimitedValue(UINT64_MAX/10);
02269   uint64_t IntTSize = TSize.getLimitedValue(UINT64_MAX/10);
02270   if (IntTSize * 10 < IntRange * 4)
02271     return false;
02272 
02273   DEBUG(dbgs() << "Lowering jump table\n"
02274                << "First entry: " << First << ". Last entry: " << Last << '\n'
02275                << "Range: " << Range << ". Size: " << TSize << ".\n\n");
02276 
02277   // Get the MachineFunction which holds the current MBB.  This is used when
02278   // inserting any additional MBBs necessary to represent the switch.
02279   MachineFunction *CurMF = FuncInfo.MF;
02280 
02281   // Figure out which block is immediately after the current one.
02282   MachineFunction::iterator BBI = CR.CaseBB;
02283   ++BBI;
02284 
02285   const BasicBlock *LLVMBB = CR.CaseBB->getBasicBlock();
02286 
02287   // Create a new basic block to hold the code for loading the address
02288   // of the jump table, and jumping to it.  Update successor information;
02289   // we will either branch to the default case for the switch, or the jump
02290   // table.
02291   MachineBasicBlock *JumpTableBB = CurMF->CreateMachineBasicBlock(LLVMBB);
02292   CurMF->insert(BBI, JumpTableBB);
02293 
02294   addSuccessorWithWeight(CR.CaseBB, Default);
02295   addSuccessorWithWeight(CR.CaseBB, JumpTableBB);
02296 
02297   // Build a vector of destination BBs, corresponding to each target
02298   // of the jump table. If the value of the jump table slot corresponds to
02299   // a case statement, push the case's BB onto the vector, otherwise, push
02300   // the default BB.
02301   std::vector<MachineBasicBlock*> DestBBs;
02302   APInt TEI = First;
02303   for (CaseItr I = CR.Range.first, E = CR.Range.second; I != E; ++TEI) {
02304     const APInt &Low = cast<ConstantInt>(I->Low)->getValue();
02305     const APInt &High = cast<ConstantInt>(I->High)->getValue();
02306 
02307     if (Low.sle(TEI) && TEI.sle(High)) {
02308       DestBBs.push_back(I->BB);
02309       if (TEI==High)
02310         ++I;
02311     } else {
02312       DestBBs.push_back(Default);
02313     }
02314   }
02315 
02316   // Calculate weight for each unique destination in CR.
02317   DenseMap<MachineBasicBlock*, uint32_t> DestWeights;
02318   if (FuncInfo.BPI)
02319     for (CaseItr I = CR.Range.first, E = CR.Range.second; I != E; ++I) {
02320       DenseMap<MachineBasicBlock*, uint32_t>::iterator Itr =
02321           DestWeights.find(I->BB);
02322       if (Itr != DestWeights.end())
02323         Itr->second += I->ExtraWeight;
02324       else
02325         DestWeights[I->BB] = I->ExtraWeight;
02326     }
02327 
02328   // Update successor info. Add one edge to each unique successor.
02329   BitVector SuccsHandled(CR.CaseBB->getParent()->getNumBlockIDs());
02330   for (std::vector<MachineBasicBlock*>::iterator I = DestBBs.begin(),
02331          E = DestBBs.end(); I != E; ++I) {
02332     if (!SuccsHandled[(*I)->getNumber()]) {
02333       SuccsHandled[(*I)->getNumber()] = true;
02334       DenseMap<MachineBasicBlock*, uint32_t>::iterator Itr =
02335           DestWeights.find(*I);
02336       addSuccessorWithWeight(JumpTableBB, *I,
02337                              Itr != DestWeights.end() ? Itr->second : 0);
02338     }
02339   }
02340 
02341   // Create a jump table index for this jump table.
02342   unsigned JTEncoding = TLI->getJumpTableEncoding();
02343   unsigned JTI = CurMF->getOrCreateJumpTableInfo(JTEncoding)
02344                        ->createJumpTableIndex(DestBBs);
02345 
02346   // Set the jump table information so that we can codegen it as a second
02347   // MachineBasicBlock
02348   JumpTable JT(-1U, JTI, JumpTableBB, Default);
02349   JumpTableHeader JTH(First, Last, SV, CR.CaseBB, (CR.CaseBB == SwitchBB));
02350   if (CR.CaseBB == SwitchBB)
02351     visitJumpTableHeader(JT, JTH, SwitchBB);
02352 
02353   JTCases.push_back(JumpTableBlock(JTH, JT));
02354   return true;
02355 }
02356 
02357 /// handleBTSplitSwitchCase - emit comparison and split binary search tree into
02358 /// 2 subtrees.
02359 bool SelectionDAGBuilder::handleBTSplitSwitchCase(CaseRec& CR,
02360                                                   CaseRecVector& WorkList,
02361                                                   const Value* SV,
02362                                                   MachineBasicBlock* Default,
02363                                                   MachineBasicBlock* SwitchBB) {
02364   // Get the MachineFunction which holds the current MBB.  This is used when
02365   // inserting any additional MBBs necessary to represent the switch.
02366   MachineFunction *CurMF = FuncInfo.MF;
02367 
02368   // Figure out which block is immediately after the current one.
02369   MachineFunction::iterator BBI = CR.CaseBB;
02370   ++BBI;
02371 
02372   Case& FrontCase = *CR.Range.first;
02373   Case& BackCase  = *(CR.Range.second-1);
02374   const BasicBlock *LLVMBB = CR.CaseBB->getBasicBlock();
02375 
02376   // Size is the number of Cases represented by this range.
02377   unsigned Size = CR.Range.second - CR.Range.first;
02378 
02379   const APInt &First = cast<ConstantInt>(FrontCase.Low)->getValue();
02380   const APInt &Last  = cast<ConstantInt>(BackCase.High)->getValue();
02381   double FMetric = 0;
02382   CaseItr Pivot = CR.Range.first + Size/2;
02383 
02384   // Select optimal pivot, maximizing sum density of LHS and RHS. This will
02385   // (heuristically) allow us to emit JumpTable's later.
02386   APInt TSize(First.getBitWidth(), 0);
02387   for (CaseItr I = CR.Range.first, E = CR.Range.second;
02388        I!=E; ++I)
02389     TSize += I->size();
02390 
02391   APInt LSize = FrontCase.size();
02392   APInt RSize = TSize-LSize;
02393   DEBUG(dbgs() << "Selecting best pivot: \n"
02394                << "First: " << First << ", Last: " << Last <<'\n'
02395                << "LSize: " << LSize << ", RSize: " << RSize << '\n');
02396   for (CaseItr I = CR.Range.first, J=I+1, E = CR.Range.second;
02397        J!=E; ++I, ++J) {
02398     const APInt &LEnd = cast<ConstantInt>(I->High)->getValue();
02399     const APInt &RBegin = cast<ConstantInt>(J->Low)->getValue();
02400     APInt Range = ComputeRange(LEnd, RBegin);
02401     assert((Range - 2ULL).isNonNegative() &&
02402            "Invalid case distance");
02403     // Use volatile double here to avoid excess precision issues on some hosts,
02404     // e.g. that use 80-bit X87 registers.
02405     volatile double LDensity =
02406        (double)LSize.roundToDouble() /
02407                            (LEnd - First + 1ULL).roundToDouble();
02408     volatile double RDensity =
02409       (double)RSize.roundToDouble() /
02410                            (Last - RBegin + 1ULL).roundToDouble();
02411     volatile double Metric = Range.logBase2()*(LDensity+RDensity);
02412     // Should always split in some non-trivial place
02413     DEBUG(dbgs() <<"=>Step\n"
02414                  << "LEnd: " << LEnd << ", RBegin: " << RBegin << '\n'
02415                  << "LDensity: " << LDensity
02416                  << ", RDensity: " << RDensity << '\n'
02417                  << "Metric: " << Metric << '\n');
02418     if (FMetric < Metric) {
02419       Pivot = J;
02420       FMetric = Metric;
02421       DEBUG(dbgs() << "Current metric set to: " << FMetric << '\n');
02422     }
02423 
02424     LSize += J->size();
02425     RSize -= J->size();
02426   }
02427 
02428   const TargetLowering *TLI = TM.getSubtargetImpl()->getTargetLowering();
02429   if (areJTsAllowed(*TLI)) {
02430     // If our case is dense we *really* should handle it earlier!
02431     assert((FMetric > 0) && "Should handle dense range earlier!");
02432   } else {
02433     Pivot = CR.Range.first + Size/2;
02434   }
02435 
02436   CaseRange LHSR(CR.Range.first, Pivot);
02437   CaseRange RHSR(Pivot, CR.Range.second);
02438   const Constant *C = Pivot->Low;
02439   MachineBasicBlock *FalseBB = nullptr, *TrueBB = nullptr;
02440 
02441   // We know that we branch to the LHS if the Value being switched on is
02442   // less than the Pivot value, C.  We use this to optimize our binary
02443   // tree a bit, by recognizing that if SV is greater than or equal to the
02444   // LHS's Case Value, and that Case Value is exactly one less than the
02445   // Pivot's Value, then we can branch directly to the LHS's Target,
02446   // rather than creating a leaf node for it.
02447   if ((LHSR.second - LHSR.first) == 1 &&
02448       LHSR.first->High == CR.GE &&
02449       cast<ConstantInt>(C)->getValue() ==
02450       (cast<ConstantInt>(CR.GE)->getValue() + 1LL)) {
02451     TrueBB = LHSR.first->BB;
02452   } else {
02453     TrueBB = CurMF->CreateMachineBasicBlock(LLVMBB);
02454     CurMF->insert(BBI, TrueBB);
02455     WorkList.push_back(CaseRec(TrueBB, C, CR.GE, LHSR));
02456 
02457     // Put SV in a virtual register to make it available from the new blocks.
02458     ExportFromCurrentBlock(SV);
02459   }
02460 
02461   // Similar to the optimization above, if the Value being switched on is
02462   // known to be less than the Constant CR.LT, and the current Case Value
02463   // is CR.LT - 1, then we can branch directly to the target block for
02464   // the current Case Value, rather than emitting a RHS leaf node for it.
02465   if ((RHSR.second - RHSR.first) == 1 && CR.LT &&
02466       cast<ConstantInt>(RHSR.first->Low)->getValue() ==
02467       (cast<ConstantInt>(CR.LT)->getValue() - 1LL)) {
02468     FalseBB = RHSR.first->BB;
02469   } else {
02470     FalseBB = CurMF->CreateMachineBasicBlock(LLVMBB);
02471     CurMF->insert(BBI, FalseBB);
02472     WorkList.push_back(CaseRec(FalseBB,CR.LT,C,RHSR));
02473 
02474     // Put SV in a virtual register to make it available from the new blocks.
02475     ExportFromCurrentBlock(SV);
02476   }
02477 
02478   // Create a CaseBlock record representing a conditional branch to
02479   // the LHS node if the value being switched on SV is less than C.
02480   // Otherwise, branch to LHS.
02481   CaseBlock CB(ISD::SETLT, SV, C, nullptr, TrueBB, FalseBB, CR.CaseBB);
02482 
02483   if (CR.CaseBB == SwitchBB)
02484     visitSwitchCase(CB, SwitchBB);
02485   else
02486     SwitchCases.push_back(CB);
02487 
02488   return true;
02489 }
02490 
02491 /// handleBitTestsSwitchCase - if current case range has few destination and
02492 /// range span less, than machine word bitwidth, encode case range into series
02493 /// of masks and emit bit tests with these masks.
02494 bool SelectionDAGBuilder::handleBitTestsSwitchCase(CaseRec& CR,
02495                                                    CaseRecVector& WorkList,
02496                                                    const Value* SV,
02497                                                    MachineBasicBlock* Default,
02498                                                    MachineBasicBlock* SwitchBB) {
02499   const TargetLowering *TLI = TM.getSubtargetImpl()->getTargetLowering();
02500   EVT PTy = TLI->getPointerTy();
02501   unsigned IntPtrBits = PTy.getSizeInBits();
02502 
02503   Case& FrontCase = *CR.Range.first;
02504   Case& BackCase  = *(CR.Range.second-1);
02505 
02506   // Get the MachineFunction which holds the current MBB.  This is used when
02507   // inserting any additional MBBs necessary to represent the switch.
02508   MachineFunction *CurMF = FuncInfo.MF;
02509 
02510   // If target does not have legal shift left, do not emit bit tests at all.
02511   if (!TLI->isOperationLegal(ISD::SHL, PTy))
02512     return false;
02513 
02514   size_t numCmps = 0;
02515   for (CaseItr I = CR.Range.first, E = CR.Range.second;
02516        I!=E; ++I) {
02517     // Single case counts one, case range - two.
02518     numCmps += (I->Low == I->High ? 1 : 2);
02519   }
02520 
02521   // Count unique destinations
02522   SmallSet<MachineBasicBlock*, 4> Dests;
02523   for (CaseItr I = CR.Range.first, E = CR.Range.second; I!=E; ++I) {
02524     Dests.insert(I->BB);
02525     if (Dests.size() > 3)
02526       // Don't bother the code below, if there are too much unique destinations
02527       return false;
02528   }
02529   DEBUG(dbgs() << "Total number of unique destinations: "
02530         << Dests.size() << '\n'
02531         << "Total number of comparisons: " << numCmps << '\n');
02532 
02533   // Compute span of values.
02534   const APInt& minValue = cast<ConstantInt>(FrontCase.Low)->getValue();
02535   const APInt& maxValue = cast<ConstantInt>(BackCase.High)->getValue();
02536   APInt cmpRange = maxValue - minValue;
02537 
02538   DEBUG(dbgs() << "Compare range: " << cmpRange << '\n'
02539                << "Low bound: " << minValue << '\n'
02540                << "High bound: " << maxValue << '\n');
02541 
02542   if (cmpRange.uge(IntPtrBits) ||
02543       (!(Dests.size() == 1 && numCmps >= 3) &&
02544        !(Dests.size() == 2 && numCmps >= 5) &&
02545        !(Dests.size() >= 3 && numCmps >= 6)))
02546     return false;
02547 
02548   DEBUG(dbgs() << "Emitting bit tests\n");
02549   APInt lowBound = APInt::getNullValue(cmpRange.getBitWidth());
02550 
02551   // Optimize the case where all the case values fit in a
02552   // word without having to subtract minValue. In this case,
02553   // we can optimize away the subtraction.
02554   if (minValue.isNonNegative() && maxValue.slt(IntPtrBits)) {
02555     cmpRange = maxValue;
02556   } else {
02557     lowBound = minValue;
02558   }
02559 
02560   CaseBitsVector CasesBits;
02561   unsigned i, count = 0;
02562 
02563   for (CaseItr I = CR.Range.first, E = CR.Range.second; I!=E; ++I) {
02564     MachineBasicBlock* Dest = I->BB;
02565     for (i = 0; i < count; ++i)
02566       if (Dest == CasesBits[i].BB)
02567         break;
02568 
02569     if (i == count) {
02570       assert((count < 3) && "Too much destinations to test!");
02571       CasesBits.push_back(CaseBits(0, Dest, 0, 0/*Weight*/));
02572       count++;
02573     }
02574 
02575     const APInt& lowValue = cast<ConstantInt>(I->Low)->getValue();
02576     const APInt& highValue = cast<ConstantInt>(I->High)->getValue();
02577 
02578     uint64_t lo = (lowValue - lowBound).getZExtValue();
02579     uint64_t hi = (highValue - lowBound).getZExtValue();
02580     CasesBits[i].ExtraWeight += I->ExtraWeight;
02581 
02582     for (uint64_t j = lo; j <= hi; j++) {
02583       CasesBits[i].Mask |=  1ULL << j;
02584       CasesBits[i].Bits++;
02585     }
02586 
02587   }
02588   std::sort(CasesBits.begin(), CasesBits.end(), CaseBitsCmp());
02589 
02590   BitTestInfo BTC;
02591 
02592   // Figure out which block is immediately after the current one.
02593   MachineFunction::iterator BBI = CR.CaseBB;
02594   ++BBI;
02595 
02596   const BasicBlock *LLVMBB = CR.CaseBB->getBasicBlock();
02597 
02598   DEBUG(dbgs() << "Cases:\n");
02599   for (unsigned i = 0, e = CasesBits.size(); i!=e; ++i) {
02600     DEBUG(dbgs() << "Mask: " << CasesBits[i].Mask
02601                  << ", Bits: " << CasesBits[i].Bits
02602                  << ", BB: " << CasesBits[i].BB << '\n');
02603 
02604     MachineBasicBlock *CaseBB = CurMF->CreateMachineBasicBlock(LLVMBB);
02605     CurMF->insert(BBI, CaseBB);
02606     BTC.push_back(BitTestCase(CasesBits[i].Mask,
02607                               CaseBB,
02608                               CasesBits[i].BB, CasesBits[i].ExtraWeight));
02609 
02610     // Put SV in a virtual register to make it available from the new blocks.
02611     ExportFromCurrentBlock(SV);
02612   }
02613 
02614   BitTestBlock BTB(lowBound, cmpRange, SV,
02615                    -1U, MVT::Other, (CR.CaseBB == SwitchBB),
02616                    CR.CaseBB, Default, BTC);
02617 
02618   if (CR.CaseBB == SwitchBB)
02619     visitBitTestHeader(BTB, SwitchBB);
02620 
02621   BitTestCases.push_back(BTB);
02622 
02623   return true;
02624 }
02625 
02626 /// Clusterify - Transform simple list of Cases into list of CaseRange's
02627 size_t SelectionDAGBuilder::Clusterify(CaseVector& Cases,
02628                                        const SwitchInst& SI) {
02629   size_t numCmps = 0;
02630 
02631   BranchProbabilityInfo *BPI = FuncInfo.BPI;
02632   // Start with "simple" cases
02633   for (SwitchInst::ConstCaseIt i = SI.case_begin(), e = SI.case_end();
02634        i != e; ++i) {
02635     const BasicBlock *SuccBB = i.getCaseSuccessor();
02636     MachineBasicBlock *SMBB = FuncInfo.MBBMap[SuccBB];
02637 
02638     uint32_t ExtraWeight =
02639       BPI ? BPI->getEdgeWeight(SI.getParent(), i.getSuccessorIndex()) : 0;
02640 
02641     Cases.push_back(Case(i.getCaseValue(), i.getCaseValue(),
02642                          SMBB, ExtraWeight));
02643   }
02644   std::sort(Cases.begin(), Cases.end(), CaseCmp());
02645 
02646   // Merge case into clusters
02647   if (Cases.size() >= 2)
02648     // Must recompute end() each iteration because it may be
02649     // invalidated by erase if we hold on to it
02650     for (CaseItr I = Cases.begin(), J = std::next(Cases.begin());
02651          J != Cases.end(); ) {
02652       const APInt& nextValue = cast<ConstantInt>(J->Low)->getValue();
02653       const APInt& currentValue = cast<ConstantInt>(I->High)->getValue();
02654       MachineBasicBlock* nextBB = J->BB;
02655       MachineBasicBlock* currentBB = I->BB;
02656 
02657       // If the two neighboring cases go to the same destination, merge them
02658       // into a single case.
02659       if ((nextValue - currentValue == 1) && (currentBB == nextBB)) {
02660         I->High = J->High;
02661         I->ExtraWeight += J->ExtraWeight;
02662         J = Cases.erase(J);
02663       } else {
02664         I = J++;
02665       }
02666     }
02667 
02668   for (CaseItr I=Cases.begin(), E=Cases.end(); I!=E; ++I, ++numCmps) {
02669     if (I->Low != I->High)
02670       // A range counts double, since it requires two compares.
02671       ++numCmps;
02672   }
02673 
02674   return numCmps;
02675 }
02676 
02677 void SelectionDAGBuilder::UpdateSplitBlock(MachineBasicBlock *First,
02678                                            MachineBasicBlock *Last) {
02679   // Update JTCases.
02680   for (unsigned i = 0, e = JTCases.size(); i != e; ++i)
02681     if (JTCases[i].first.HeaderBB == First)
02682       JTCases[i].first.HeaderBB = Last;
02683 
02684   // Update BitTestCases.
02685   for (unsigned i = 0, e = BitTestCases.size(); i != e; ++i)
02686     if (BitTestCases[i].Parent == First)
02687       BitTestCases[i].Parent = Last;
02688 }
02689 
02690 void SelectionDAGBuilder::visitSwitch(const SwitchInst &SI) {
02691   MachineBasicBlock *SwitchMBB = FuncInfo.MBB;
02692 
02693   // Figure out which block is immediately after the current one.
02694   MachineBasicBlock *NextBlock = nullptr;
02695   MachineBasicBlock *Default = FuncInfo.MBBMap[SI.getDefaultDest()];
02696 
02697   // If there is only the default destination, branch to it if it is not the
02698   // next basic block.  Otherwise, just fall through.
02699   if (!SI.getNumCases()) {
02700     // Update machine-CFG edges.
02701 
02702     // If this is not a fall-through branch, emit the branch.
02703     SwitchMBB->addSuccessor(Default);
02704     if (Default != NextBlock)
02705       DAG.setRoot(DAG.getNode(ISD::BR, getCurSDLoc(),
02706                               MVT::Other, getControlRoot(),
02707                               DAG.getBasicBlock(Default)));
02708 
02709     return;
02710   }
02711 
02712   // If there are any non-default case statements, create a vector of Cases
02713   // representing each one, and sort the vector so that we can efficiently
02714   // create a binary search tree from them.
02715   CaseVector Cases;
02716   size_t numCmps = Clusterify(Cases, SI);
02717   DEBUG(dbgs() << "Clusterify finished. Total clusters: " << Cases.size()
02718                << ". Total compares: " << numCmps << '\n');
02719   (void)numCmps;
02720 
02721   // Get the Value to be switched on and default basic blocks, which will be
02722   // inserted into CaseBlock records, representing basic blocks in the binary
02723   // search tree.
02724   const Value *SV = SI.getCondition();
02725 
02726   // Push the initial CaseRec onto the worklist
02727   CaseRecVector WorkList;
02728   WorkList.push_back(CaseRec(SwitchMBB,nullptr,nullptr,
02729                              CaseRange(Cases.begin(),Cases.end())));
02730 
02731   while (!WorkList.empty()) {
02732     // Grab a record representing a case range to process off the worklist
02733     CaseRec CR = WorkList.back();
02734     WorkList.pop_back();
02735 
02736     if (handleBitTestsSwitchCase(CR, WorkList, SV, Default, SwitchMBB))
02737       continue;
02738 
02739     // If the range has few cases (two or less) emit a series of specific
02740     // tests.
02741     if (handleSmallSwitchRange(CR, WorkList, SV, Default, SwitchMBB))
02742       continue;
02743 
02744     // If the switch has more than N blocks, and is at least 40% dense, and the
02745     // target supports indirect branches, then emit a jump table rather than
02746     // lowering the switch to a binary tree of conditional branches.
02747     // N defaults to 4 and is controlled via TLS.getMinimumJumpTableEntries().
02748     if (handleJTSwitchCase(CR, WorkList, SV, Default, SwitchMBB))
02749       continue;
02750 
02751     // Emit binary tree. We need to pick a pivot, and push left and right ranges
02752     // onto the worklist. Leafs are handled via handleSmallSwitchRange() call.
02753     handleBTSplitSwitchCase(CR, WorkList, SV, Default, SwitchMBB);
02754   }
02755 }
02756 
02757 void SelectionDAGBuilder::visitIndirectBr(const IndirectBrInst &I) {
02758   MachineBasicBlock *IndirectBrMBB = FuncInfo.MBB;
02759 
02760   // Update machine-CFG edges with unique successors.
02761   SmallSet<BasicBlock*, 32> Done;
02762   for (unsigned i = 0, e = I.getNumSuccessors(); i != e; ++i) {
02763     BasicBlock *BB = I.getSuccessor(i);
02764     bool Inserted = Done.insert(BB);
02765     if (!Inserted)
02766         continue;
02767 
02768     MachineBasicBlock *Succ = FuncInfo.MBBMap[BB];
02769     addSuccessorWithWeight(IndirectBrMBB, Succ);
02770   }
02771 
02772   DAG.setRoot(DAG.getNode(ISD::BRIND, getCurSDLoc(),
02773                           MVT::Other, getControlRoot(),
02774                           getValue(I.getAddress())));
02775 }
02776 
02777 void SelectionDAGBuilder::visitUnreachable(const UnreachableInst &I) {
02778   if (DAG.getTarget().Options.TrapUnreachable)
02779     DAG.setRoot(DAG.getNode(ISD::TRAP, getCurSDLoc(), MVT::Other, DAG.getRoot()));
02780 }
02781 
02782 void SelectionDAGBuilder::visitFSub(const User &I) {
02783   // -0.0 - X --> fneg
02784   Type *Ty = I.getType();
02785   if (isa<Constant>(I.getOperand(0)) &&
02786       I.getOperand(0) == ConstantFP::getZeroValueForNegation(Ty)) {
02787     SDValue Op2 = getValue(I.getOperand(1));
02788     setValue(&I, DAG.getNode(ISD::FNEG, getCurSDLoc(),
02789                              Op2.getValueType(), Op2));
02790     return;
02791   }
02792 
02793   visitBinary(I, ISD::FSUB);
02794 }
02795 
02796 void SelectionDAGBuilder::visitBinary(const User &I, unsigned OpCode) {
02797   SDValue Op1 = getValue(I.getOperand(0));
02798   SDValue Op2 = getValue(I.getOperand(1));
02799 
02800   bool nuw = false;
02801   bool nsw = false;
02802   bool exact = false;
02803   if (const OverflowingBinaryOperator *OFBinOp =
02804           dyn_cast<const OverflowingBinaryOperator>(&I)) {
02805     nuw = OFBinOp->hasNoUnsignedWrap();
02806     nsw = OFBinOp->hasNoSignedWrap();
02807   }
02808   if (const PossiblyExactOperator *ExactOp =
02809           dyn_cast<const PossiblyExactOperator>(&I))
02810     exact = ExactOp->isExact();
02811 
02812   SDValue BinNodeValue = DAG.getNode(OpCode, getCurSDLoc(), Op1.getValueType(),
02813                                      Op1, Op2, nuw, nsw, exact);
02814   setValue(&I, BinNodeValue);
02815 }
02816 
02817 void SelectionDAGBuilder::visitShift(const User &I, unsigned Opcode) {
02818   SDValue Op1 = getValue(I.getOperand(0));
02819   SDValue Op2 = getValue(I.getOperand(1));
02820 
02821   EVT ShiftTy = TM.getSubtargetImpl()->getTargetLowering()->getShiftAmountTy(
02822       Op2.getValueType());
02823 
02824   // Coerce the shift amount to the right type if we can.
02825   if (!I.getType()->isVectorTy() && Op2.getValueType() != ShiftTy) {
02826     unsigned ShiftSize = ShiftTy.getSizeInBits();
02827     unsigned Op2Size = Op2.getValueType().getSizeInBits();
02828     SDLoc DL = getCurSDLoc();
02829 
02830     // If the operand is smaller than the shift count type, promote it.
02831     if (ShiftSize > Op2Size)
02832       Op2 = DAG.getNode(ISD::ZERO_EXTEND, DL, ShiftTy, Op2);
02833 
02834     // If the operand is larger than the shift count type but the shift
02835     // count type has enough bits to represent any shift value, truncate
02836     // it now. This is a common case and it exposes the truncate to
02837     // optimization early.
02838     else if (ShiftSize >= Log2_32_Ceil(Op2.getValueType().getSizeInBits()))
02839       Op2 = DAG.getNode(ISD::TRUNCATE, DL, ShiftTy, Op2);
02840     // Otherwise we'll need to temporarily settle for some other convenient
02841     // type.  Type legalization will make adjustments once the shiftee is split.
02842     else
02843       Op2 = DAG.getZExtOrTrunc(Op2, DL, MVT::i32);
02844   }
02845 
02846   bool nuw = false;
02847   bool nsw = false;
02848   bool exact = false;
02849 
02850   if (Opcode == ISD::SRL || Opcode == ISD::SRA || Opcode == ISD::SHL) {
02851 
02852     if (const OverflowingBinaryOperator *OFBinOp =
02853             dyn_cast<const OverflowingBinaryOperator>(&I)) {
02854       nuw = OFBinOp->hasNoUnsignedWrap();
02855       nsw = OFBinOp->hasNoSignedWrap();
02856     }
02857     if (const PossiblyExactOperator *ExactOp =
02858             dyn_cast<const PossiblyExactOperator>(&I))
02859       exact = ExactOp->isExact();
02860   }
02861 
02862   SDValue Res = DAG.getNode(Opcode, getCurSDLoc(), Op1.getValueType(), Op1, Op2,
02863                             nuw, nsw, exact);
02864   setValue(&I, Res);
02865 }
02866 
02867 void SelectionDAGBuilder::visitSDiv(const User &I) {
02868   SDValue Op1 = getValue(I.getOperand(0));
02869   SDValue Op2 = getValue(I.getOperand(1));
02870 
02871   // Turn exact SDivs into multiplications.
02872   // FIXME: This should be in DAGCombiner, but it doesn't have access to the
02873   // exact bit.
02874   if (isa<BinaryOperator>(&I) && cast<BinaryOperator>(&I)->isExact() &&
02875       !isa<ConstantSDNode>(Op1) &&
02876       isa<ConstantSDNode>(Op2) && !cast<ConstantSDNode>(Op2)->isNullValue())
02877     setValue(&I, TM.getSubtargetImpl()->getTargetLowering()->BuildExactSDIV(
02878                      Op1, Op2, getCurSDLoc(), DAG));
02879   else
02880     setValue(&I, DAG.getNode(ISD::SDIV, getCurSDLoc(), Op1.getValueType(),
02881                              Op1, Op2));
02882 }
02883 
02884 void SelectionDAGBuilder::visitICmp(const User &I) {
02885   ICmpInst::Predicate predicate = ICmpInst::BAD_ICMP_PREDICATE;
02886   if (const ICmpInst *IC = dyn_cast<ICmpInst>(&I))
02887     predicate = IC->getPredicate();
02888   else if (const ConstantExpr *IC = dyn_cast<ConstantExpr>(&I))
02889     predicate = ICmpInst::Predicate(IC->getPredicate());
02890   SDValue Op1 = getValue(I.getOperand(0));
02891   SDValue Op2 = getValue(I.getOperand(1));
02892   ISD::CondCode Opcode = getICmpCondCode(predicate);
02893 
02894   EVT DestVT =
02895       TM.getSubtargetImpl()->getTargetLowering()->getValueType(I.getType());
02896   setValue(&I, DAG.getSetCC(getCurSDLoc(), DestVT, Op1, Op2, Opcode));
02897 }
02898 
02899 void SelectionDAGBuilder::visitFCmp(const User &I) {
02900   FCmpInst::Predicate predicate = FCmpInst::BAD_FCMP_PREDICATE;
02901   if (const FCmpInst *FC = dyn_cast<FCmpInst>(&I))
02902     predicate = FC->getPredicate();
02903   else if (const ConstantExpr *FC = dyn_cast<ConstantExpr>(&I))
02904     predicate = FCmpInst::Predicate(FC->getPredicate());
02905   SDValue Op1 = getValue(I.getOperand(0));
02906   SDValue Op2 = getValue(I.getOperand(1));
02907   ISD::CondCode Condition = getFCmpCondCode(predicate);
02908   if (TM.Options.NoNaNsFPMath)
02909     Condition = getFCmpCodeWithoutNaN(Condition);
02910   EVT DestVT =
02911       TM.getSubtargetImpl()->getTargetLowering()->getValueType(I.getType());
02912   setValue(&I, DAG.getSetCC(getCurSDLoc(), DestVT, Op1, Op2, Condition));
02913 }
02914 
02915 void SelectionDAGBuilder::visitSelect(const User &I) {
02916   SmallVector<EVT, 4> ValueVTs;
02917   ComputeValueVTs(*TM.getSubtargetImpl()->getTargetLowering(), I.getType(),
02918                   ValueVTs);
02919   unsigned NumValues = ValueVTs.size();
02920   if (NumValues == 0) return;
02921 
02922   SmallVector<SDValue, 4> Values(NumValues);
02923   SDValue Cond     = getValue(I.getOperand(0));
02924   SDValue TrueVal  = getValue(I.getOperand(1));
02925   SDValue FalseVal = getValue(I.getOperand(2));
02926   ISD::NodeType OpCode = Cond.getValueType().isVector() ?
02927     ISD::VSELECT : ISD::SELECT;
02928 
02929   for (unsigned i = 0; i != NumValues; ++i)
02930     Values[i] = DAG.getNode(OpCode, getCurSDLoc(),
02931                             TrueVal.getNode()->getValueType(TrueVal.getResNo()+i),
02932                             Cond,
02933                             SDValue(TrueVal.getNode(),
02934                                     TrueVal.getResNo() + i),
02935                             SDValue(FalseVal.getNode(),
02936                                     FalseVal.getResNo() + i));
02937 
02938   setValue(&I, DAG.getNode(ISD::MERGE_VALUES, getCurSDLoc(),
02939                            DAG.getVTList(ValueVTs), Values));
02940 }
02941 
02942 void SelectionDAGBuilder::visitTrunc(const User &I) {
02943   // TruncInst cannot be a no-op cast because sizeof(src) > sizeof(dest).
02944   SDValue N = getValue(I.getOperand(0));
02945   EVT DestVT =
02946       TM.getSubtargetImpl()->getTargetLowering()->getValueType(I.getType());
02947   setValue(&I, DAG.getNode(ISD::TRUNCATE, getCurSDLoc(), DestVT, N));
02948 }
02949 
02950 void SelectionDAGBuilder::visitZExt(const User &I) {
02951   // ZExt cannot be a no-op cast because sizeof(src) < sizeof(dest).
02952   // ZExt also can't be a cast to bool for same reason. So, nothing much to do
02953   SDValue N = getValue(I.getOperand(0));
02954   EVT DestVT =
02955       TM.getSubtargetImpl()->getTargetLowering()->getValueType(I.getType());
02956   setValue(&I, DAG.getNode(ISD::ZERO_EXTEND, getCurSDLoc(), DestVT, N));
02957 }
02958 
02959 void SelectionDAGBuilder::visitSExt(const User &I) {
02960   // SExt cannot be a no-op cast because sizeof(src) < sizeof(dest).
02961   // SExt also can't be a cast to bool for same reason. So, nothing much to do
02962   SDValue N = getValue(I.getOperand(0));
02963   EVT DestVT =
02964       TM.getSubtargetImpl()->getTargetLowering()->getValueType(I.getType());
02965   setValue(&I, DAG.getNode(ISD::SIGN_EXTEND, getCurSDLoc(), DestVT, N));
02966 }
02967 
02968 void SelectionDAGBuilder::visitFPTrunc(const User &I) {
02969   // FPTrunc is never a no-op cast, no need to check
02970   SDValue N = getValue(I.getOperand(0));
02971   const TargetLowering *TLI = TM.getSubtargetImpl()->getTargetLowering();
02972   EVT DestVT = TLI->getValueType(I.getType());
02973   setValue(&I, DAG.getNode(ISD::FP_ROUND, getCurSDLoc(),
02974                            DestVT, N,
02975                            DAG.getTargetConstant(0, TLI->getPointerTy())));
02976 }
02977 
02978 void SelectionDAGBuilder::visitFPExt(const User &I) {
02979   // FPExt is never a no-op cast, no need to check
02980   SDValue N = getValue(I.getOperand(0));
02981   EVT DestVT =
02982       TM.getSubtargetImpl()->getTargetLowering()->getValueType(I.getType());
02983   setValue(&I, DAG.getNode(ISD::FP_EXTEND, getCurSDLoc(), DestVT, N));
02984 }
02985 
02986 void SelectionDAGBuilder::visitFPToUI(const User &I) {
02987   // FPToUI is never a no-op cast, no need to check
02988   SDValue N = getValue(I.getOperand(0));
02989   EVT DestVT =
02990       TM.getSubtargetImpl()->getTargetLowering()->getValueType(I.getType());
02991   setValue(&I, DAG.getNode(ISD::FP_TO_UINT, getCurSDLoc(), DestVT, N));
02992 }
02993 
02994 void SelectionDAGBuilder::visitFPToSI(const User &I) {
02995   // FPToSI is never a no-op cast, no need to check
02996   SDValue N = getValue(I.getOperand(0));
02997   EVT DestVT =
02998       TM.getSubtargetImpl()->getTargetLowering()->getValueType(I.getType());
02999   setValue(&I, DAG.getNode(ISD::FP_TO_SINT, getCurSDLoc(), DestVT, N));
03000 }
03001 
03002 void SelectionDAGBuilder::visitUIToFP(const User &I) {
03003   // UIToFP is never a no-op cast, no need to check
03004   SDValue N = getValue(I.getOperand(0));
03005   EVT DestVT =
03006       TM.getSubtargetImpl()->getTargetLowering()->getValueType(I.getType());
03007   setValue(&I, DAG.getNode(ISD::UINT_TO_FP, getCurSDLoc(), DestVT, N));
03008 }
03009 
03010 void SelectionDAGBuilder::visitSIToFP(const User &I) {
03011   // SIToFP is never a no-op cast, no need to check
03012   SDValue N = getValue(I.getOperand(0));
03013   EVT DestVT =
03014       TM.getSubtargetImpl()->getTargetLowering()->getValueType(I.getType());
03015   setValue(&I, DAG.getNode(ISD::SINT_TO_FP, getCurSDLoc(), DestVT, N));
03016 }
03017 
03018 void SelectionDAGBuilder::visitPtrToInt(const User &I) {
03019   // What to do depends on the size of the integer and the size of the pointer.
03020   // We can either truncate, zero extend, or no-op, accordingly.
03021   SDValue N = getValue(I.getOperand(0));
03022   EVT DestVT =
03023       TM.getSubtargetImpl()->getTargetLowering()->getValueType(I.getType());
03024   setValue(&I, DAG.getZExtOrTrunc(N, getCurSDLoc(), DestVT));
03025 }
03026 
03027 void SelectionDAGBuilder::visitIntToPtr(const User &I) {
03028   // What to do depends on the size of the integer and the size of the pointer.
03029   // We can either truncate, zero extend, or no-op, accordingly.
03030   SDValue N = getValue(I.getOperand(0));
03031   EVT DestVT =
03032       TM.getSubtargetImpl()->getTargetLowering()->getValueType(I.getType());
03033   setValue(&I, DAG.getZExtOrTrunc(N, getCurSDLoc(), DestVT));
03034 }
03035 
03036 void SelectionDAGBuilder::visitBitCast(const User &I) {
03037   SDValue N = getValue(I.getOperand(0));
03038   EVT DestVT =
03039       TM.getSubtargetImpl()->getTargetLowering()->getValueType(I.getType());
03040 
03041   // BitCast assures us that source and destination are the same size so this is
03042   // either a BITCAST or a no-op.
03043   if (DestVT != N.getValueType())
03044     setValue(&I, DAG.getNode(ISD::BITCAST, getCurSDLoc(),
03045                              DestVT, N)); // convert types.
03046   // Check if the original LLVM IR Operand was a ConstantInt, because getValue()
03047   // might fold any kind of constant expression to an integer constant and that
03048   // is not what we are looking for. Only regcognize a bitcast of a genuine
03049   // constant integer as an opaque constant.
03050   else if(ConstantInt *C = dyn_cast<ConstantInt>(I.getOperand(0)))
03051     setValue(&I, DAG.getConstant(C->getValue(), DestVT, /*isTarget=*/false,
03052                                  /*isOpaque*/true));
03053   else
03054     setValue(&I, N);            // noop cast.
03055 }
03056 
03057 void SelectionDAGBuilder::visitAddrSpaceCast(const User &I) {
03058   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
03059   const Value *SV = I.getOperand(0);
03060   SDValue N = getValue(SV);
03061   EVT DestVT =
03062       TM.getSubtargetImpl()->getTargetLowering()->getValueType(I.getType());
03063 
03064   unsigned SrcAS = SV->getType()->getPointerAddressSpace();
03065   unsigned DestAS = I.getType()->getPointerAddressSpace();
03066 
03067   if (!TLI.isNoopAddrSpaceCast(SrcAS, DestAS))
03068     N = DAG.getAddrSpaceCast(getCurSDLoc(), DestVT, N, SrcAS, DestAS);
03069 
03070   setValue(&I, N);
03071 }
03072 
03073 void SelectionDAGBuilder::visitInsertElement(const User &I) {
03074   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
03075   SDValue InVec = getValue(I.getOperand(0));
03076   SDValue InVal = getValue(I.getOperand(1));
03077   SDValue InIdx = DAG.getSExtOrTrunc(getValue(I.getOperand(2)),
03078                                      getCurSDLoc(), TLI.getVectorIdxTy());
03079   setValue(&I,
03080            DAG.getNode(ISD::INSERT_VECTOR_ELT, getCurSDLoc(),
03081                        TM.getSubtargetImpl()->getTargetLowering()->getValueType(
03082                            I.getType()),
03083                        InVec, InVal, InIdx));
03084 }
03085 
03086 void SelectionDAGBuilder::visitExtractElement(const User &I) {
03087   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
03088   SDValue InVec = getValue(I.getOperand(0));
03089   SDValue InIdx = DAG.getSExtOrTrunc(getValue(I.getOperand(1)),
03090                                      getCurSDLoc(), TLI.getVectorIdxTy());
03091   setValue(&I,
03092            DAG.getNode(ISD::EXTRACT_VECTOR_ELT, getCurSDLoc(),
03093                        TM.getSubtargetImpl()->getTargetLowering()->getValueType(
03094                            I.getType()),
03095                        InVec, InIdx));
03096 }
03097 
03098 // Utility for visitShuffleVector - Return true if every element in Mask,
03099 // beginning from position Pos and ending in Pos+Size, falls within the
03100 // specified sequential range [L, L+Pos). or is undef.
03101 static bool isSequentialInRange(const SmallVectorImpl<int> &Mask,
03102                                 unsigned Pos, unsigned Size, int Low) {
03103   for (unsigned i = Pos, e = Pos+Size; i != e; ++i, ++Low)
03104     if (Mask[i] >= 0 && Mask[i] != Low)
03105       return false;
03106   return true;
03107 }
03108 
03109 void SelectionDAGBuilder::visitShuffleVector(const User &I) {
03110   SDValue Src1 = getValue(I.getOperand(0));
03111   SDValue Src2 = getValue(I.getOperand(1));
03112 
03113   SmallVector<int, 8> Mask;
03114   ShuffleVectorInst::getShuffleMask(cast<Constant>(I.getOperand(2)), Mask);
03115   unsigned MaskNumElts = Mask.size();
03116 
03117   const TargetLowering *TLI = TM.getSubtargetImpl()->getTargetLowering();
03118   EVT VT = TLI->getValueType(I.getType());
03119   EVT SrcVT = Src1.getValueType();
03120   unsigned SrcNumElts = SrcVT.getVectorNumElements();
03121 
03122   if (SrcNumElts == MaskNumElts) {
03123     setValue(&I, DAG.getVectorShuffle(VT, getCurSDLoc(), Src1, Src2,
03124                                       &Mask[0]));
03125     return;
03126   }
03127 
03128   // Normalize the shuffle vector since mask and vector length don't match.
03129   if (SrcNumElts < MaskNumElts && MaskNumElts % SrcNumElts == 0) {
03130     // Mask is longer than the source vectors and is a multiple of the source
03131     // vectors.  We can use concatenate vector to make the mask and vectors
03132     // lengths match.
03133     if (SrcNumElts*2 == MaskNumElts) {
03134       // First check for Src1 in low and Src2 in high
03135       if (isSequentialInRange(Mask, 0, SrcNumElts, 0) &&
03136           isSequentialInRange(Mask, SrcNumElts, SrcNumElts, SrcNumElts)) {
03137         // The shuffle is concatenating two vectors together.
03138         setValue(&I, DAG.getNode(ISD::CONCAT_VECTORS, getCurSDLoc(),
03139                                  VT, Src1, Src2));
03140         return;
03141       }
03142       // Then check for Src2 in low and Src1 in high
03143       if (isSequentialInRange(Mask, 0, SrcNumElts, SrcNumElts) &&
03144           isSequentialInRange(Mask, SrcNumElts, SrcNumElts, 0)) {
03145         // The shuffle is concatenating two vectors together.
03146         setValue(&I, DAG.getNode(ISD::CONCAT_VECTORS, getCurSDLoc(),
03147                                  VT, Src2, Src1));
03148         return;
03149       }
03150     }
03151 
03152     // Pad both vectors with undefs to make them the same length as the mask.
03153     unsigned NumConcat = MaskNumElts / SrcNumElts;
03154     bool Src1U = Src1.getOpcode() == ISD::UNDEF;
03155     bool Src2U = Src2.getOpcode() == ISD::UNDEF;
03156     SDValue UndefVal = DAG.getUNDEF(SrcVT);
03157 
03158     SmallVector<SDValue, 8> MOps1(NumConcat, UndefVal);
03159     SmallVector<SDValue, 8> MOps2(NumConcat, UndefVal);
03160     MOps1[0] = Src1;
03161     MOps2[0] = Src2;
03162 
03163     Src1 = Src1U ? DAG.getUNDEF(VT) : DAG.getNode(ISD::CONCAT_VECTORS,
03164                                                   getCurSDLoc(), VT, MOps1);
03165     Src2 = Src2U ? DAG.getUNDEF(VT) : DAG.getNode(ISD::CONCAT_VECTORS,
03166                                                   getCurSDLoc(), VT, MOps2);
03167 
03168     // Readjust mask for new input vector length.
03169     SmallVector<int, 8> MappedOps;
03170     for (unsigned i = 0; i != MaskNumElts; ++i) {
03171       int Idx = Mask[i];
03172       if (Idx >= (int)SrcNumElts)
03173         Idx -= SrcNumElts - MaskNumElts;
03174       MappedOps.push_back(Idx);
03175     }
03176 
03177     setValue(&I, DAG.getVectorShuffle(VT, getCurSDLoc(), Src1, Src2,
03178                                       &MappedOps[0]));
03179     return;
03180   }
03181 
03182   if (SrcNumElts > MaskNumElts) {
03183     // Analyze the access pattern of the vector to see if we can extract
03184     // two subvectors and do the shuffle. The analysis is done by calculating
03185     // the range of elements the mask access on both vectors.
03186     int MinRange[2] = { static_cast<int>(SrcNumElts),
03187                         static_cast<int>(SrcNumElts)};
03188     int MaxRange[2] = {-1, -1};
03189 
03190     for (unsigned i = 0; i != MaskNumElts; ++i) {
03191       int Idx = Mask[i];
03192       unsigned Input = 0;
03193       if (Idx < 0)
03194         continue;
03195 
03196       if (Idx >= (int)SrcNumElts) {
03197         Input = 1;
03198         Idx -= SrcNumElts;
03199       }
03200       if (Idx > MaxRange[Input])
03201         MaxRange[Input] = Idx;
03202       if (Idx < MinRange[Input])
03203         MinRange[Input] = Idx;
03204     }
03205 
03206     // Check if the access is smaller than the vector size and can we find
03207     // a reasonable extract index.
03208     int RangeUse[2] = { -1, -1 };  // 0 = Unused, 1 = Extract, -1 = Can not
03209                                    // Extract.
03210     int StartIdx[2];  // StartIdx to extract from
03211     for (unsigned Input = 0; Input < 2; ++Input) {
03212       if (MinRange[Input] >= (int)SrcNumElts && MaxRange[Input] < 0) {
03213         RangeUse[Input] = 0; // Unused
03214         StartIdx[Input] = 0;
03215         continue;
03216       }
03217 
03218       // Find a good start index that is a multiple of the mask length. Then
03219       // see if the rest of the elements are in range.
03220       StartIdx[Input] = (MinRange[Input]/MaskNumElts)*MaskNumElts;
03221       if (MaxRange[Input] - StartIdx[Input] < (int)MaskNumElts &&
03222           StartIdx[Input] + MaskNumElts <= SrcNumElts)
03223         RangeUse[Input] = 1; // Extract from a multiple of the mask length.
03224     }
03225 
03226     if (RangeUse[0] == 0 && RangeUse[1] == 0) {
03227       setValue(&I, DAG.getUNDEF(VT)); // Vectors are not used.
03228       return;
03229     }
03230     if (RangeUse[0] >= 0 && RangeUse[1] >= 0) {
03231       // Extract appropriate subvector and generate a vector shuffle
03232       for (unsigned Input = 0; Input < 2; ++Input) {
03233         SDValue &Src = Input == 0 ? Src1 : Src2;
03234         if (RangeUse[Input] == 0)
03235           Src = DAG.getUNDEF(VT);
03236         else
03237           Src = DAG.getNode(ISD::EXTRACT_SUBVECTOR, getCurSDLoc(), VT,
03238                             Src, DAG.getConstant(StartIdx[Input],
03239                                                  TLI->getVectorIdxTy()));
03240       }
03241 
03242       // Calculate new mask.
03243       SmallVector<int, 8> MappedOps;
03244       for (unsigned i = 0; i != MaskNumElts; ++i) {
03245         int Idx = Mask[i];
03246         if (Idx >= 0) {
03247           if (Idx < (int)SrcNumElts)
03248             Idx -= StartIdx[0];
03249           else
03250             Idx -= SrcNumElts + StartIdx[1] - MaskNumElts;
03251         }
03252         MappedOps.push_back(Idx);
03253       }
03254 
03255       setValue(&I, DAG.getVectorShuffle(VT, getCurSDLoc(), Src1, Src2,
03256                                         &MappedOps[0]));
03257       return;
03258     }
03259   }
03260 
03261   // We can't use either concat vectors or extract subvectors so fall back to
03262   // replacing the shuffle with extract and build vector.
03263   // to insert and build vector.
03264   EVT EltVT = VT.getVectorElementType();
03265   EVT IdxVT = TLI->getVectorIdxTy();
03266   SmallVector<SDValue,8> Ops;
03267   for (unsigned i = 0; i != MaskNumElts; ++i) {
03268     int Idx = Mask[i];
03269     SDValue Res;
03270 
03271     if (Idx < 0) {
03272       Res = DAG.getUNDEF(EltVT);
03273     } else {
03274       SDValue &Src = Idx < (int)SrcNumElts ? Src1 : Src2;
03275       if (Idx >= (int)SrcNumElts) Idx -= SrcNumElts;
03276 
03277       Res = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, getCurSDLoc(),
03278                         EltVT, Src, DAG.getConstant(Idx, IdxVT));
03279     }
03280 
03281     Ops.push_back(Res);
03282   }
03283 
03284   setValue(&I, DAG.getNode(ISD::BUILD_VECTOR, getCurSDLoc(), VT, Ops));
03285 }
03286 
03287 void SelectionDAGBuilder::visitInsertValue(const InsertValueInst &I) {
03288   const Value *Op0 = I.getOperand(0);
03289   const Value *Op1 = I.getOperand(1);
03290   Type *AggTy = I.getType();
03291   Type *ValTy = Op1->getType();
03292   bool IntoUndef = isa<UndefValue>(Op0);
03293   bool FromUndef = isa<UndefValue>(Op1);
03294 
03295   unsigned LinearIndex = ComputeLinearIndex(AggTy, I.getIndices());
03296 
03297   const TargetLowering *TLI = TM.getSubtargetImpl()->getTargetLowering();
03298   SmallVector<EVT, 4> AggValueVTs;
03299   ComputeValueVTs(*TLI, AggTy, AggValueVTs);
03300   SmallVector<EVT, 4> ValValueVTs;
03301   ComputeValueVTs(*TLI, ValTy, ValValueVTs);
03302 
03303   unsigned NumAggValues = AggValueVTs.size();
03304   unsigned NumValValues = ValValueVTs.size();
03305   SmallVector<SDValue, 4> Values(NumAggValues);
03306 
03307   SDValue Agg = getValue(Op0);
03308   unsigned i = 0;
03309   // Copy the beginning value(s) from the original aggregate.
03310   for (; i != LinearIndex; ++i)
03311     Values[i] = IntoUndef ? DAG.getUNDEF(AggValueVTs[i]) :
03312                 SDValue(Agg.getNode(), Agg.getResNo() + i);
03313   // Copy values from the inserted value(s).
03314   if (NumValValues) {
03315     SDValue Val = getValue(Op1);
03316     for (; i != LinearIndex + NumValValues; ++i)
03317       Values[i] = FromUndef ? DAG.getUNDEF(AggValueVTs[i]) :
03318                   SDValue(Val.getNode(), Val.getResNo() + i - LinearIndex);
03319   }
03320   // Copy remaining value(s) from the original aggregate.
03321   for (; i != NumAggValues; ++i)
03322     Values[i] = IntoUndef ? DAG.getUNDEF(AggValueVTs[i]) :
03323                 SDValue(Agg.getNode(), Agg.getResNo() + i);
03324 
03325   setValue(&I, DAG.getNode(ISD::MERGE_VALUES, getCurSDLoc(),
03326                            DAG.getVTList(AggValueVTs), Values));
03327 }
03328 
03329 void SelectionDAGBuilder::visitExtractValue(const ExtractValueInst &I) {
03330   const Value *Op0 = I.getOperand(0);
03331   Type *AggTy = Op0->getType();
03332   Type *ValTy = I.getType();
03333   bool OutOfUndef = isa<UndefValue>(Op0);
03334 
03335   unsigned LinearIndex = ComputeLinearIndex(AggTy, I.getIndices());
03336 
03337   const TargetLowering *TLI = TM.getSubtargetImpl()->getTargetLowering();
03338   SmallVector<EVT, 4> ValValueVTs;
03339   ComputeValueVTs(*TLI, ValTy, ValValueVTs);
03340 
03341   unsigned NumValValues = ValValueVTs.size();
03342 
03343   // Ignore a extractvalue that produces an empty object
03344   if (!NumValValues) {
03345     setValue(&I, DAG.getUNDEF(MVT(MVT::Other)));
03346     return;
03347   }
03348 
03349   SmallVector<SDValue, 4> Values(NumValValues);
03350 
03351   SDValue Agg = getValue(Op0);
03352   // Copy out the selected value(s).
03353   for (unsigned i = LinearIndex; i != LinearIndex + NumValValues; ++i)
03354     Values[i - LinearIndex] =
03355       OutOfUndef ?
03356         DAG.getUNDEF(Agg.getNode()->getValueType(Agg.getResNo() + i)) :
03357         SDValue(Agg.getNode(), Agg.getResNo() + i);
03358 
03359   setValue(&I, DAG.getNode(ISD::MERGE_VALUES, getCurSDLoc(),
03360                            DAG.getVTList(ValValueVTs), Values));
03361 }
03362 
03363 void SelectionDAGBuilder::visitGetElementPtr(const User &I) {
03364   Value *Op0 = I.getOperand(0);
03365   // Note that the pointer operand may be a vector of pointers. Take the scalar
03366   // element which holds a pointer.
03367   Type *Ty = Op0->getType()->getScalarType();
03368   unsigned AS = Ty->getPointerAddressSpace();
03369   SDValue N = getValue(Op0);
03370 
03371   for (GetElementPtrInst::const_op_iterator OI = I.op_begin()+1, E = I.op_end();
03372        OI != E; ++OI) {
03373     const Value *Idx = *OI;
03374     if (StructType *StTy = dyn_cast<StructType>(Ty)) {
03375       unsigned Field = cast<Constant>(Idx)->getUniqueInteger().getZExtValue();
03376       if (Field) {
03377         // N = N + Offset
03378         uint64_t Offset = DL->getStructLayout(StTy)->getElementOffset(Field);
03379         N = DAG.getNode(ISD::ADD, getCurSDLoc(), N.getValueType(), N,
03380                         DAG.getConstant(Offset, N.getValueType()));
03381       }
03382 
03383       Ty = StTy->getElementType(Field);
03384     } else {
03385       Ty = cast<SequentialType>(Ty)->getElementType();
03386 
03387       // If this is a constant subscript, handle it quickly.
03388       const TargetLowering *TLI = TM.getSubtargetImpl()->getTargetLowering();
03389       if (const ConstantInt *CI = dyn_cast<ConstantInt>(Idx)) {
03390         if (CI->isZero()) continue;
03391         uint64_t Offs =
03392             DL->getTypeAllocSize(Ty)*cast<ConstantInt>(CI)->getSExtValue();
03393         SDValue OffsVal;
03394         EVT PTy = TLI->getPointerTy(AS);
03395         unsigned PtrBits = PTy.getSizeInBits();
03396         if (PtrBits < 64)
03397           OffsVal = DAG.getNode(ISD::TRUNCATE, getCurSDLoc(), PTy,
03398                                 DAG.getConstant(Offs, MVT::i64));
03399         else
03400           OffsVal = DAG.getConstant(Offs, PTy);
03401 
03402         N = DAG.getNode(ISD::ADD, getCurSDLoc(), N.getValueType(), N,
03403                         OffsVal);
03404         continue;
03405       }
03406 
03407       // N = N + Idx * ElementSize;
03408       APInt ElementSize = APInt(TLI->getPointerSizeInBits(AS),
03409                                 DL->getTypeAllocSize(Ty));
03410       SDValue IdxN = getValue(Idx);
03411 
03412       // If the index is smaller or larger than intptr_t, truncate or extend
03413       // it.
03414       IdxN = DAG.getSExtOrTrunc(IdxN, getCurSDLoc(), N.getValueType());
03415 
03416       // If this is a multiply by a power of two, turn it into a shl
03417       // immediately.  This is a very common case.
03418       if (ElementSize != 1) {
03419         if (ElementSize.isPowerOf2()) {
03420           unsigned Amt = ElementSize.logBase2();
03421           IdxN = DAG.getNode(ISD::SHL, getCurSDLoc(),
03422                              N.getValueType(), IdxN,
03423                              DAG.getConstant(Amt, IdxN.getValueType()));
03424         } else {
03425           SDValue Scale = DAG.getConstant(ElementSize, IdxN.getValueType());
03426           IdxN = DAG.getNode(ISD::MUL, getCurSDLoc(),
03427                              N.getValueType(), IdxN, Scale);
03428         }
03429       }
03430 
03431       N = DAG.getNode(ISD::ADD, getCurSDLoc(),
03432                       N.getValueType(), N, IdxN);
03433     }
03434   }
03435 
03436   setValue(&I, N);
03437 }
03438 
03439 void SelectionDAGBuilder::visitAlloca(const AllocaInst &I) {
03440   // If this is a fixed sized alloca in the entry block of the function,
03441   // allocate it statically on the stack.
03442   if (FuncInfo.StaticAllocaMap.count(&I))
03443     return;   // getValue will auto-populate this.
03444 
03445   Type *Ty = I.getAllocatedType();
03446   const TargetLowering *TLI = TM.getSubtargetImpl()->getTargetLowering();
03447   uint64_t TySize = TLI->getDataLayout()->getTypeAllocSize(Ty);
03448   unsigned Align =
03449     std::max((unsigned)TLI->getDataLayout()->getPrefTypeAlignment(Ty),
03450              I.getAlignment());
03451 
03452   SDValue AllocSize = getValue(I.getArraySize());
03453 
03454   EVT IntPtr = TLI->getPointerTy();
03455   if (AllocSize.getValueType() != IntPtr)
03456     AllocSize = DAG.getZExtOrTrunc(AllocSize, getCurSDLoc(), IntPtr);
03457 
03458   AllocSize = DAG.getNode(ISD::MUL, getCurSDLoc(), IntPtr,
03459                           AllocSize,
03460                           DAG.getConstant(TySize, IntPtr));
03461 
03462   // Handle alignment.  If the requested alignment is less than or equal to
03463   // the stack alignment, ignore it.  If the size is greater than or equal to
03464   // the stack alignment, we note this in the DYNAMIC_STACKALLOC node.
03465   unsigned StackAlign =
03466       TM.getSubtargetImpl()->getFrameLowering()->getStackAlignment();
03467   if (Align <= StackAlign)
03468     Align = 0;
03469 
03470   // Round the size of the allocation up to the stack alignment size
03471   // by add SA-1 to the size.
03472   AllocSize = DAG.getNode(ISD::ADD, getCurSDLoc(),
03473                           AllocSize.getValueType(), AllocSize,
03474                           DAG.getIntPtrConstant(StackAlign-1));
03475 
03476   // Mask out the low bits for alignment purposes.
03477   AllocSize = DAG.getNode(ISD::AND, getCurSDLoc(),
03478                           AllocSize.getValueType(), AllocSize,
03479                           DAG.getIntPtrConstant(~(uint64_t)(StackAlign-1)));
03480 
03481   SDValue Ops[] = { getRoot(), AllocSize, DAG.getIntPtrConstant(Align) };
03482   SDVTList VTs = DAG.getVTList(AllocSize.getValueType(), MVT::Other);
03483   SDValue DSA = DAG.getNode(ISD::DYNAMIC_STACKALLOC, getCurSDLoc(), VTs, Ops);
03484   setValue(&I, DSA);
03485   DAG.setRoot(DSA.getValue(1));
03486 
03487   assert(FuncInfo.MF->getFrameInfo()->hasVarSizedObjects());
03488 }
03489 
03490 void SelectionDAGBuilder::visitLoad(const LoadInst &I) {
03491   if (I.isAtomic())
03492     return visitAtomicLoad(I);
03493 
03494   const Value *SV = I.getOperand(0);
03495   SDValue Ptr = getValue(SV);
03496 
03497   Type *Ty = I.getType();
03498 
03499   bool isVolatile = I.isVolatile();
03500   bool isNonTemporal = I.getMetadata("nontemporal") != nullptr;
03501   bool isInvariant = I.getMetadata("invariant.load") != nullptr;
03502   unsigned Alignment = I.getAlignment();
03503 
03504   AAMDNodes AAInfo;
03505   I.getAAMetadata(AAInfo);
03506   const MDNode *Ranges = I.getMetadata(LLVMContext::MD_range);
03507 
03508   SmallVector<EVT, 4> ValueVTs;
03509   SmallVector<uint64_t, 4> Offsets;
03510   ComputeValueVTs(*TM.getSubtargetImpl()->getTargetLowering(), Ty, ValueVTs,
03511                   &Offsets);
03512   unsigned NumValues = ValueVTs.size();
03513   if (NumValues == 0)
03514     return;
03515 
03516   SDValue Root;
03517   bool ConstantMemory = false;
03518   if (isVolatile || NumValues > MaxParallelChains)
03519     // Serialize volatile loads with other side effects.
03520     Root = getRoot();
03521   else if (AA->pointsToConstantMemory(
03522              AliasAnalysis::Location(SV, AA->getTypeStoreSize(Ty), AAInfo))) {
03523     // Do not serialize (non-volatile) loads of constant memory with anything.
03524     Root = DAG.getEntryNode();
03525     ConstantMemory = true;
03526   } else {
03527     // Do not serialize non-volatile loads against each other.
03528     Root = DAG.getRoot();
03529   }
03530 
03531   const TargetLowering *TLI = TM.getSubtargetImpl()->getTargetLowering();
03532   if (isVolatile)
03533     Root = TLI->prepareVolatileOrAtomicLoad(Root, getCurSDLoc(), DAG);
03534 
03535   SmallVector<SDValue, 4> Values(NumValues);
03536   SmallVector<SDValue, 4> Chains(std::min(unsigned(MaxParallelChains),
03537                                           NumValues));
03538   EVT PtrVT = Ptr.getValueType();
03539   unsigned ChainI = 0;
03540   for (unsigned i = 0; i != NumValues; ++i, ++ChainI) {
03541     // Serializing loads here may result in excessive register pressure, and
03542     // TokenFactor places arbitrary choke points on the scheduler. SD scheduling
03543     // could recover a bit by hoisting nodes upward in the chain by recognizing
03544     // they are side-effect free or do not alias. The optimizer should really
03545     // avoid this case by converting large object/array copies to llvm.memcpy
03546     // (MaxParallelChains should always remain as failsafe).
03547     if (ChainI == MaxParallelChains) {
03548       assert(PendingLoads.empty() && "PendingLoads must be serialized first");
03549       SDValue Chain = DAG.getNode(ISD::TokenFactor, getCurSDLoc(), MVT::Other,
03550                                   makeArrayRef(Chains.data(), ChainI));
03551       Root = Chain;
03552       ChainI = 0;
03553     }
03554     SDValue A = DAG.getNode(ISD::ADD, getCurSDLoc(),
03555                             PtrVT, Ptr,
03556                             DAG.getConstant(Offsets[i], PtrVT));
03557     SDValue L = DAG.getLoad(ValueVTs[i], getCurSDLoc(), Root,
03558                             A, MachinePointerInfo(SV, Offsets[i]), isVolatile,
03559                             isNonTemporal, isInvariant, Alignment, AAInfo,
03560                             Ranges);
03561 
03562     Values[i] = L;
03563     Chains[ChainI] = L.getValue(1);
03564   }
03565 
03566   if (!ConstantMemory) {
03567     SDValue Chain = DAG.getNode(ISD::TokenFactor, getCurSDLoc(), MVT::Other,
03568                                 makeArrayRef(Chains.data(), ChainI));
03569     if (isVolatile)
03570       DAG.setRoot(Chain);
03571     else
03572       PendingLoads.push_back(Chain);
03573   }
03574 
03575   setValue(&I, DAG.getNode(ISD::MERGE_VALUES, getCurSDLoc(),
03576                            DAG.getVTList(ValueVTs), Values));
03577 }
03578 
03579 void SelectionDAGBuilder::visitStore(const StoreInst &I) {
03580   if (I.isAtomic())
03581     return visitAtomicStore(I);
03582 
03583   const Value *SrcV = I.getOperand(0);
03584   const Value *PtrV = I.getOperand(1);
03585 
03586   SmallVector<EVT, 4> ValueVTs;
03587   SmallVector<uint64_t, 4> Offsets;
03588   ComputeValueVTs(*TM.getSubtargetImpl()->getTargetLowering(), SrcV->getType(),
03589                   ValueVTs, &Offsets);
03590   unsigned NumValues = ValueVTs.size();
03591   if (NumValues == 0)
03592     return;
03593 
03594   // Get the lowered operands. Note that we do this after
03595   // checking if NumResults is zero, because with zero results
03596   // the operands won't have values in the map.
03597   SDValue Src = getValue(SrcV);
03598   SDValue Ptr = getValue(PtrV);
03599 
03600   SDValue Root = getRoot();
03601   SmallVector<SDValue, 4> Chains(std::min(unsigned(MaxParallelChains),
03602                                           NumValues));
03603   EVT PtrVT = Ptr.getValueType();
03604   bool isVolatile = I.isVolatile();
03605   bool isNonTemporal = I.getMetadata("nontemporal") != nullptr;
03606   unsigned Alignment = I.getAlignment();
03607 
03608   AAMDNodes AAInfo;
03609   I.getAAMetadata(AAInfo);
03610 
03611   unsigned ChainI = 0;
03612   for (unsigned i = 0; i != NumValues; ++i, ++ChainI) {
03613     // See visitLoad comments.
03614     if (ChainI == MaxParallelChains) {
03615       SDValue Chain = DAG.getNode(ISD::TokenFactor, getCurSDLoc(), MVT::Other,
03616                                   makeArrayRef(Chains.data(), ChainI));
03617       Root = Chain;
03618       ChainI = 0;
03619     }
03620     SDValue Add = DAG.getNode(ISD::ADD, getCurSDLoc(), PtrVT, Ptr,
03621                               DAG.getConstant(Offsets[i], PtrVT));
03622     SDValue St = DAG.getStore(Root, getCurSDLoc(),
03623                               SDValue(Src.getNode(), Src.getResNo() + i),
03624                               Add, MachinePointerInfo(PtrV, Offsets[i]),
03625                               isVolatile, isNonTemporal, Alignment, AAInfo);
03626     Chains[ChainI] = St;
03627   }
03628 
03629   SDValue StoreNode = DAG.getNode(ISD::TokenFactor, getCurSDLoc(), MVT::Other,
03630                                   makeArrayRef(Chains.data(), ChainI));
03631   DAG.setRoot(StoreNode);
03632 }
03633 
03634 static SDValue InsertFenceForAtomic(SDValue Chain, AtomicOrdering Order,
03635                                     SynchronizationScope Scope,
03636                                     bool Before, SDLoc dl,
03637                                     SelectionDAG &DAG,
03638                                     const TargetLowering &TLI) {
03639   // Fence, if necessary
03640   if (Before) {
03641     if (Order == AcquireRelease || Order == SequentiallyConsistent)
03642       Order = Release;
03643     else if (Order == Acquire || Order == Monotonic || Order == Unordered)
03644       return Chain;
03645   } else {
03646     if (Order == AcquireRelease)
03647       Order = Acquire;
03648     else if (Order == Release || Order == Monotonic || Order == Unordered)
03649       return Chain;
03650   }
03651   SDValue Ops[3];
03652   Ops[0] = Chain;
03653   Ops[1] = DAG.getConstant(Order, TLI.getPointerTy());
03654   Ops[2] = DAG.getConstant(Scope, TLI.getPointerTy());
03655   return DAG.getNode(ISD::ATOMIC_FENCE, dl, MVT::Other, Ops);
03656 }
03657 
03658 void SelectionDAGBuilder::visitAtomicCmpXchg(const AtomicCmpXchgInst &I) {
03659   SDLoc dl = getCurSDLoc();
03660   AtomicOrdering SuccessOrder = I.getSuccessOrdering();
03661   AtomicOrdering FailureOrder = I.getFailureOrdering();
03662   SynchronizationScope Scope = I.getSynchScope();
03663 
03664   SDValue InChain = getRoot();
03665 
03666   const TargetLowering *TLI = TM.getSubtargetImpl()->getTargetLowering();
03667   if (TLI->getInsertFencesForAtomic())
03668     InChain = InsertFenceForAtomic(InChain, SuccessOrder, Scope, true, dl,
03669                                    DAG, *TLI);
03670 
03671   MVT MemVT = getValue(I.getCompareOperand()).getSimpleValueType();
03672   SDVTList VTs = DAG.getVTList(MemVT, MVT::i1, MVT::Other);
03673   SDValue L = DAG.getAtomicCmpSwap(
03674       ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS, dl, MemVT, VTs, InChain,
03675       getValue(I.getPointerOperand()), getValue(I.getCompareOperand()),
03676       getValue(I.getNewValOperand()), MachinePointerInfo(I.getPointerOperand()),
03677       0 /* Alignment */,
03678       TLI->getInsertFencesForAtomic() ? Monotonic : SuccessOrder,
03679       TLI->getInsertFencesForAtomic() ? Monotonic : FailureOrder, Scope);
03680 
03681   SDValue OutChain = L.getValue(2);
03682 
03683   if (TLI->getInsertFencesForAtomic())
03684     OutChain = InsertFenceForAtomic(OutChain, SuccessOrder, Scope, false, dl,
03685                                     DAG, *TLI);
03686 
03687   setValue(&I, L);
03688   DAG.setRoot(OutChain);
03689 }
03690 
03691 void SelectionDAGBuilder::visitAtomicRMW(const AtomicRMWInst &I) {
03692   SDLoc dl = getCurSDLoc();
03693   ISD::NodeType NT;
03694   switch (I.getOperation()) {
03695   default: llvm_unreachable("Unknown atomicrmw operation");
03696   case AtomicRMWInst::Xchg: NT = ISD::ATOMIC_SWAP; break;
03697   case AtomicRMWInst::Add:  NT = ISD::ATOMIC_LOAD_ADD; break;
03698   case AtomicRMWInst::Sub:  NT = ISD::ATOMIC_LOAD_SUB; break;
03699   case AtomicRMWInst::And:  NT = ISD::ATOMIC_LOAD_AND; break;
03700   case AtomicRMWInst::Nand: NT = ISD::ATOMIC_LOAD_NAND; break;
03701   case AtomicRMWInst::Or:   NT = ISD::ATOMIC_LOAD_OR; break;
03702   case AtomicRMWInst::Xor:  NT = ISD::ATOMIC_LOAD_XOR; break;
03703   case AtomicRMWInst::Max:  NT = ISD::ATOMIC_LOAD_MAX; break;
03704   case AtomicRMWInst::Min:  NT = ISD::ATOMIC_LOAD_MIN; break;
03705   case AtomicRMWInst::UMax: NT = ISD::ATOMIC_LOAD_UMAX; break;
03706   case AtomicRMWInst::UMin: NT = ISD::ATOMIC_LOAD_UMIN; break;
03707   }
03708   AtomicOrdering Order = I.getOrdering();
03709   SynchronizationScope Scope = I.getSynchScope();
03710 
03711   SDValue InChain = getRoot();
03712 
03713   const TargetLowering *TLI = TM.getSubtargetImpl()->getTargetLowering();
03714   if (TLI->getInsertFencesForAtomic())
03715     InChain = InsertFenceForAtomic(InChain, Order, Scope, true, dl,
03716                                    DAG, *TLI);
03717 
03718   SDValue L =
03719     DAG.getAtomic(NT, dl,
03720                   getValue(I.getValOperand()).getSimpleValueType(),
03721                   InChain,
03722                   getValue(I.getPointerOperand()),
03723                   getValue(I.getValOperand()),
03724                   I.getPointerOperand(), 0 /* Alignment */,
03725                   TLI->getInsertFencesForAtomic() ? Monotonic : Order,
03726                   Scope);
03727 
03728   SDValue OutChain = L.getValue(1);
03729 
03730   if (TLI->getInsertFencesForAtomic())
03731     OutChain = InsertFenceForAtomic(OutChain, Order, Scope, false, dl,
03732                                     DAG, *TLI);
03733 
03734   setValue(&I, L);
03735   DAG.setRoot(OutChain);
03736 }
03737 
03738 void SelectionDAGBuilder::visitFence(const FenceInst &I) {
03739   SDLoc dl = getCurSDLoc();
03740   const TargetLowering *TLI = TM.getSubtargetImpl()->getTargetLowering();
03741   SDValue Ops[3];
03742   Ops[0] = getRoot();
03743   Ops[1] = DAG.getConstant(I.getOrdering(), TLI->getPointerTy());
03744   Ops[2] = DAG.getConstant(I.getSynchScope(), TLI->getPointerTy());
03745   DAG.setRoot(DAG.getNode(ISD::ATOMIC_FENCE, dl, MVT::Other, Ops));
03746 }
03747 
03748 void SelectionDAGBuilder::visitAtomicLoad(const LoadInst &I) {
03749   SDLoc dl = getCurSDLoc();
03750   AtomicOrdering Order = I.getOrdering();
03751   SynchronizationScope Scope = I.getSynchScope();
03752 
03753   SDValue InChain = getRoot();
03754 
03755   const TargetLowering *TLI = TM.getSubtargetImpl()->getTargetLowering();
03756   EVT VT = TLI->getValueType(I.getType());
03757 
03758   if (I.getAlignment() < VT.getSizeInBits() / 8)
03759     report_fatal_error("Cannot generate unaligned atomic load");
03760 
03761   MachineMemOperand *MMO =
03762       DAG.getMachineFunction().
03763       getMachineMemOperand(MachinePointerInfo(I.getPointerOperand()),
03764                            MachineMemOperand::MOVolatile |
03765                            MachineMemOperand::MOLoad,
03766                            VT.getStoreSize(),
03767                            I.getAlignment() ? I.getAlignment() :
03768                                               DAG.getEVTAlignment(VT));
03769 
03770   InChain = TLI->prepareVolatileOrAtomicLoad(InChain, dl, DAG);
03771   SDValue L =
03772       DAG.getAtomic(ISD::ATOMIC_LOAD, dl, VT, VT, InChain,
03773                     getValue(I.getPointerOperand()), MMO,
03774                     TLI->getInsertFencesForAtomic() ? Monotonic : Order,
03775                     Scope);
03776 
03777   SDValue OutChain = L.getValue(1);
03778 
03779   if (TLI->getInsertFencesForAtomic())
03780     OutChain = InsertFenceForAtomic(OutChain, Order, Scope, false, dl,
03781                                     DAG, *TLI);
03782 
03783   setValue(&I, L);
03784   DAG.setRoot(OutChain);
03785 }
03786 
03787 void SelectionDAGBuilder::visitAtomicStore(const StoreInst &I) {
03788   SDLoc dl = getCurSDLoc();
03789 
03790   AtomicOrdering Order = I.getOrdering();
03791   SynchronizationScope Scope = I.getSynchScope();
03792 
03793   SDValue InChain = getRoot();
03794 
03795   const TargetLowering *TLI = TM.getSubtargetImpl()->getTargetLowering();
03796   EVT VT = TLI->getValueType(I.getValueOperand()->getType());
03797 
03798   if (I.getAlignment() < VT.getSizeInBits() / 8)
03799     report_fatal_error("Cannot generate unaligned atomic store");
03800 
03801   if (TLI->getInsertFencesForAtomic())
03802     InChain = InsertFenceForAtomic(InChain, Order, Scope, true, dl,
03803                                    DAG, *TLI);
03804 
03805   SDValue OutChain =
03806     DAG.getAtomic(ISD::ATOMIC_STORE, dl, VT,
03807                   InChain,
03808                   getValue(I.getPointerOperand()),
03809                   getValue(I.getValueOperand()),
03810                   I.getPointerOperand(), I.getAlignment(),
03811                   TLI->getInsertFencesForAtomic() ? Monotonic : Order,
03812                   Scope);
03813 
03814   if (TLI->getInsertFencesForAtomic())
03815     OutChain = InsertFenceForAtomic(OutChain, Order, Scope, false, dl,
03816                                     DAG, *TLI);
03817 
03818   DAG.setRoot(OutChain);
03819 }
03820 
03821 /// visitTargetIntrinsic - Lower a call of a target intrinsic to an INTRINSIC
03822 /// node.
03823 void SelectionDAGBuilder::visitTargetIntrinsic(const CallInst &I,
03824                                                unsigned Intrinsic) {
03825   bool HasChain = !I.doesNotAccessMemory();
03826   bool OnlyLoad = HasChain && I.onlyReadsMemory();
03827 
03828   // Build the operand list.
03829   SmallVector<SDValue, 8> Ops;
03830   if (HasChain) {  // If this intrinsic has side-effects, chainify it.
03831     if (OnlyLoad) {
03832       // We don't need to serialize loads against other loads.
03833       Ops.push_back(DAG.getRoot());
03834     } else {
03835       Ops.push_back(getRoot());
03836     }
03837   }
03838 
03839   // Info is set by getTgtMemInstrinsic
03840   TargetLowering::IntrinsicInfo Info;
03841   const TargetLowering *TLI = TM.getSubtargetImpl()->getTargetLowering();
03842   bool IsTgtIntrinsic = TLI->getTgtMemIntrinsic(Info, I, Intrinsic);
03843 
03844   // Add the intrinsic ID as an integer operand if it's not a target intrinsic.
03845   if (!IsTgtIntrinsic || Info.opc == ISD::INTRINSIC_VOID ||
03846       Info.opc == ISD::INTRINSIC_W_CHAIN)
03847     Ops.push_back(DAG.getTargetConstant(Intrinsic, TLI->getPointerTy()));
03848 
03849   // Add all operands of the call to the operand list.
03850   for (unsigned i = 0, e = I.getNumArgOperands(); i != e; ++i) {
03851     SDValue Op = getValue(I.getArgOperand(i));
03852     Ops.push_back(Op);
03853   }
03854 
03855   SmallVector<EVT, 4> ValueVTs;
03856   ComputeValueVTs(*TLI, I.getType(), ValueVTs);
03857 
03858   if (HasChain)
03859     ValueVTs.push_back(MVT::Other);
03860 
03861   SDVTList VTs = DAG.getVTList(ValueVTs);
03862 
03863   // Create the node.
03864   SDValue Result;
03865   if (IsTgtIntrinsic) {
03866     // This is target intrinsic that touches memory
03867     Result = DAG.getMemIntrinsicNode(Info.opc, getCurSDLoc(),
03868                                      VTs, Ops, Info.memVT,
03869                                    MachinePointerInfo(Info.ptrVal, Info.offset),
03870                                      Info.align, Info.vol,
03871                                      Info.readMem, Info.writeMem, Info.size);
03872   } else if (!HasChain) {
03873     Result = DAG.getNode(ISD::INTRINSIC_WO_CHAIN, getCurSDLoc(), VTs, Ops);
03874   } else if (!I.getType()->isVoidTy()) {
03875     Result = DAG.getNode(ISD::INTRINSIC_W_CHAIN, getCurSDLoc(), VTs, Ops);
03876   } else {
03877     Result = DAG.getNode(ISD::INTRINSIC_VOID, getCurSDLoc(), VTs, Ops);
03878   }
03879 
03880   if (HasChain) {
03881     SDValue Chain = Result.getValue(Result.getNode()->getNumValues()-1);
03882     if (OnlyLoad)
03883       PendingLoads.push_back(Chain);
03884     else
03885       DAG.setRoot(Chain);
03886   }
03887 
03888   if (!I.getType()->isVoidTy()) {
03889     if (VectorType *PTy = dyn_cast<VectorType>(I.getType())) {
03890       EVT VT = TLI->getValueType(PTy);
03891       Result = DAG.getNode(ISD::BITCAST, getCurSDLoc(), VT, Result);
03892     }
03893 
03894     setValue(&I, Result);
03895   }
03896 }
03897 
03898 /// GetSignificand - Get the significand and build it into a floating-point
03899 /// number with exponent of 1:
03900 ///
03901 ///   Op = (Op & 0x007fffff) | 0x3f800000;
03902 ///
03903 /// where Op is the hexadecimal representation of floating point value.
03904 static SDValue
03905 GetSignificand(SelectionDAG &DAG, SDValue Op, SDLoc dl) {
03906   SDValue t1 = DAG.getNode(ISD::AND, dl, MVT::i32, Op,
03907                            DAG.getConstant(0x007fffff, MVT::i32));
03908   SDValue t2 = DAG.getNode(ISD::OR, dl, MVT::i32, t1,
03909                            DAG.getConstant(0x3f800000, MVT::i32));
03910   return DAG.getNode(ISD::BITCAST, dl, MVT::f32, t2);
03911 }
03912 
03913 /// GetExponent - Get the exponent:
03914 ///
03915 ///   (float)(int)(((Op & 0x7f800000) >> 23) - 127);
03916 ///
03917 /// where Op is the hexadecimal representation of floating point value.
03918 static SDValue
03919 GetExponent(SelectionDAG &DAG, SDValue Op, const TargetLowering &TLI,
03920             SDLoc dl) {
03921   SDValue t0 = DAG.getNode(ISD::AND, dl, MVT::i32, Op,
03922                            DAG.getConstant(0x7f800000, MVT::i32));
03923   SDValue t1 = DAG.getNode(ISD::SRL, dl, MVT::i32, t0,
03924                            DAG.getConstant(23, TLI.getPointerTy()));
03925   SDValue t2 = DAG.getNode(ISD::SUB, dl, MVT::i32, t1,
03926                            DAG.getConstant(127, MVT::i32));
03927   return DAG.getNode(ISD::SINT_TO_FP, dl, MVT::f32, t2);
03928 }
03929 
03930 /// getF32Constant - Get 32-bit floating point constant.
03931 static SDValue
03932 getF32Constant(SelectionDAG &DAG, unsigned Flt) {
03933   return DAG.getConstantFP(APFloat(APFloat::IEEEsingle, APInt(32, Flt)),
03934                            MVT::f32);
03935 }
03936 
03937 /// expandExp - Lower an exp intrinsic. Handles the special sequences for
03938 /// limited-precision mode.
03939 static SDValue expandExp(SDLoc dl, SDValue Op, SelectionDAG &DAG,
03940                          const TargetLowering &TLI) {
03941   if (Op.getValueType() == MVT::f32 &&
03942       LimitFloatPrecision > 0 && LimitFloatPrecision <= 18) {
03943 
03944     // Put the exponent in the right bit position for later addition to the
03945     // final result:
03946     //
03947     //   #define LOG2OFe 1.4426950f
03948     //   IntegerPartOfX = ((int32_t)(X * LOG2OFe));
03949     SDValue t0 = DAG.getNode(ISD::FMUL, dl, MVT::f32, Op,
03950                              getF32Constant(DAG, 0x3fb8aa3b));
03951     SDValue IntegerPartOfX = DAG.getNode(ISD::FP_TO_SINT, dl, MVT::i32, t0);
03952 
03953     //   FractionalPartOfX = (X * LOG2OFe) - (float)IntegerPartOfX;
03954     SDValue t1 = DAG.getNode(ISD::SINT_TO_FP, dl, MVT::f32, IntegerPartOfX);
03955     SDValue X = DAG.getNode(ISD::FSUB, dl, MVT::f32, t0, t1);
03956 
03957     //   IntegerPartOfX <<= 23;
03958     IntegerPartOfX = DAG.getNode(ISD::SHL, dl, MVT::i32, IntegerPartOfX,
03959                                  DAG.getConstant(23, TLI.getPointerTy()));
03960 
03961     SDValue TwoToFracPartOfX;
03962     if (LimitFloatPrecision <= 6) {
03963       // For floating-point precision of 6:
03964       //
03965       //   TwoToFractionalPartOfX =
03966       //     0.997535578f +
03967       //       (0.735607626f + 0.252464424f * x) * x;
03968       //
03969       // error 0.0144103317, which is 6 bits
03970       SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X,
03971                                getF32Constant(DAG, 0x3e814304));
03972       SDValue t3 = DAG.getNode(ISD::FADD, dl, MVT::f32, t2,
03973                                getF32Constant(DAG, 0x3f3c50c8));
03974       SDValue t4 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t3, X);
03975       TwoToFracPartOfX = DAG.getNode(ISD::FADD, dl, MVT::f32, t4,
03976                                      getF32Constant(DAG, 0x3f7f5e7e));
03977     } else if (LimitFloatPrecision <= 12) {
03978       // For floating-point precision of 12:
03979       //
03980       //   TwoToFractionalPartOfX =
03981       //     0.999892986f +
03982       //       (0.696457318f +
03983       //         (0.224338339f + 0.792043434e-1f * x) * x) * x;
03984       //
03985       // 0.000107046256 error, which is 13 to 14 bits
03986       SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X,
03987                                getF32Constant(DAG, 0x3da235e3));
03988       SDValue t3 = DAG.getNode(ISD::FADD, dl, MVT::f32, t2,
03989                                getF32Constant(DAG, 0x3e65b8f3));
03990       SDValue t4 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t3, X);
03991       SDValue t5 = DAG.getNode(ISD::FADD, dl, MVT::f32, t4,
03992                                getF32Constant(DAG, 0x3f324b07));
03993       SDValue t6 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t5, X);
03994       TwoToFracPartOfX = DAG.getNode(ISD::FADD, dl, MVT::f32, t6,
03995                                      getF32Constant(DAG, 0x3f7ff8fd));
03996     } else { // LimitFloatPrecision <= 18
03997       // For floating-point precision of 18:
03998       //
03999       //   TwoToFractionalPartOfX =
04000       //     0.999999982f +
04001       //       (0.693148872f +
04002       //         (0.240227044f +
04003       //           (0.554906021e-1f +
04004       //             (0.961591928e-2f +
04005       //               (0.136028312e-2f + 0.157059148e-3f *x)*x)*x)*x)*x)*x;
04006       //
04007       // error 2.47208000*10^(-7), which is better than 18 bits
04008       SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X,
04009                                getF32Constant(DAG, 0x3924b03e));
04010       SDValue t3 = DAG.getNode(ISD::FADD, dl, MVT::f32, t2,
04011                                getF32Constant(DAG, 0x3ab24b87));
04012       SDValue t4 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t3, X);
04013       SDValue t5 = DAG.getNode(ISD::FADD, dl, MVT::f32, t4,
04014                                getF32Constant(DAG, 0x3c1d8c17));
04015       SDValue t6 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t5, X);
04016       SDValue t7 = DAG.getNode(ISD::FADD, dl, MVT::f32, t6,
04017                                getF32Constant(DAG, 0x3d634a1d));
04018       SDValue t8 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t7, X);
04019       SDValue t9 = DAG.getNode(ISD::FADD, dl, MVT::f32, t8,
04020                                getF32Constant(DAG, 0x3e75fe14));
04021       SDValue t10 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t9, X);
04022       SDValue t11 = DAG.getNode(ISD::FADD, dl, MVT::f32, t10,
04023                                 getF32Constant(DAG, 0x3f317234));
04024       SDValue t12 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t11, X);
04025       TwoToFracPartOfX = DAG.getNode(ISD::FADD, dl, MVT::f32, t12,
04026                                      getF32Constant(DAG, 0x3f800000));
04027     }
04028 
04029     // Add the exponent into the result in integer domain.
04030     SDValue t13 = DAG.getNode(ISD::BITCAST, dl, MVT::i32, TwoToFracPartOfX);
04031     return DAG.getNode(ISD::BITCAST, dl, MVT::f32,
04032                        DAG.getNode(ISD::ADD, dl, MVT::i32,
04033                                    t13, IntegerPartOfX));
04034   }
04035 
04036   // No special expansion.
04037   return DAG.getNode(ISD::FEXP, dl, Op.getValueType(), Op);
04038 }
04039 
04040 /// expandLog - Lower a log intrinsic. Handles the special sequences for
04041 /// limited-precision mode.
04042 static SDValue expandLog(SDLoc dl, SDValue Op, SelectionDAG &DAG,
04043                          const TargetLowering &TLI) {
04044   if (Op.getValueType() == MVT::f32 &&
04045       LimitFloatPrecision > 0 && LimitFloatPrecision <= 18) {
04046     SDValue Op1 = DAG.getNode(ISD::BITCAST, dl, MVT::i32, Op);
04047 
04048     // Scale the exponent by log(2) [0.69314718f].
04049     SDValue Exp = GetExponent(DAG, Op1, TLI, dl);
04050     SDValue LogOfExponent = DAG.getNode(ISD::FMUL, dl, MVT::f32, Exp,
04051                                         getF32Constant(DAG, 0x3f317218));
04052 
04053     // Get the significand and build it into a floating-point number with
04054     // exponent of 1.
04055     SDValue X = GetSignificand(DAG, Op1, dl);
04056 
04057     SDValue LogOfMantissa;
04058     if (LimitFloatPrecision <= 6) {
04059       // For floating-point precision of 6:
04060       //
04061       //   LogofMantissa =
04062       //     -1.1609546f +
04063       //       (1.4034025f - 0.23903021f * x) * x;
04064       //
04065       // error 0.0034276066, which is better than 8 bits
04066       SDValue t0 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X,
04067                                getF32Constant(DAG, 0xbe74c456));
04068       SDValue t1 = DAG.getNode(ISD::FADD, dl, MVT::f32, t0,
04069                                getF32Constant(DAG, 0x3fb3a2b1));
04070       SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t1, X);
04071       LogOfMantissa = DAG.getNode(ISD::FSUB, dl, MVT::f32, t2,
04072                                   getF32Constant(DAG, 0x3f949a29));
04073     } else if (LimitFloatPrecision <= 12) {
04074       // For floating-point precision of 12:
04075       //
04076       //   LogOfMantissa =
04077       //     -1.7417939f +
04078       //       (2.8212026f +
04079       //         (-1.4699568f +
04080       //           (0.44717955f - 0.56570851e-1f * x) * x) * x) * x;
04081       //
04082       // error 0.000061011436, which is 14 bits
04083       SDValue t0 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X,
04084                                getF32Constant(DAG, 0xbd67b6d6));
04085       SDValue t1 = DAG.getNode(ISD::FADD, dl, MVT::f32, t0,
04086                                getF32Constant(DAG, 0x3ee4f4b8));
04087       SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t1, X);
04088       SDValue t3 = DAG.getNode(ISD::FSUB, dl, MVT::f32, t2,
04089                                getF32Constant(DAG, 0x3fbc278b));
04090       SDValue t4 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t3, X);
04091       SDValue t5 = DAG.getNode(ISD::FADD, dl, MVT::f32, t4,
04092                                getF32Constant(DAG, 0x40348e95));
04093       SDValue t6 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t5, X);
04094       LogOfMantissa = DAG.getNode(ISD::FSUB, dl, MVT::f32, t6,
04095                                   getF32Constant(DAG, 0x3fdef31a));
04096     } else { // LimitFloatPrecision <= 18
04097       // For floating-point precision of 18:
04098       //
04099       //   LogOfMantissa =
04100       //     -2.1072184f +
04101       //       (4.2372794f +
04102       //         (-3.7029485f +
04103       //           (2.2781945f +
04104       //             (-0.87823314f +
04105       //               (0.19073739f - 0.17809712e-1f * x) * x) * x) * x) * x)*x;
04106       //
04107       // error 0.0000023660568, which is better than 18 bits
04108       SDValue t0 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X,
04109                                getF32Constant(DAG, 0xbc91e5ac));
04110       SDValue t1 = DAG.getNode(ISD::FADD, dl, MVT::f32, t0,
04111                                getF32Constant(DAG, 0x3e4350aa));
04112       SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t1, X);
04113       SDValue t3 = DAG.getNode(ISD::FSUB, dl, MVT::f32, t2,
04114                                getF32Constant(DAG, 0x3f60d3e3));
04115       SDValue t4 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t3, X);
04116       SDValue t5 = DAG.getNode(ISD::FADD, dl, MVT::f32, t4,
04117                                getF32Constant(DAG, 0x4011cdf0));
04118       SDValue t6 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t5, X);
04119       SDValue t7 = DAG.getNode(ISD::FSUB, dl, MVT::f32, t6,
04120                                getF32Constant(DAG, 0x406cfd1c));
04121       SDValue t8 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t7, X);
04122       SDValue t9 = DAG.getNode(ISD::FADD, dl, MVT::f32, t8,
04123                                getF32Constant(DAG, 0x408797cb));
04124       SDValue t10 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t9, X);
04125       LogOfMantissa = DAG.getNode(ISD::FSUB, dl, MVT::f32, t10,
04126                                   getF32Constant(DAG, 0x4006dcab));
04127     }
04128 
04129     return DAG.getNode(ISD::FADD, dl, MVT::f32, LogOfExponent, LogOfMantissa);
04130   }
04131 
04132   // No special expansion.
04133   return DAG.getNode(ISD::FLOG, dl, Op.getValueType(), Op);
04134 }
04135 
04136 /// expandLog2 - Lower a log2 intrinsic. Handles the special sequences for
04137 /// limited-precision mode.
04138 static SDValue expandLog2(SDLoc dl, SDValue Op, SelectionDAG &DAG,
04139                           const TargetLowering &TLI) {
04140   if (Op.getValueType() == MVT::f32 &&
04141       LimitFloatPrecision > 0 && LimitFloatPrecision <= 18) {
04142     SDValue Op1 = DAG.getNode(ISD::BITCAST, dl, MVT::i32, Op);
04143 
04144     // Get the exponent.
04145     SDValue LogOfExponent = GetExponent(DAG, Op1, TLI, dl);
04146 
04147     // Get the significand and build it into a floating-point number with
04148     // exponent of 1.
04149     SDValue X = GetSignificand(DAG, Op1, dl);
04150 
04151     // Different possible minimax approximations of significand in
04152     // floating-point for various degrees of accuracy over [1,2].
04153     SDValue Log2ofMantissa;
04154     if (LimitFloatPrecision <= 6) {
04155       // For floating-point precision of 6:
04156       //
04157       //   Log2ofMantissa = -1.6749035f + (2.0246817f - .34484768f * x) * x;
04158       //
04159       // error 0.0049451742, which is more than 7 bits
04160       SDValue t0 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X,
04161                                getF32Constant(DAG, 0xbeb08fe0));
04162       SDValue t1 = DAG.getNode(ISD::FADD, dl, MVT::f32, t0,
04163                                getF32Constant(DAG, 0x40019463));
04164       SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t1, X);
04165       Log2ofMantissa = DAG.getNode(ISD::FSUB, dl, MVT::f32, t2,
04166                                    getF32Constant(DAG, 0x3fd6633d));
04167     } else if (LimitFloatPrecision <= 12) {
04168       // For floating-point precision of 12:
04169       //
04170       //   Log2ofMantissa =
04171       //     -2.51285454f +
04172       //       (4.07009056f +
04173       //         (-2.12067489f +
04174       //           (.645142248f - 0.816157886e-1f * x) * x) * x) * x;
04175       //
04176       // error 0.0000876136000, which is better than 13 bits
04177       SDValue t0 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X,
04178                                getF32Constant(DAG, 0xbda7262e));
04179       SDValue t1 = DAG.getNode(ISD::FADD, dl, MVT::f32, t0,
04180                                getF32Constant(DAG, 0x3f25280b));
04181       SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t1, X);
04182       SDValue t3 = DAG.getNode(ISD::FSUB, dl, MVT::f32, t2,
04183                                getF32Constant(DAG, 0x4007b923));
04184       SDValue t4 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t3, X);
04185       SDValue t5 = DAG.getNode(ISD::FADD, dl, MVT::f32, t4,
04186                                getF32Constant(DAG, 0x40823e2f));
04187       SDValue t6 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t5, X);
04188       Log2ofMantissa = DAG.getNode(ISD::FSUB, dl, MVT::f32, t6,
04189                                    getF32Constant(DAG, 0x4020d29c));
04190     } else { // LimitFloatPrecision <= 18
04191       // For floating-point precision of 18:
04192       //
04193       //   Log2ofMantissa =
04194       //     -3.0400495f +
04195       //       (6.1129976f +
04196       //         (-5.3420409f +
04197       //           (3.2865683f +
04198       //             (-1.2669343f +
04199       //               (0.27515199f -
04200       //                 0.25691327e-1f * x) * x) * x) * x) * x) * x;
04201       //
04202       // error 0.0000018516, which is better than 18 bits
04203       SDValue t0 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X,
04204                                getF32Constant(DAG, 0xbcd2769e));
04205       SDValue t1 = DAG.getNode(ISD::FADD, dl, MVT::f32, t0,
04206                                getF32Constant(DAG, 0x3e8ce0b9));
04207       SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t1, X);
04208       SDValue t3 = DAG.getNode(ISD::FSUB, dl, MVT::f32, t2,
04209                                getF32Constant(DAG, 0x3fa22ae7));
04210       SDValue t4 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t3, X);
04211       SDValue t5 = DAG.getNode(ISD::FADD, dl, MVT::f32, t4,
04212                                getF32Constant(DAG, 0x40525723));
04213       SDValue t6 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t5, X);
04214       SDValue t7 = DAG.getNode(ISD::FSUB, dl, MVT::f32, t6,
04215                                getF32Constant(DAG, 0x40aaf200));
04216       SDValue t8 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t7, X);
04217       SDValue t9 = DAG.getNode(ISD::FADD, dl, MVT::f32, t8,
04218                                getF32Constant(DAG, 0x40c39dad));
04219       SDValue t10 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t9, X);
04220       Log2ofMantissa = DAG.getNode(ISD::FSUB, dl, MVT::f32, t10,
04221                                    getF32Constant(DAG, 0x4042902c));
04222     }
04223 
04224     return DAG.getNode(ISD::FADD, dl, MVT::f32, LogOfExponent, Log2ofMantissa);
04225   }
04226 
04227   // No special expansion.
04228   return DAG.getNode(ISD::FLOG2, dl, Op.getValueType(), Op);
04229 }
04230 
04231 /// expandLog10 - Lower a log10 intrinsic. Handles the special sequences for
04232 /// limited-precision mode.
04233 static SDValue expandLog10(SDLoc dl, SDValue Op, SelectionDAG &DAG,
04234                            const TargetLowering &TLI) {
04235   if (Op.getValueType() == MVT::f32 &&
04236       LimitFloatPrecision > 0 && LimitFloatPrecision <= 18) {
04237     SDValue Op1 = DAG.getNode(ISD::BITCAST, dl, MVT::i32, Op);
04238 
04239     // Scale the exponent by log10(2) [0.30102999f].
04240     SDValue Exp = GetExponent(DAG, Op1, TLI, dl);
04241     SDValue LogOfExponent = DAG.getNode(ISD::FMUL, dl, MVT::f32, Exp,
04242                                         getF32Constant(DAG, 0x3e9a209a));
04243 
04244     // Get the significand and build it into a floating-point number with
04245     // exponent of 1.
04246     SDValue X = GetSignificand(DAG, Op1, dl);
04247 
04248     SDValue Log10ofMantissa;
04249     if (LimitFloatPrecision <= 6) {
04250       // For floating-point precision of 6:
04251       //
04252       //   Log10ofMantissa =
04253       //     -0.50419619f +
04254       //       (0.60948995f - 0.10380950f * x) * x;
04255       //
04256       // error 0.0014886165, which is 6 bits
04257       SDValue t0 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X,
04258                                getF32Constant(DAG, 0xbdd49a13));
04259       SDValue t1 = DAG.getNode(ISD::FADD, dl, MVT::f32, t0,
04260                                getF32Constant(DAG, 0x3f1c0789));
04261       SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t1, X);
04262       Log10ofMantissa = DAG.getNode(ISD::FSUB, dl, MVT::f32, t2,
04263                                     getF32Constant(DAG, 0x3f011300));
04264     } else if (LimitFloatPrecision <= 12) {
04265       // For floating-point precision of 12:
04266       //
04267       //   Log10ofMantissa =
04268       //     -0.64831180f +
04269       //       (0.91751397f +
04270       //         (-0.31664806f + 0.47637168e-1f * x) * x) * x;
04271       //
04272       // error 0.00019228036, which is better than 12 bits
04273       SDValue t0 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X,
04274                                getF32Constant(DAG, 0x3d431f31));
04275       SDValue t1 = DAG.getNode(ISD::FSUB, dl, MVT::f32, t0,
04276                                getF32Constant(DAG, 0x3ea21fb2));
04277       SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t1, X);
04278       SDValue t3 = DAG.getNode(ISD::FADD, dl, MVT::f32, t2,
04279                                getF32Constant(DAG, 0x3f6ae232));
04280       SDValue t4 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t3, X);
04281       Log10ofMantissa = DAG.getNode(ISD::FSUB, dl, MVT::f32, t4,
04282                                     getF32Constant(DAG, 0x3f25f7c3));
04283     } else { // LimitFloatPrecision <= 18
04284       // For floating-point precision of 18:
04285       //
04286       //   Log10ofMantissa =
04287       //     -0.84299375f +
04288       //       (1.5327582f +
04289       //         (-1.0688956f +
04290       //           (0.49102474f +
04291       //             (-0.12539807f + 0.13508273e-1f * x) * x) * x) * x) * x;
04292       //
04293       // error 0.0000037995730, which is better than 18 bits
04294       SDValue t0 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X,
04295                                getF32Constant(DAG, 0x3c5d51ce));
04296       SDValue t1 = DAG.getNode(ISD::FSUB, dl, MVT::f32, t0,
04297                                getF32Constant(DAG, 0x3e00685a));
04298       SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t1, X);
04299       SDValue t3 = DAG.getNode(ISD::FADD, dl, MVT::f32, t2,
04300                                getF32Constant(DAG, 0x3efb6798));
04301       SDValue t4 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t3, X);
04302       SDValue t5 = DAG.getNode(ISD::FSUB, dl, MVT::f32, t4,
04303                                getF32Constant(DAG, 0x3f88d192));
04304       SDValue t6 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t5, X);
04305       SDValue t7 = DAG.getNode(ISD::FADD, dl, MVT::f32, t6,
04306                                getF32Constant(DAG, 0x3fc4316c));
04307       SDValue t8 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t7, X);
04308       Log10ofMantissa = DAG.getNode(ISD::FSUB, dl, MVT::f32, t8,
04309                                     getF32Constant(DAG, 0x3f57ce70));
04310     }
04311 
04312     return DAG.getNode(ISD::FADD, dl, MVT::f32, LogOfExponent, Log10ofMantissa);
04313   }
04314 
04315   // No special expansion.
04316   return DAG.getNode(ISD::FLOG10, dl, Op.getValueType(), Op);
04317 }
04318 
04319 /// expandExp2 - Lower an exp2 intrinsic. Handles the special sequences for
04320 /// limited-precision mode.
04321 static SDValue expandExp2(SDLoc dl, SDValue Op, SelectionDAG &DAG,
04322                           const TargetLowering &TLI) {
04323   if (Op.getValueType() == MVT::f32 &&
04324       LimitFloatPrecision > 0 && LimitFloatPrecision <= 18) {
04325     SDValue IntegerPartOfX = DAG.getNode(ISD::FP_TO_SINT, dl, MVT::i32, Op);
04326 
04327     //   FractionalPartOfX = x - (float)IntegerPartOfX;
04328     SDValue t1 = DAG.getNode(ISD::SINT_TO_FP, dl, MVT::f32, IntegerPartOfX);
04329     SDValue X = DAG.getNode(ISD::FSUB, dl, MVT::f32, Op, t1);
04330 
04331     //   IntegerPartOfX <<= 23;
04332     IntegerPartOfX = DAG.getNode(ISD::SHL, dl, MVT::i32, IntegerPartOfX,
04333                                  DAG.getConstant(23, TLI.getPointerTy()));
04334 
04335     SDValue TwoToFractionalPartOfX;
04336     if (LimitFloatPrecision <= 6) {
04337       // For floating-point precision of 6:
04338       //
04339       //   TwoToFractionalPartOfX =
04340       //     0.997535578f +
04341       //       (0.735607626f + 0.252464424f * x) * x;
04342       //
04343       // error 0.0144103317, which is 6 bits
04344       SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X,
04345                                getF32Constant(DAG, 0x3e814304));
04346       SDValue t3 = DAG.getNode(ISD::FADD, dl, MVT::f32, t2,
04347                                getF32Constant(DAG, 0x3f3c50c8));
04348       SDValue t4 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t3, X);
04349       TwoToFractionalPartOfX = DAG.getNode(ISD::FADD, dl, MVT::f32, t4,
04350                                            getF32Constant(DAG, 0x3f7f5e7e));
04351     } else if (LimitFloatPrecision <= 12) {
04352       // For floating-point precision of 12:
04353       //
04354       //   TwoToFractionalPartOfX =
04355       //     0.999892986f +
04356       //       (0.696457318f +
04357       //         (0.224338339f + 0.792043434e-1f * x) * x) * x;
04358       //
04359       // error 0.000107046256, which is 13 to 14 bits
04360       SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X,
04361                                getF32Constant(DAG, 0x3da235e3));
04362       SDValue t3 = DAG.getNode(ISD::FADD, dl, MVT::f32, t2,
04363                                getF32Constant(DAG, 0x3e65b8f3));
04364       SDValue t4 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t3, X);
04365       SDValue t5 = DAG.getNode(ISD::FADD, dl, MVT::f32, t4,
04366                                getF32Constant(DAG, 0x3f324b07));
04367       SDValue t6 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t5, X);
04368       TwoToFractionalPartOfX = DAG.getNode(ISD::FADD, dl, MVT::f32, t6,
04369                                            getF32Constant(DAG, 0x3f7ff8fd));
04370     } else { // LimitFloatPrecision <= 18
04371       // For floating-point precision of 18:
04372       //
04373       //   TwoToFractionalPartOfX =
04374       //     0.999999982f +
04375       //       (0.693148872f +
04376       //         (0.240227044f +
04377       //           (0.554906021e-1f +
04378       //             (0.961591928e-2f +
04379       //               (0.136028312e-2f + 0.157059148e-3f *x)*x)*x)*x)*x)*x;
04380       // error 2.47208000*10^(-7), which is better than 18 bits
04381       SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X,
04382                                getF32Constant(DAG, 0x3924b03e));
04383       SDValue t3 = DAG.getNode(ISD::FADD, dl, MVT::f32, t2,
04384                                getF32Constant(DAG, 0x3ab24b87));
04385       SDValue t4 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t3, X);
04386       SDValue t5 = DAG.getNode(ISD::FADD, dl, MVT::f32, t4,
04387                                getF32Constant(DAG, 0x3c1d8c17));
04388       SDValue t6 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t5, X);
04389       SDValue t7 = DAG.getNode(ISD::FADD, dl, MVT::f32, t6,
04390                                getF32Constant(DAG, 0x3d634a1d));
04391       SDValue t8 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t7, X);
04392       SDValue t9 = DAG.getNode(ISD::FADD, dl, MVT::f32, t8,
04393                                getF32Constant(DAG, 0x3e75fe14));
04394       SDValue t10 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t9, X);
04395       SDValue t11 = DAG.getNode(ISD::FADD, dl, MVT::f32, t10,
04396                                 getF32Constant(DAG, 0x3f317234));
04397       SDValue t12 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t11, X);
04398       TwoToFractionalPartOfX = DAG.getNode(ISD::FADD, dl, MVT::f32, t12,
04399                                            getF32Constant(DAG, 0x3f800000));
04400     }
04401 
04402     // Add the exponent into the result in integer domain.
04403     SDValue t13 = DAG.getNode(ISD::BITCAST, dl, MVT::i32,
04404                               TwoToFractionalPartOfX);
04405     return DAG.getNode(ISD::BITCAST, dl, MVT::f32,
04406                        DAG.getNode(ISD::ADD, dl, MVT::i32,
04407                                    t13, IntegerPartOfX));
04408   }
04409 
04410   // No special expansion.
04411   return DAG.getNode(ISD::FEXP2, dl, Op.getValueType(), Op);
04412 }
04413 
04414 /// visitPow - Lower a pow intrinsic. Handles the special sequences for
04415 /// limited-precision mode with x == 10.0f.
04416 static SDValue expandPow(SDLoc dl, SDValue LHS, SDValue RHS,
04417                          SelectionDAG &DAG, const TargetLowering &TLI) {
04418   bool IsExp10 = false;
04419   if (LHS.getValueType() == MVT::f32 && RHS.getValueType() == MVT::f32 &&
04420       LimitFloatPrecision > 0 && LimitFloatPrecision <= 18) {
04421     if (ConstantFPSDNode *LHSC = dyn_cast<ConstantFPSDNode>(LHS)) {
04422       APFloat Ten(10.0f);
04423       IsExp10 = LHSC->isExactlyValue(Ten);
04424     }
04425   }
04426 
04427   if (IsExp10) {
04428     // Put the exponent in the right bit position for later addition to the
04429     // final result:
04430     //
04431     //   #define LOG2OF10 3.3219281f
04432     //   IntegerPartOfX = (int32_t)(x * LOG2OF10);
04433     SDValue t0 = DAG.getNode(ISD::FMUL, dl, MVT::f32, RHS,
04434                              getF32Constant(DAG, 0x40549a78));
04435     SDValue IntegerPartOfX = DAG.getNode(ISD::FP_TO_SINT, dl, MVT::i32, t0);
04436 
04437     //   FractionalPartOfX = x - (float)IntegerPartOfX;
04438     SDValue t1 = DAG.getNode(ISD::SINT_TO_FP, dl, MVT::f32, IntegerPartOfX);
04439     SDValue X = DAG.getNode(ISD::FSUB, dl, MVT::f32, t0, t1);
04440 
04441     //   IntegerPartOfX <<= 23;
04442     IntegerPartOfX = DAG.getNode(ISD::SHL, dl, MVT::i32, IntegerPartOfX,
04443                                  DAG.getConstant(23, TLI.getPointerTy()));
04444 
04445     SDValue TwoToFractionalPartOfX;
04446     if (LimitFloatPrecision <= 6) {
04447       // For floating-point precision of 6:
04448       //
04449       //   twoToFractionalPartOfX =
04450       //     0.997535578f +
04451       //       (0.735607626f + 0.252464424f * x) * x;
04452       //
04453       // error 0.0144103317, which is 6 bits
04454       SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X,
04455                                getF32Constant(DAG, 0x3e814304));
04456       SDValue t3 = DAG.getNode(ISD::FADD, dl, MVT::f32, t2,
04457                                getF32Constant(DAG, 0x3f3c50c8));
04458       SDValue t4 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t3, X);
04459       TwoToFractionalPartOfX = DAG.getNode(ISD::FADD, dl, MVT::f32, t4,
04460                                            getF32Constant(DAG, 0x3f7f5e7e));
04461     } else if (LimitFloatPrecision <= 12) {
04462       // For floating-point precision of 12:
04463       //
04464       //   TwoToFractionalPartOfX =
04465       //     0.999892986f +
04466       //       (0.696457318f +
04467       //         (0.224338339f + 0.792043434e-1f * x) * x) * x;
04468       //
04469       // error 0.000107046256, which is 13 to 14 bits
04470       SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X,
04471                                getF32Constant(DAG, 0x3da235e3));
04472       SDValue t3 = DAG.getNode(ISD::FADD, dl, MVT::f32, t2,
04473                                getF32Constant(DAG, 0x3e65b8f3));
04474       SDValue t4 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t3, X);
04475       SDValue t5 = DAG.getNode(ISD::FADD, dl, MVT::f32, t4,
04476                                getF32Constant(DAG, 0x3f324b07));
04477       SDValue t6 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t5, X);
04478       TwoToFractionalPartOfX = DAG.getNode(ISD::FADD, dl, MVT::f32, t6,
04479                                            getF32Constant(DAG, 0x3f7ff8fd));
04480     } else { // LimitFloatPrecision <= 18
04481       // For floating-point precision of 18:
04482       //
04483       //   TwoToFractionalPartOfX =
04484       //     0.999999982f +
04485       //       (0.693148872f +
04486       //         (0.240227044f +
04487       //           (0.554906021e-1f +
04488       //             (0.961591928e-2f +
04489       //               (0.136028312e-2f + 0.157059148e-3f *x)*x)*x)*x)*x)*x;
04490       // error 2.47208000*10^(-7), which is better than 18 bits
04491       SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X,
04492                                getF32Constant(DAG, 0x3924b03e));
04493       SDValue t3 = DAG.getNode(ISD::FADD, dl, MVT::f32, t2,
04494                                getF32Constant(DAG, 0x3ab24b87));
04495       SDValue t4 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t3, X);
04496       SDValue t5 = DAG.getNode(ISD::FADD, dl, MVT::f32, t4,
04497                                getF32Constant(DAG, 0x3c1d8c17));
04498       SDValue t6 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t5, X);
04499       SDValue t7 = DAG.getNode(ISD::FADD, dl, MVT::f32, t6,
04500                                getF32Constant(DAG, 0x3d634a1d));
04501       SDValue t8 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t7, X);
04502       SDValue t9 = DAG.getNode(ISD::FADD, dl, MVT::f32, t8,
04503                                getF32Constant(DAG, 0x3e75fe14));
04504       SDValue t10 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t9, X);
04505       SDValue t11 = DAG.getNode(ISD::FADD, dl, MVT::f32, t10,
04506                                 getF32Constant(DAG, 0x3f317234));
04507       SDValue t12 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t11, X);
04508       TwoToFractionalPartOfX = DAG.getNode(ISD::FADD, dl, MVT::f32, t12,
04509                                            getF32Constant(DAG, 0x3f800000));
04510     }
04511 
04512     SDValue t13 = DAG.getNode(ISD::BITCAST, dl,MVT::i32,TwoToFractionalPartOfX);
04513     return DAG.getNode(ISD::BITCAST, dl, MVT::f32,
04514                        DAG.getNode(ISD::ADD, dl, MVT::i32,
04515                                    t13, IntegerPartOfX));
04516   }
04517 
04518   // No special expansion.
04519   return DAG.getNode(ISD::FPOW, dl, LHS.getValueType(), LHS, RHS);
04520 }
04521 
04522 
04523 /// ExpandPowI - Expand a llvm.powi intrinsic.
04524 static SDValue ExpandPowI(SDLoc DL, SDValue LHS, SDValue RHS,
04525                           SelectionDAG &DAG) {
04526   // If RHS is a constant, we can expand this out to a multiplication tree,
04527   // otherwise we end up lowering to a call to __powidf2 (for example).  When
04528   // optimizing for size, we only want to do this if the expansion would produce
04529   // a small number of multiplies, otherwise we do the full expansion.
04530   if (ConstantSDNode *RHSC = dyn_cast<ConstantSDNode>(RHS)) {
04531     // Get the exponent as a positive value.
04532     unsigned Val = RHSC->getSExtValue();
04533     if ((int)Val < 0) Val = -Val;
04534 
04535     // powi(x, 0) -> 1.0
04536     if (Val == 0)
04537       return DAG.getConstantFP(1.0, LHS.getValueType());
04538 
04539     const Function *F = DAG.getMachineFunction().getFunction();
04540     if (!F->getAttributes().hasAttribute(AttributeSet::FunctionIndex,
04541                                          Attribute::OptimizeForSize) ||
04542         // If optimizing for size, don't insert too many multiplies.  This
04543         // inserts up to 5 multiplies.
04544         CountPopulation_32(Val)+Log2_32(Val) < 7) {
04545       // We use the simple binary decomposition method to generate the multiply
04546       // sequence.  There are more optimal ways to do this (for example,
04547       // powi(x,15) generates one more multiply than it should), but this has
04548       // the benefit of being both really simple and much better than a libcall.
04549       SDValue Res;  // Logically starts equal to 1.0
04550       SDValue CurSquare = LHS;
04551       while (Val) {
04552         if (Val & 1) {
04553           if (Res.getNode())
04554             Res = DAG.getNode(ISD::FMUL, DL,Res.getValueType(), Res, CurSquare);
04555           else
04556             Res = CurSquare;  // 1.0*CurSquare.
04557         }
04558 
04559         CurSquare = DAG.getNode(ISD::FMUL, DL, CurSquare.getValueType(),
04560                                 CurSquare, CurSquare);
04561         Val >>= 1;
04562       }
04563 
04564       // If the original was negative, invert the result, producing 1/(x*x*x).
04565       if (RHSC->getSExtValue() < 0)
04566         Res = DAG.getNode(ISD::FDIV, DL, LHS.getValueType(),
04567                           DAG.getConstantFP(1.0, LHS.getValueType()), Res);
04568       return Res;
04569     }
04570   }
04571 
04572   // Otherwise, expand to a libcall.
04573   return DAG.getNode(ISD::FPOWI, DL, LHS.getValueType(), LHS, RHS);
04574 }
04575 
04576 // getTruncatedArgReg - Find underlying register used for an truncated
04577 // argument.
04578 static unsigned getTruncatedArgReg(const SDValue &N) {
04579   if (N.getOpcode() != ISD::TRUNCATE)
04580     return 0;
04581 
04582   const SDValue &Ext = N.getOperand(0);
04583   if (Ext.getOpcode() == ISD::AssertZext ||
04584       Ext.getOpcode() == ISD::AssertSext) {
04585     const SDValue &CFR = Ext.getOperand(0);
04586     if (CFR.getOpcode() == ISD::CopyFromReg)
04587       return cast<RegisterSDNode>(CFR.getOperand(1))->getReg();
04588     if (CFR.getOpcode() == ISD::TRUNCATE)
04589       return getTruncatedArgReg(CFR);
04590   }
04591   return 0;
04592 }
04593 
04594 /// EmitFuncArgumentDbgValue - If the DbgValueInst is a dbg_value of a function
04595 /// argument, create the corresponding DBG_VALUE machine instruction for it now.
04596 /// At the end of instruction selection, they will be inserted to the entry BB.
04597 bool
04598 SelectionDAGBuilder::EmitFuncArgumentDbgValue(const Value *V, MDNode *Variable,
04599                                               int64_t Offset, bool IsIndirect,
04600                                               const SDValue &N) {
04601   const Argument *Arg = dyn_cast<Argument>(V);
04602   if (!Arg)
04603     return false;
04604 
04605   MachineFunction &MF = DAG.getMachineFunction();
04606   const TargetInstrInfo *TII = DAG.getSubtarget().getInstrInfo();
04607 
04608   // Ignore inlined function arguments here.
04609   DIVariable DV(Variable);
04610   if (DV.isInlinedFnArgument(MF.getFunction()))
04611     return false;
04612 
04613   Optional<MachineOperand> Op;
04614   // Some arguments' frame index is recorded during argument lowering.
04615   if (int FI = FuncInfo.getArgumentFrameIndex(Arg))
04616     Op = MachineOperand::CreateFI(FI);
04617 
04618   if (!Op && N.getNode()) {
04619     unsigned Reg;
04620     if (N.getOpcode() == ISD::CopyFromReg)
04621       Reg = cast<RegisterSDNode>(N.getOperand(1))->getReg();
04622     else
04623       Reg = getTruncatedArgReg(N);
04624     if (Reg && TargetRegisterInfo::isVirtualRegister(Reg)) {
04625       MachineRegisterInfo &RegInfo = MF.getRegInfo();
04626       unsigned PR = RegInfo.getLiveInPhysReg(Reg);
04627       if (PR)
04628         Reg = PR;
04629     }
04630     if (Reg)
04631       Op = MachineOperand::CreateReg(Reg, false);
04632   }
04633 
04634   if (!Op) {
04635     // Check if ValueMap has reg number.
04636     DenseMap<const Value *, unsigned>::iterator VMI = FuncInfo.ValueMap.find(V);
04637     if (VMI != FuncInfo.ValueMap.end())
04638       Op = MachineOperand::CreateReg(VMI->second, false);
04639   }
04640 
04641   if (!Op && N.getNode())
04642     // Check if frame index is available.
04643     if (LoadSDNode *LNode = dyn_cast<LoadSDNode>(N.getNode()))
04644       if (FrameIndexSDNode *FINode =
04645           dyn_cast<FrameIndexSDNode>(LNode->getBasePtr().getNode()))
04646         Op = MachineOperand::CreateFI(FINode->getIndex());
04647 
04648   if (!Op)
04649     return false;
04650 
04651   if (Op->isReg())
04652     FuncInfo.ArgDbgValues.push_back(BuildMI(MF, getCurDebugLoc(),
04653                                             TII->get(TargetOpcode::DBG_VALUE),
04654                                             IsIndirect,
04655                                             Op->getReg(), Offset, Variable));
04656   else
04657     FuncInfo.ArgDbgValues.push_back(
04658       BuildMI(MF, getCurDebugLoc(), TII->get(TargetOpcode::DBG_VALUE))
04659           .addOperand(*Op).addImm(Offset).addMetadata(Variable));
04660 
04661   return true;
04662 }
04663 
04664 // VisualStudio defines setjmp as _setjmp
04665 #if defined(_MSC_VER) && defined(setjmp) && \
04666                          !defined(setjmp_undefined_for_msvc)
04667 #  pragma push_macro("setjmp")
04668 #  undef setjmp
04669 #  define setjmp_undefined_for_msvc
04670 #endif
04671 
04672 /// visitIntrinsicCall - Lower the call to the specified intrinsic function.  If
04673 /// we want to emit this as a call to a named external function, return the name
04674 /// otherwise lower it and return null.
04675 const char *
04676 SelectionDAGBuilder::visitIntrinsicCall(const CallInst &I, unsigned Intrinsic) {
04677   const TargetLowering *TLI = TM.getSubtargetImpl()->getTargetLowering();
04678   SDLoc sdl = getCurSDLoc();
04679   DebugLoc dl = getCurDebugLoc();
04680   SDValue Res;
04681 
04682   switch (Intrinsic) {
04683   default:
04684     // By default, turn this into a target intrinsic node.
04685     visitTargetIntrinsic(I, Intrinsic);
04686     return nullptr;
04687   case Intrinsic::vastart:  visitVAStart(I); return nullptr;
04688   case Intrinsic::vaend:    visitVAEnd(I); return nullptr;
04689   case Intrinsic::vacopy:   visitVACopy(I); return nullptr;
04690   case Intrinsic::returnaddress:
04691     setValue(&I, DAG.getNode(ISD::RETURNADDR, sdl, TLI->getPointerTy(),
04692                              getValue(I.getArgOperand(0))));
04693     return nullptr;
04694   case Intrinsic::frameaddress:
04695     setValue(&I, DAG.getNode(ISD::FRAMEADDR, sdl, TLI->getPointerTy(),
04696                              getValue(I.getArgOperand(0))));
04697     return nullptr;
04698   case Intrinsic::read_register: {
04699     Value *Reg = I.getArgOperand(0);
04700     SDValue RegName = DAG.getMDNode(cast<MDNode>(Reg));
04701     EVT VT =
04702         TM.getSubtargetImpl()->getTargetLowering()->getValueType(I.getType());
04703     setValue(&I, DAG.getNode(ISD::READ_REGISTER, sdl, VT, RegName));
04704     return nullptr;
04705   }
04706   case Intrinsic::write_register: {
04707     Value *Reg = I.getArgOperand(0);
04708     Value *RegValue = I.getArgOperand(1);
04709     SDValue Chain = getValue(RegValue).getOperand(0);
04710     SDValue RegName = DAG.getMDNode(cast<MDNode>(Reg));
04711     DAG.setRoot(DAG.getNode(ISD::WRITE_REGISTER, sdl, MVT::Other, Chain,
04712                             RegName, getValue(RegValue)));
04713     return nullptr;
04714   }
04715   case Intrinsic::setjmp:
04716     return &"_setjmp"[!TLI->usesUnderscoreSetJmp()];
04717   case Intrinsic::longjmp:
04718     return &"_longjmp"[!TLI->usesUnderscoreLongJmp()];
04719   case Intrinsic::memcpy: {
04720     // Assert for address < 256 since we support only user defined address
04721     // spaces.
04722     assert(cast<PointerType>(I.getArgOperand(0)->getType())->getAddressSpace()
04723            < 256 &&
04724            cast<PointerType>(I.getArgOperand(1)->getType())->getAddressSpace()
04725            < 256 &&
04726            "Unknown address space");
04727     SDValue Op1 = getValue(I.getArgOperand(0));
04728     SDValue Op2 = getValue(I.getArgOperand(1));
04729     SDValue Op3 = getValue(I.getArgOperand(2));
04730     unsigned Align = cast<ConstantInt>(I.getArgOperand(3))->getZExtValue();
04731     if (!Align)
04732       Align = 1; // @llvm.memcpy defines 0 and 1 to both mean no alignment.
04733     bool isVol = cast<ConstantInt>(I.getArgOperand(4))->getZExtValue();
04734     DAG.setRoot(DAG.getMemcpy(getRoot(), sdl, Op1, Op2, Op3, Align, isVol, false,
04735                               MachinePointerInfo(I.getArgOperand(0)),
04736                               MachinePointerInfo(I.getArgOperand(1))));
04737     return nullptr;
04738   }
04739   case Intrinsic::memset: {
04740     // Assert for address < 256 since we support only user defined address
04741     // spaces.
04742     assert(cast<PointerType>(I.getArgOperand(0)->getType())->getAddressSpace()
04743            < 256 &&
04744            "Unknown address space");
04745     SDValue Op1 = getValue(I.getArgOperand(0));
04746     SDValue Op2 = getValue(I.getArgOperand(1));
04747     SDValue Op3 = getValue(I.getArgOperand(2));
04748     unsigned Align = cast<ConstantInt>(I.getArgOperand(3))->getZExtValue();
04749     if (!Align)
04750       Align = 1; // @llvm.memset defines 0 and 1 to both mean no alignment.
04751     bool isVol = cast<ConstantInt>(I.getArgOperand(4))->getZExtValue();
04752     DAG.setRoot(DAG.getMemset(getRoot(), sdl, Op1, Op2, Op3, Align, isVol,
04753                               MachinePointerInfo(I.getArgOperand(0))));
04754     return nullptr;
04755   }
04756   case Intrinsic::memmove: {
04757     // Assert for address < 256 since we support only user defined address
04758     // spaces.
04759     assert(cast<PointerType>(I.getArgOperand(0)->getType())->getAddressSpace()
04760            < 256 &&
04761            cast<PointerType>(I.getArgOperand(1)->getType())->getAddressSpace()
04762            < 256 &&
04763            "Unknown address space");
04764     SDValue Op1 = getValue(I.getArgOperand(0));
04765     SDValue Op2 = getValue(I.getArgOperand(1));
04766     SDValue Op3 = getValue(I.getArgOperand(2));
04767     unsigned Align = cast<ConstantInt>(I.getArgOperand(3))->getZExtValue();
04768     if (!Align)
04769       Align = 1; // @llvm.memmove defines 0 and 1 to both mean no alignment.
04770     bool isVol = cast<ConstantInt>(I.getArgOperand(4))->getZExtValue();
04771     DAG.setRoot(DAG.getMemmove(getRoot(), sdl, Op1, Op2, Op3, Align, isVol,
04772                                MachinePointerInfo(I.getArgOperand(0)),
04773                                MachinePointerInfo(I.getArgOperand(1))));
04774     return nullptr;
04775   }
04776   case Intrinsic::dbg_declare: {
04777     const DbgDeclareInst &DI = cast<DbgDeclareInst>(I);
04778     MDNode *Variable = DI.getVariable();
04779     const Value *Address = DI.getAddress();
04780     DIVariable DIVar(Variable);
04781     assert((!DIVar || DIVar.isVariable()) &&
04782       "Variable in DbgDeclareInst should be either null or a DIVariable.");
04783     if (!Address || !DIVar) {
04784       DEBUG(dbgs() << "Dropping debug info for " << DI << "\n");
04785       return nullptr;
04786     }
04787 
04788     // Check if address has undef value.
04789     if (isa<UndefValue>(Address) ||
04790         (Address->use_empty() && !isa<Argument>(Address))) {
04791       DEBUG(dbgs() << "Dropping debug info for " << DI << "\n");
04792       return nullptr;
04793     }
04794 
04795     SDValue &N = NodeMap[Address];
04796     if (!N.getNode() && isa<Argument>(Address))
04797       // Check unused arguments map.
04798       N = UnusedArgNodeMap[Address];
04799     SDDbgValue *SDV;
04800     if (N.getNode()) {
04801       if (const BitCastInst *BCI = dyn_cast<BitCastInst>(Address))
04802         Address = BCI->getOperand(0);
04803       // Parameters are handled specially.
04804       bool isParameter =
04805         (DIVariable(Variable).getTag() == dwarf::DW_TAG_arg_variable ||
04806          isa<Argument>(Address));
04807 
04808       const AllocaInst *AI = dyn_cast<AllocaInst>(Address);
04809 
04810       if (isParameter && !AI) {
04811         FrameIndexSDNode *FINode = dyn_cast<FrameIndexSDNode>(N.getNode());
04812         if (FINode)
04813           // Byval parameter.  We have a frame index at this point.
04814           SDV = DAG.getFrameIndexDbgValue(Variable, FINode->getIndex(),
04815             0, dl, SDNodeOrder);
04816         else {
04817           // Address is an argument, so try to emit its dbg value using
04818           // virtual register info from the FuncInfo.ValueMap.
04819           EmitFuncArgumentDbgValue(Address, Variable, 0, false, N);
04820           return nullptr;
04821         }
04822       } else if (AI)
04823         SDV = DAG.getDbgValue(Variable, N.getNode(), N.getResNo(),
04824                               true, 0, dl, SDNodeOrder);
04825       else {
04826         // Can't do anything with other non-AI cases yet.
04827         DEBUG(dbgs() << "Dropping debug info for " << DI << "\n");
04828         DEBUG(dbgs() << "non-AllocaInst issue for Address: \n\t");
04829         DEBUG(Address->dump());
04830         return nullptr;
04831       }
04832       DAG.AddDbgValue(SDV, N.getNode(), isParameter);
04833     } else {
04834       // If Address is an argument then try to emit its dbg value using
04835       // virtual register info from the FuncInfo.ValueMap.
04836       if (!EmitFuncArgumentDbgValue(Address, Variable, 0, false, N)) {
04837         // If variable is pinned by a alloca in dominating bb then
04838         // use StaticAllocaMap.
04839         if (const AllocaInst *AI = dyn_cast<AllocaInst>(Address)) {
04840           if (AI->getParent() != DI.getParent()) {
04841             DenseMap<const AllocaInst*, int>::iterator SI =
04842               FuncInfo.StaticAllocaMap.find(AI);
04843             if (SI != FuncInfo.StaticAllocaMap.end()) {
04844               SDV = DAG.getFrameIndexDbgValue(Variable, SI->second,
04845                                               0, dl, SDNodeOrder);
04846               DAG.AddDbgValue(SDV, nullptr, false);
04847               return nullptr;
04848             }
04849           }
04850         }
04851         DEBUG(dbgs() << "Dropping debug info for " << DI << "\n");
04852       }
04853     }
04854     return nullptr;
04855   }
04856   case Intrinsic::dbg_value: {
04857     const DbgValueInst &DI = cast<DbgValueInst>(I);
04858     DIVariable DIVar(DI.getVariable());
04859     assert((!DIVar || DIVar.isVariable()) &&
04860       "Variable in DbgValueInst should be either null or a DIVariable.");
04861     if (!DIVar)
04862       return nullptr;
04863 
04864     MDNode *Variable = DI.getVariable();
04865     uint64_t Offset = DI.getOffset();
04866     const Value *V = DI.getValue();
04867     if (!V)
04868       return nullptr;
04869 
04870     SDDbgValue *SDV;
04871     if (isa<ConstantInt>(V) || isa<ConstantFP>(V) || isa<UndefValue>(V)) {
04872       SDV = DAG.getConstantDbgValue(Variable, V, Offset, dl, SDNodeOrder);
04873       DAG.AddDbgValue(SDV, nullptr, false);
04874     } else {
04875       // Do not use getValue() in here; we don't want to generate code at
04876       // this point if it hasn't been done yet.
04877       SDValue N = NodeMap[V];
04878       if (!N.getNode() && isa<Argument>(V))
04879         // Check unused arguments map.
04880         N = UnusedArgNodeMap[V];
04881       if (N.getNode()) {
04882         // A dbg.value for an alloca is always indirect.
04883         bool IsIndirect = isa<AllocaInst>(V) || Offset != 0;
04884         if (!EmitFuncArgumentDbgValue(V, Variable, Offset, IsIndirect, N)) {
04885           SDV = DAG.getDbgValue(Variable, N.getNode(),
04886                                 N.getResNo(), IsIndirect,
04887         Offset, dl, SDNodeOrder);
04888           DAG.AddDbgValue(SDV, N.getNode(), false);
04889         }
04890       } else if (!V->use_empty() ) {
04891         // Do not call getValue(V) yet, as we don't want to generate code.
04892         // Remember it for later.
04893         DanglingDebugInfo DDI(&DI, dl, SDNodeOrder);
04894         DanglingDebugInfoMap[V] = DDI;
04895       } else {
04896         // We may expand this to cover more cases.  One case where we have no
04897         // data available is an unreferenced parameter.
04898         DEBUG(dbgs() << "Dropping debug info for " << DI << "\n");
04899       }
04900     }
04901 
04902     // Build a debug info table entry.
04903     if (const BitCastInst *BCI = dyn_cast<BitCastInst>(V))
04904       V = BCI->getOperand(0);
04905     const AllocaInst *AI = dyn_cast<AllocaInst>(V);
04906     // Don't handle byval struct arguments or VLAs, for example.
04907     if (!AI) {
04908       DEBUG(dbgs() << "Dropping debug location info for:\n  " << DI << "\n");
04909       DEBUG(dbgs() << "  Last seen at:\n    " << *V << "\n");
04910       return nullptr;
04911     }
04912     DenseMap<const AllocaInst*, int>::iterator SI =
04913       FuncInfo.StaticAllocaMap.find(AI);
04914     if (SI == FuncInfo.StaticAllocaMap.end())
04915       return nullptr; // VLAs.
04916     return nullptr;
04917   }
04918 
04919   case Intrinsic::eh_typeid_for: {
04920     // Find the type id for the given typeinfo.
04921     GlobalVariable *GV = ExtractTypeInfo(I.getArgOperand(0));
04922     unsigned TypeID = DAG.getMachineFunction().getMMI().getTypeIDFor(GV);
04923     Res = DAG.getConstant(TypeID, MVT::i32);
04924     setValue(&I, Res);
04925     return nullptr;
04926   }
04927 
04928   case Intrinsic::eh_return_i32:
04929   case Intrinsic::eh_return_i64:
04930     DAG.getMachineFunction().getMMI().setCallsEHReturn(true);
04931     DAG.setRoot(DAG.getNode(ISD::EH_RETURN, sdl,
04932                             MVT::Other,
04933                             getControlRoot(),
04934                             getValue(I.getArgOperand(0)),
04935                             getValue(I.getArgOperand(1))));
04936     return nullptr;
04937   case Intrinsic::eh_unwind_init:
04938     DAG.getMachineFunction().getMMI().setCallsUnwindInit(true);
04939     return nullptr;
04940   case Intrinsic::eh_dwarf_cfa: {
04941     SDValue CfaArg = DAG.getSExtOrTrunc(getValue(I.getArgOperand(0)), sdl,
04942                                         TLI->getPointerTy());
04943     SDValue Offset = DAG.getNode(ISD::ADD, sdl,
04944                                  CfaArg.getValueType(),
04945                                  DAG.getNode(ISD::FRAME_TO_ARGS_OFFSET, sdl,
04946                                              CfaArg.getValueType()),
04947                                  CfaArg);
04948     SDValue FA = DAG.getNode(ISD::FRAMEADDR, sdl,
04949                              TLI->getPointerTy(),
04950                              DAG.getConstant(0, TLI->getPointerTy()));
04951     setValue(&I, DAG.getNode(ISD::ADD, sdl, FA.getValueType(),
04952                              FA, Offset));
04953     return nullptr;
04954   }
04955   case Intrinsic::eh_sjlj_callsite: {
04956     MachineModuleInfo &MMI = DAG.getMachineFunction().getMMI();
04957     ConstantInt *CI = dyn_cast<ConstantInt>(I.getArgOperand(0));
04958     assert(CI && "Non-constant call site value in eh.sjlj.callsite!");
04959     assert(MMI.getCurrentCallSite() == 0 && "Overlapping call sites!");
04960 
04961     MMI.setCurrentCallSite(CI->getZExtValue());
04962     return nullptr;
04963   }
04964   case Intrinsic::eh_sjlj_functioncontext: {
04965     // Get and store the index of the function context.
04966     MachineFrameInfo *MFI = DAG.getMachineFunction().getFrameInfo();
04967     AllocaInst *FnCtx =
04968       cast<AllocaInst>(I.getArgOperand(0)->stripPointerCasts());
04969     int FI = FuncInfo.StaticAllocaMap[FnCtx];
04970     MFI->setFunctionContextIndex(FI);
04971     return nullptr;
04972   }
04973   case Intrinsic::eh_sjlj_setjmp: {
04974     SDValue Ops[2];
04975     Ops[0] = getRoot();
04976     Ops[1] = getValue(I.getArgOperand(0));
04977     SDValue Op = DAG.getNode(ISD::EH_SJLJ_SETJMP, sdl,
04978                              DAG.getVTList(MVT::i32, MVT::Other), Ops);
04979     setValue(&I, Op.getValue(0));
04980     DAG.setRoot(Op.getValue(1));
04981     return nullptr;
04982   }
04983   case Intrinsic::eh_sjlj_longjmp: {
04984     DAG.setRoot(DAG.getNode(ISD::EH_SJLJ_LONGJMP, sdl, MVT::Other,
04985                             getRoot(), getValue(I.getArgOperand(0))));
04986     return nullptr;
04987   }
04988 
04989   case Intrinsic::x86_mmx_pslli_w:
04990   case Intrinsic::x86_mmx_pslli_d:
04991   case Intrinsic::x86_mmx_pslli_q:
04992   case Intrinsic::x86_mmx_psrli_w:
04993   case Intrinsic::x86_mmx_psrli_d:
04994   case Intrinsic::x86_mmx_psrli_q:
04995   case Intrinsic::x86_mmx_psrai_w:
04996   case Intrinsic::x86_mmx_psrai_d: {
04997     SDValue ShAmt = getValue(I.getArgOperand(1));
04998     if (isa<ConstantSDNode>(ShAmt)) {
04999       visitTargetIntrinsic(I, Intrinsic);
05000       return nullptr;
05001     }
05002     unsigned NewIntrinsic = 0;
05003     EVT ShAmtVT = MVT::v2i32;
05004     switch (Intrinsic) {
05005     case Intrinsic::x86_mmx_pslli_w:
05006       NewIntrinsic = Intrinsic::x86_mmx_psll_w;
05007       break;
05008     case Intrinsic::x86_mmx_pslli_d:
05009       NewIntrinsic = Intrinsic::x86_mmx_psll_d;
05010       break;
05011     case Intrinsic::x86_mmx_pslli_q:
05012       NewIntrinsic = Intrinsic::x86_mmx_psll_q;
05013       break;
05014     case Intrinsic::x86_mmx_psrli_w:
05015       NewIntrinsic = Intrinsic::x86_mmx_psrl_w;
05016       break;
05017     case Intrinsic::x86_mmx_psrli_d:
05018       NewIntrinsic = Intrinsic::x86_mmx_psrl_d;
05019       break;
05020     case Intrinsic::x86_mmx_psrli_q:
05021       NewIntrinsic = Intrinsic::x86_mmx_psrl_q;
05022       break;
05023     case Intrinsic::x86_mmx_psrai_w:
05024       NewIntrinsic = Intrinsic::x86_mmx_psra_w;
05025       break;
05026     case Intrinsic::x86_mmx_psrai_d:
05027       NewIntrinsic = Intrinsic::x86_mmx_psra_d;
05028       break;
05029     default: llvm_unreachable("Impossible intrinsic");  // Can't reach here.
05030     }
05031 
05032     // The vector shift intrinsics with scalars uses 32b shift amounts but
05033     // the sse2/mmx shift instructions reads 64 bits. Set the upper 32 bits
05034     // to be zero.
05035     // We must do this early because v2i32 is not a legal type.
05036     SDValue ShOps[2];
05037     ShOps[0] = ShAmt;
05038     ShOps[1] = DAG.getConstant(0, MVT::i32);
05039     ShAmt =  DAG.getNode(ISD::BUILD_VECTOR, sdl, ShAmtVT, ShOps);
05040     EVT DestVT = TLI->getValueType(I.getType());
05041     ShAmt = DAG.getNode(ISD::BITCAST, sdl, DestVT, ShAmt);
05042     Res = DAG.getNode(ISD::INTRINSIC_WO_CHAIN, sdl, DestVT,
05043                        DAG.getConstant(NewIntrinsic, MVT::i32),
05044                        getValue(I.getArgOperand(0)), ShAmt);
05045     setValue(&I, Res);
05046     return nullptr;
05047   }
05048   case Intrinsic::x86_avx_vinsertf128_pd_256:
05049   case Intrinsic::x86_avx_vinsertf128_ps_256:
05050   case Intrinsic::x86_avx_vinsertf128_si_256:
05051   case Intrinsic::x86_avx2_vinserti128: {
05052     EVT DestVT = TLI->getValueType(I.getType());
05053     EVT ElVT = TLI->getValueType(I.getArgOperand(1)->getType());
05054     uint64_t Idx = (cast<ConstantInt>(I.getArgOperand(2))->getZExtValue() & 1) *
05055                    ElVT.getVectorNumElements();
05056     Res = DAG.getNode(ISD::INSERT_SUBVECTOR, sdl, DestVT,
05057                       getValue(I.getArgOperand(0)),
05058                       getValue(I.getArgOperand(1)),
05059                       DAG.getConstant(Idx, TLI->getVectorIdxTy()));
05060     setValue(&I, Res);
05061     return nullptr;
05062   }
05063   case Intrinsic::x86_avx_vextractf128_pd_256:
05064   case Intrinsic::x86_avx_vextractf128_ps_256:
05065   case Intrinsic::x86_avx_vextractf128_si_256:
05066   case Intrinsic::x86_avx2_vextracti128: {
05067     EVT DestVT = TLI->getValueType(I.getType());
05068     uint64_t Idx = (cast<ConstantInt>(I.getArgOperand(1))->getZExtValue() & 1) *
05069                    DestVT.getVectorNumElements();
05070     Res = DAG.getNode(ISD::EXTRACT_SUBVECTOR, sdl, DestVT,
05071                       getValue(I.getArgOperand(0)),
05072                       DAG.getConstant(Idx, TLI->getVectorIdxTy()));
05073     setValue(&I, Res);
05074     return nullptr;
05075   }
05076   case Intrinsic::convertff:
05077   case Intrinsic::convertfsi:
05078   case Intrinsic::convertfui:
05079   case Intrinsic::convertsif:
05080   case Intrinsic::convertuif:
05081   case Intrinsic::convertss:
05082   case Intrinsic::convertsu:
05083   case Intrinsic::convertus:
05084   case Intrinsic::convertuu: {
05085     ISD::CvtCode Code = ISD::CVT_INVALID;
05086     switch (Intrinsic) {
05087     default: llvm_unreachable("Impossible intrinsic");  // Can't reach here.
05088     case Intrinsic::convertff:  Code = ISD::CVT_FF; break;
05089     case Intrinsic::convertfsi: Code = ISD::CVT_FS; break;
05090     case Intrinsic::convertfui: Code = ISD::CVT_FU; break;
05091     case Intrinsic::convertsif: Code = ISD::CVT_SF; break;
05092     case Intrinsic::convertuif: Code = ISD::CVT_UF; break;
05093     case Intrinsic::convertss:  Code = ISD::CVT_SS; break;
05094     case Intrinsic::convertsu:  Code = ISD::CVT_SU; break;
05095     case Intrinsic::convertus:  Code = ISD::CVT_US; break;
05096     case Intrinsic::convertuu:  Code = ISD::CVT_UU; break;
05097     }
05098     EVT DestVT = TLI->getValueType(I.getType());
05099     const Value *Op1 = I.getArgOperand(0);
05100     Res = DAG.getConvertRndSat(DestVT, sdl, getValue(Op1),
05101                                DAG.getValueType(DestVT),
05102                                DAG.getValueType(getValue(Op1).getValueType()),
05103                                getValue(I.getArgOperand(1)),
05104                                getValue(I.getArgOperand(2)),
05105                                Code);
05106     setValue(&I, Res);
05107     return nullptr;
05108   }
05109   case Intrinsic::powi:
05110     setValue(&I, ExpandPowI(sdl, getValue(I.getArgOperand(0)),
05111                             getValue(I.getArgOperand(1)), DAG));
05112     return nullptr;
05113   case Intrinsic::log:
05114     setValue(&I, expandLog(sdl, getValue(I.getArgOperand(0)), DAG, *TLI));
05115     return nullptr;
05116   case Intrinsic::log2:
05117     setValue(&I, expandLog2(sdl, getValue(I.getArgOperand(0)), DAG, *TLI));
05118     return nullptr;
05119   case Intrinsic::log10:
05120     setValue(&I, expandLog10(sdl, getValue(I.getArgOperand(0)), DAG, *TLI));
05121     return nullptr;
05122   case Intrinsic::exp:
05123     setValue(&I, expandExp(sdl, getValue(I.getArgOperand(0)), DAG, *TLI));
05124     return nullptr;
05125   case Intrinsic::exp2:
05126     setValue(&I, expandExp2(sdl, getValue(I.getArgOperand(0)), DAG, *TLI));
05127     return nullptr;
05128   case Intrinsic::pow:
05129     setValue(&I, expandPow(sdl, getValue(I.getArgOperand(0)),
05130                            getValue(I.getArgOperand(1)), DAG, *TLI));
05131     return nullptr;
05132   case Intrinsic::sqrt:
05133   case Intrinsic::fabs:
05134   case Intrinsic::sin:
05135   case Intrinsic::cos:
05136   case Intrinsic::floor:
05137   case Intrinsic::ceil:
05138   case Intrinsic::trunc:
05139   case Intrinsic::rint:
05140   case Intrinsic::nearbyint:
05141   case Intrinsic::round: {
05142     unsigned Opcode;
05143     switch (Intrinsic) {
05144     default: llvm_unreachable("Impossible intrinsic");  // Can't reach here.
05145     case Intrinsic::sqrt:      Opcode = ISD::FSQRT;      break;
05146     case Intrinsic::fabs:      Opcode = ISD::FABS;       break;
05147     case Intrinsic::sin:       Opcode = ISD::FSIN;       break;
05148     case Intrinsic::cos:       Opcode = ISD::FCOS;       break;
05149     case Intrinsic::floor:     Opcode = ISD::FFLOOR;     break;
05150     case Intrinsic::ceil:      Opcode = ISD::FCEIL;      break;
05151     case Intrinsic::trunc:     Opcode = ISD::FTRUNC;     break;
05152     case Intrinsic::rint:      Opcode = ISD::FRINT;      break;
05153     case Intrinsic::nearbyint: Opcode = ISD::FNEARBYINT; break;
05154     case Intrinsic::round:     Opcode = ISD::FROUND;     break;
05155     }
05156 
05157     setValue(&I, DAG.getNode(Opcode, sdl,
05158                              getValue(I.getArgOperand(0)).getValueType(),
05159                              getValue(I.getArgOperand(0))));
05160     return nullptr;
05161   }
05162   case Intrinsic::copysign:
05163     setValue(&I, DAG.getNode(ISD::FCOPYSIGN, sdl,
05164                              getValue(I.getArgOperand(0)).getValueType(),
05165                              getValue(I.getArgOperand(0)),
05166                              getValue(I.getArgOperand(1))));
05167     return nullptr;
05168   case Intrinsic::fma:
05169     setValue(&I, DAG.getNode(ISD::FMA, sdl,
05170                              getValue(I.getArgOperand(0)).getValueType(),
05171                              getValue(I.getArgOperand(0)),
05172                              getValue(I.getArgOperand(1)),
05173                              getValue(I.getArgOperand(2))));
05174     return nullptr;
05175   case Intrinsic::fmuladd: {
05176     EVT VT = TLI->getValueType(I.getType());
05177     if (TM.Options.AllowFPOpFusion != FPOpFusion::Strict &&
05178         TLI->isFMAFasterThanFMulAndFAdd(VT)) {
05179       setValue(&I, DAG.getNode(ISD::FMA, sdl,
05180                                getValue(I.getArgOperand(0)).getValueType(),
05181                                getValue(I.getArgOperand(0)),
05182                                getValue(I.getArgOperand(1)),
05183                                getValue(I.getArgOperand(2))));
05184     } else {
05185       SDValue Mul = DAG.getNode(ISD::FMUL, sdl,
05186                                 getValue(I.getArgOperand(0)).getValueType(),
05187                                 getValue(I.getArgOperand(0)),
05188                                 getValue(I.getArgOperand(1)));
05189       SDValue Add = DAG.getNode(ISD::FADD, sdl,
05190                                 getValue(I.getArgOperand(0)).getValueType(),
05191                                 Mul,
05192                                 getValue(I.getArgOperand(2)));
05193       setValue(&I, Add);
05194     }
05195     return nullptr;
05196   }
05197   case Intrinsic::convert_to_fp16:
05198     setValue(&I, DAG.getNode(ISD::BITCAST, sdl, MVT::i16,
05199                              DAG.getNode(ISD::FP_ROUND, sdl, MVT::f16,
05200                                          getValue(I.getArgOperand(0)),
05201                                          DAG.getTargetConstant(0, MVT::i32))));
05202     return nullptr;
05203   case Intrinsic::convert_from_fp16:
05204     setValue(&I,
05205              DAG.getNode(ISD::FP_EXTEND, sdl, TLI->getValueType(I.getType()),
05206                          DAG.getNode(ISD::BITCAST, sdl, MVT::f16,
05207                                      getValue(I.getArgOperand(0)))));
05208     return nullptr;
05209   case Intrinsic::pcmarker: {
05210     SDValue Tmp = getValue(I.getArgOperand(0));
05211     DAG.setRoot(DAG.getNode(ISD::PCMARKER, sdl, MVT::Other, getRoot(), Tmp));
05212     return nullptr;
05213   }
05214   case Intrinsic::readcyclecounter: {
05215     SDValue Op = getRoot();
05216     Res = DAG.getNode(ISD::READCYCLECOUNTER, sdl,
05217                       DAG.getVTList(MVT::i64, MVT::Other), Op);
05218     setValue(&I, Res);
05219     DAG.setRoot(Res.getValue(1));
05220     return nullptr;
05221   }
05222   case Intrinsic::bswap:
05223     setValue(&I, DAG.getNode(ISD::BSWAP, sdl,
05224                              getValue(I.getArgOperand(0)).getValueType(),
05225                              getValue(I.getArgOperand(0))));
05226     return nullptr;
05227   case Intrinsic::cttz: {
05228     SDValue Arg = getValue(I.getArgOperand(0));
05229     ConstantInt *CI = cast<ConstantInt>(I.getArgOperand(1));
05230     EVT Ty = Arg.getValueType();
05231     setValue(&I, DAG.getNode(CI->isZero() ? ISD::CTTZ : ISD::CTTZ_ZERO_UNDEF,
05232                              sdl, Ty, Arg));
05233     return nullptr;
05234   }
05235   case Intrinsic::ctlz: {
05236     SDValue Arg = getValue(I.getArgOperand(0));
05237     ConstantInt *CI = cast<ConstantInt>(I.getArgOperand(1));
05238     EVT Ty = Arg.getValueType();
05239     setValue(&I, DAG.getNode(CI->isZero() ? ISD::CTLZ : ISD::CTLZ_ZERO_UNDEF,
05240                              sdl, Ty, Arg));
05241     return nullptr;
05242   }
05243   case Intrinsic::ctpop: {
05244     SDValue Arg = getValue(I.getArgOperand(0));
05245     EVT Ty = Arg.getValueType();
05246     setValue(&I, DAG.getNode(ISD::CTPOP, sdl, Ty, Arg));
05247     return nullptr;
05248   }
05249   case Intrinsic::stacksave: {
05250     SDValue Op = getRoot();
05251     Res = DAG.getNode(ISD::STACKSAVE, sdl,
05252                       DAG.getVTList(TLI->getPointerTy(), MVT::Other), Op);
05253     setValue(&I, Res);
05254     DAG.setRoot(Res.getValue(1));
05255     return nullptr;
05256   }
05257   case Intrinsic::stackrestore: {
05258     Res = getValue(I.getArgOperand(0));
05259     DAG.setRoot(DAG.getNode(ISD::STACKRESTORE, sdl, MVT::Other, getRoot(), Res));
05260     return nullptr;
05261   }
05262   case Intrinsic::stackprotector: {
05263     // Emit code into the DAG to store the stack guard onto the stack.
05264     MachineFunction &MF = DAG.getMachineFunction();
05265     MachineFrameInfo *MFI = MF.getFrameInfo();
05266     EVT PtrTy = TLI->getPointerTy();
05267     SDValue Src, Chain = getRoot();
05268     const Value *Ptr = cast<LoadInst>(I.getArgOperand(0))->getPointerOperand();
05269     const GlobalVariable *GV = dyn_cast<GlobalVariable>(Ptr);
05270 
05271     // See if Ptr is a bitcast. If it is, look through it and see if we can get
05272     // global variable __stack_chk_guard.
05273     if (!GV)
05274       if (const Operator *BC = dyn_cast<Operator>(Ptr))
05275         if (BC->getOpcode() == Instruction::BitCast)
05276           GV = dyn_cast<GlobalVariable>(BC->getOperand(0));
05277 
05278     if (GV && TLI->useLoadStackGuardNode()) {
05279       // Emit a LOAD_STACK_GUARD node.
05280       MachineSDNode *Node = DAG.getMachineNode(TargetOpcode::LOAD_STACK_GUARD,
05281                                                sdl, PtrTy, Chain);
05282       MachinePointerInfo MPInfo(GV);
05283       MachineInstr::mmo_iterator MemRefs = MF.allocateMemRefsArray(1);
05284       unsigned Flags = MachineMemOperand::MOLoad |
05285                        MachineMemOperand::MOInvariant;
05286       *MemRefs = MF.getMachineMemOperand(MPInfo, Flags,
05287                                          PtrTy.getSizeInBits() / 8,
05288                                          DAG.getEVTAlignment(PtrTy));
05289       Node->setMemRefs(MemRefs, MemRefs + 1);
05290 
05291       // Copy the guard value to a virtual register so that it can be
05292       // retrieved in the epilogue.
05293       Src = SDValue(Node, 0);
05294       const TargetRegisterClass *RC =
05295           TLI->getRegClassFor(Src.getSimpleValueType());
05296       unsigned Reg = MF.getRegInfo().createVirtualRegister(RC);
05297 
05298       SPDescriptor.setGuardReg(Reg);
05299       Chain = DAG.getCopyToReg(Chain, sdl, Reg, Src);
05300     } else {
05301       Src = getValue(I.getArgOperand(0));   // The guard's value.
05302     }
05303 
05304     AllocaInst *Slot = cast<AllocaInst>(I.getArgOperand(1));
05305 
05306     int FI = FuncInfo.StaticAllocaMap[Slot];
05307     MFI->setStackProtectorIndex(FI);
05308 
05309     SDValue FIN = DAG.getFrameIndex(FI, PtrTy);
05310 
05311     // Store the stack protector onto the stack.
05312     Res = DAG.getStore(Chain, sdl, Src, FIN,
05313                        MachinePointerInfo::getFixedStack(FI),
05314                        true, false, 0);
05315     setValue(&I, Res);
05316     DAG.setRoot(Res);
05317     return nullptr;
05318   }
05319   case Intrinsic::objectsize: {
05320     // If we don't know by now, we're never going to know.
05321     ConstantInt *CI = dyn_cast<ConstantInt>(I.getArgOperand(1));
05322 
05323     assert(CI && "Non-constant type in __builtin_object_size?");
05324 
05325     SDValue Arg = getValue(I.getCalledValue());
05326     EVT Ty = Arg.getValueType();
05327 
05328     if (CI->isZero())
05329       Res = DAG.getConstant(-1ULL, Ty);
05330     else
05331       Res = DAG.getConstant(0, Ty);
05332 
05333     setValue(&I, Res);
05334     return nullptr;
05335   }
05336   case Intrinsic::annotation:
05337   case Intrinsic::ptr_annotation:
05338     // Drop the intrinsic, but forward the value
05339     setValue(&I, getValue(I.getOperand(0)));
05340     return nullptr;
05341   case Intrinsic::assume:
05342   case Intrinsic::var_annotation:
05343     // Discard annotate attributes and assumptions
05344     return nullptr;
05345 
05346   case Intrinsic::init_trampoline: {
05347     const Function *F = cast<Function>(I.getArgOperand(1)->stripPointerCasts());
05348 
05349     SDValue Ops[6];
05350     Ops[0] = getRoot();
05351     Ops[1] = getValue(I.getArgOperand(0));
05352     Ops[2] = getValue(I.getArgOperand(1));
05353     Ops[3] = getValue(I.getArgOperand(2));
05354     Ops[4] = DAG.getSrcValue(I.getArgOperand(0));
05355     Ops[5] = DAG.getSrcValue(F);
05356 
05357     Res = DAG.getNode(ISD::INIT_TRAMPOLINE, sdl, MVT::Other, Ops);
05358 
05359     DAG.setRoot(Res);
05360     return nullptr;
05361   }
05362   case Intrinsic::adjust_trampoline: {
05363     setValue(&I, DAG.getNode(ISD::ADJUST_TRAMPOLINE, sdl,
05364                              TLI->getPointerTy(),
05365                              getValue(I.getArgOperand(0))));
05366     return nullptr;
05367   }
05368   case Intrinsic::gcroot:
05369     if (GFI) {
05370       const Value *Alloca = I.getArgOperand(0)->stripPointerCasts();
05371       const Constant *TypeMap = cast<Constant>(I.getArgOperand(1));
05372 
05373       FrameIndexSDNode *FI = cast<FrameIndexSDNode>(getValue(Alloca).getNode());
05374       GFI->addStackRoot(FI->getIndex(), TypeMap);
05375     }
05376     return nullptr;
05377   case Intrinsic::gcread:
05378   case Intrinsic::gcwrite:
05379     llvm_unreachable("GC failed to lower gcread/gcwrite intrinsics!");
05380   case Intrinsic::flt_rounds:
05381     setValue(&I, DAG.getNode(ISD::FLT_ROUNDS_, sdl, MVT::i32));
05382     return nullptr;
05383 
05384   case Intrinsic::expect: {
05385     // Just replace __builtin_expect(exp, c) with EXP.
05386     setValue(&I, getValue(I.getArgOperand(0)));
05387     return nullptr;
05388   }
05389 
05390   case Intrinsic::debugtrap:
05391   case Intrinsic::trap: {
05392     StringRef TrapFuncName = TM.Options.getTrapFunctionName();
05393     if (TrapFuncName.empty()) {
05394       ISD::NodeType Op = (Intrinsic == Intrinsic::trap) ?
05395         ISD::TRAP : ISD::DEBUGTRAP;
05396       DAG.setRoot(DAG.getNode(Op, sdl,MVT::Other, getRoot()));
05397       return nullptr;
05398     }
05399     TargetLowering::ArgListTy Args;
05400 
05401     TargetLowering::CallLoweringInfo CLI(DAG);
05402     CLI.setDebugLoc(sdl).setChain(getRoot())
05403       .setCallee(CallingConv::C, I.getType(),
05404                  DAG.getExternalSymbol(TrapFuncName.data(), TLI->getPointerTy()),
05405                  std::move(Args), 0);
05406 
05407     std::pair<SDValue, SDValue> Result = TLI->LowerCallTo(CLI);
05408     DAG.setRoot(Result.second);
05409     return nullptr;
05410   }
05411 
05412   case Intrinsic::uadd_with_overflow:
05413   case Intrinsic::sadd_with_overflow:
05414   case Intrinsic::usub_with_overflow:
05415   case Intrinsic::ssub_with_overflow:
05416   case Intrinsic::umul_with_overflow:
05417   case Intrinsic::smul_with_overflow: {
05418     ISD::NodeType Op;
05419     switch (Intrinsic) {
05420     default: llvm_unreachable("Impossible intrinsic");  // Can't reach here.
05421     case Intrinsic::uadd_with_overflow: Op = ISD::UADDO; break;
05422     case Intrinsic::sadd_with_overflow: Op = ISD::SADDO; break;
05423     case Intrinsic::usub_with_overflow: Op = ISD::USUBO; break;
05424     case Intrinsic::ssub_with_overflow: Op = ISD::SSUBO; break;
05425     case Intrinsic::umul_with_overflow: Op = ISD::UMULO; break;
05426     case Intrinsic::smul_with_overflow: Op = ISD::SMULO; break;
05427     }
05428     SDValue Op1 = getValue(I.getArgOperand(0));
05429     SDValue Op2 = getValue(I.getArgOperand(1));
05430 
05431     SDVTList VTs = DAG.getVTList(Op1.getValueType(), MVT::i1);
05432     setValue(&I, DAG.getNode(Op, sdl, VTs, Op1, Op2));
05433     return nullptr;
05434   }
05435   case Intrinsic::prefetch: {
05436     SDValue Ops[5];
05437     unsigned rw = cast<ConstantInt>(I.getArgOperand(1))->getZExtValue();
05438     Ops[0] = getRoot();
05439     Ops[1] = getValue(I.getArgOperand(0));
05440     Ops[2] = getValue(I.getArgOperand(1));
05441     Ops[3] = getValue(I.getArgOperand(2));
05442     Ops[4] = getValue(I.getArgOperand(3));
05443     DAG.setRoot(DAG.getMemIntrinsicNode(ISD::PREFETCH, sdl,
05444                                         DAG.getVTList(MVT::Other), Ops,
05445                                         EVT::getIntegerVT(*Context, 8),
05446                                         MachinePointerInfo(I.getArgOperand(0)),
05447                                         0, /* align */
05448                                         false, /* volatile */
05449                                         rw==0, /* read */
05450                                         rw==1)); /* write */
05451     return nullptr;
05452   }
05453   case Intrinsic::lifetime_start:
05454   case Intrinsic::lifetime_end: {
05455     bool IsStart = (Intrinsic == Intrinsic::lifetime_start);
05456     // Stack coloring is not enabled in O0, discard region information.
05457     if (TM.getOptLevel() == CodeGenOpt::None)
05458       return nullptr;
05459 
05460     SmallVector<Value *, 4> Allocas;
05461     GetUnderlyingObjects(I.getArgOperand(1), Allocas, DL);
05462 
05463     for (SmallVectorImpl<Value*>::iterator Object = Allocas.begin(),
05464            E = Allocas.end(); Object != E; ++Object) {
05465       AllocaInst *LifetimeObject = dyn_cast_or_null<AllocaInst>(*Object);
05466 
05467       // Could not find an Alloca.
05468       if (!LifetimeObject)
05469         continue;
05470 
05471       int FI = FuncInfo.StaticAllocaMap[LifetimeObject];
05472 
05473       SDValue Ops[2];
05474       Ops[0] = getRoot();
05475       Ops[1] = DAG.getFrameIndex(FI, TLI->getPointerTy(), true);
05476       unsigned Opcode = (IsStart ? ISD::LIFETIME_START : ISD::LIFETIME_END);
05477 
05478       Res = DAG.getNode(Opcode, sdl, MVT::Other, Ops);
05479       DAG.setRoot(Res);
05480     }
05481     return nullptr;
05482   }
05483   case Intrinsic::invariant_start:
05484     // Discard region information.
05485     setValue(&I, DAG.getUNDEF(TLI->getPointerTy()));
05486     return nullptr;
05487   case Intrinsic::invariant_end:
05488     // Discard region information.
05489     return nullptr;
05490   case Intrinsic::stackprotectorcheck: {
05491     // Do not actually emit anything for this basic block. Instead we initialize
05492     // the stack protector descriptor and export the guard variable so we can
05493     // access it in FinishBasicBlock.
05494     const BasicBlock *BB = I.getParent();
05495     SPDescriptor.initialize(BB, FuncInfo.MBBMap[BB], I);
05496     ExportFromCurrentBlock(SPDescriptor.getGuard());
05497 
05498     // Flush our exports since we are going to process a terminator.
05499     (void)getControlRoot();
05500     return nullptr;
05501   }
05502   case Intrinsic::clear_cache:
05503     return TLI->getClearCacheBuiltinName();
05504   case Intrinsic::donothing:
05505     // ignore
05506     return nullptr;
05507   case Intrinsic::experimental_stackmap: {
05508     visitStackmap(I);
05509     return nullptr;
05510   }
05511   case Intrinsic::experimental_patchpoint_void:
05512   case Intrinsic::experimental_patchpoint_i64: {
05513     visitPatchpoint(I);
05514     return nullptr;
05515   }
05516   }
05517 }
05518 
05519 void SelectionDAGBuilder::LowerCallTo(ImmutableCallSite CS, SDValue Callee,
05520                                       bool isTailCall,
05521                                       MachineBasicBlock *LandingPad) {
05522   const TargetLowering *TLI = TM.getSubtargetImpl()->getTargetLowering();
05523   PointerType *PT = cast<PointerType>(CS.getCalledValue()->getType());
05524   FunctionType *FTy = cast<FunctionType>(PT->getElementType());
05525   Type *RetTy = FTy->getReturnType();
05526   MachineModuleInfo &MMI = DAG.getMachineFunction().getMMI();
05527   MCSymbol *BeginLabel = nullptr;
05528 
05529   TargetLowering::ArgListTy Args;
05530   TargetLowering::ArgListEntry Entry;
05531   Args.reserve(CS.arg_size());
05532 
05533   for (ImmutableCallSite::arg_iterator i = CS.arg_begin(), e = CS.arg_end();
05534        i != e; ++i) {
05535     const Value *V = *i;
05536 
05537     // Skip empty types
05538     if (V->getType()->isEmptyTy())
05539       continue;
05540 
05541     SDValue ArgNode = getValue(V);
05542     Entry.Node = ArgNode; Entry.Ty = V->getType();
05543 
05544     // Skip the first return-type Attribute to get to params.
05545     Entry.setAttributes(&CS, i - CS.arg_begin() + 1);
05546     Args.push_back(Entry);
05547   }
05548 
05549   if (LandingPad) {
05550     // Insert a label before the invoke call to mark the try range.  This can be
05551     // used to detect deletion of the invoke via the MachineModuleInfo.
05552     BeginLabel = MMI.getContext().CreateTempSymbol();
05553 
05554     // For SjLj, keep track of which landing pads go with which invokes
05555     // so as to maintain the ordering of pads in the LSDA.
05556     unsigned CallSiteIndex = MMI.getCurrentCallSite();
05557     if (CallSiteIndex) {
05558       MMI.setCallSiteBeginLabel(BeginLabel, CallSiteIndex);
05559       LPadToCallSiteMap[LandingPad].push_back(CallSiteIndex);
05560 
05561       // Now that the call site is handled, stop tracking it.
05562       MMI.setCurrentCallSite(0);
05563     }
05564 
05565     // Both PendingLoads and PendingExports must be flushed here;
05566     // this call might not return.
05567     (void)getRoot();
05568     DAG.setRoot(DAG.getEHLabel(getCurSDLoc(), getControlRoot(), BeginLabel));
05569   }
05570 
05571   // Check if target-independent constraints permit a tail call here.
05572   // Target-dependent constraints are checked within TLI->LowerCallTo.
05573   if (isTailCall && !isInTailCallPosition(CS, DAG.getTarget()))
05574     isTailCall = false;
05575 
05576   TargetLowering::CallLoweringInfo CLI(DAG);
05577   CLI.setDebugLoc(getCurSDLoc()).setChain(getRoot())
05578     .setCallee(RetTy, FTy, Callee, std::move(Args), CS).setTailCall(isTailCall);
05579 
05580   std::pair<SDValue,SDValue> Result = TLI->LowerCallTo(CLI);
05581   assert((isTailCall || Result.second.getNode()) &&
05582          "Non-null chain expected with non-tail call!");
05583   assert((Result.second.getNode() || !Result.first.getNode()) &&
05584          "Null value expected with tail call!");
05585   if (Result.first.getNode())
05586     setValue(CS.getInstruction(), Result.first);
05587 
05588   if (!Result.second.getNode()) {
05589     // As a special case, a null chain means that a tail call has been emitted
05590     // and the DAG root is already updated.
05591     HasTailCall = true;
05592 
05593     // Since there's no actual continuation from this block, nothing can be
05594     // relying on us setting vregs for them.
05595     PendingExports.clear();
05596   } else {
05597     DAG.setRoot(Result.second);
05598   }
05599 
05600   if (LandingPad) {
05601     // Insert a label at the end of the invoke call to mark the try range.  This
05602     // can be used to detect deletion of the invoke via the MachineModuleInfo.
05603     MCSymbol *EndLabel = MMI.getContext().CreateTempSymbol();
05604     DAG.setRoot(DAG.getEHLabel(getCurSDLoc(), getRoot(), EndLabel));
05605 
05606     // Inform MachineModuleInfo of range.
05607     MMI.addInvoke(LandingPad, BeginLabel, EndLabel);
05608   }
05609 }
05610 
05611 /// IsOnlyUsedInZeroEqualityComparison - Return true if it only matters that the
05612 /// value is equal or not-equal to zero.
05613 static bool IsOnlyUsedInZeroEqualityComparison(const Value *V) {
05614   for (const User *U : V->users()) {
05615     if (const ICmpInst *IC = dyn_cast<ICmpInst>(U))
05616       if (IC->isEquality())
05617         if (const Constant *C = dyn_cast<Constant>(IC->getOperand(1)))
05618           if (C->isNullValue())
05619             continue;
05620     // Unknown instruction.
05621     return false;
05622   }
05623   return true;
05624 }
05625 
05626 static SDValue getMemCmpLoad(const Value *PtrVal, MVT LoadVT,
05627                              Type *LoadTy,
05628                              SelectionDAGBuilder &Builder) {
05629 
05630   // Check to see if this load can be trivially constant folded, e.g. if the
05631   // input is from a string literal.
05632   if (const Constant *LoadInput = dyn_cast<Constant>(PtrVal)) {
05633     // Cast pointer to the type we really want to load.
05634     LoadInput = ConstantExpr::getBitCast(const_cast<Constant *>(LoadInput),
05635                                          PointerType::getUnqual(LoadTy));
05636 
05637     if (const Constant *LoadCst =
05638           ConstantFoldLoadFromConstPtr(const_cast<Constant *>(LoadInput),
05639                                        Builder.DL))
05640       return Builder.getValue(LoadCst);
05641   }
05642 
05643   // Otherwise, we have to emit the load.  If the pointer is to unfoldable but
05644   // still constant memory, the input chain can be the entry node.
05645   SDValue Root;
05646   bool ConstantMemory = false;
05647 
05648   // Do not serialize (non-volatile) loads of constant memory with anything.
05649   if (Builder.AA->pointsToConstantMemory(PtrVal)) {
05650     Root = Builder.DAG.getEntryNode();
05651     ConstantMemory = true;
05652   } else {
05653     // Do not serialize non-volatile loads against each other.
05654     Root = Builder.DAG.getRoot();
05655   }
05656 
05657   SDValue Ptr = Builder.getValue(PtrVal);
05658   SDValue LoadVal = Builder.DAG.getLoad(LoadVT, Builder.getCurSDLoc(), Root,
05659                                         Ptr, MachinePointerInfo(PtrVal),
05660                                         false /*volatile*/,
05661                                         false /*nontemporal*/,
05662                                         false /*isinvariant*/, 1 /* align=1 */);
05663 
05664   if (!ConstantMemory)
05665     Builder.PendingLoads.push_back(LoadVal.getValue(1));
05666   return LoadVal;
05667 }
05668 
05669 /// processIntegerCallValue - Record the value for an instruction that
05670 /// produces an integer result, converting the type where necessary.
05671 void SelectionDAGBuilder::processIntegerCallValue(const Instruction &I,
05672                                                   SDValue Value,
05673                                                   bool IsSigned) {
05674   EVT VT = TM.getSubtargetImpl()->getTargetLowering()->getValueType(I.getType(),
05675                                                                     true);
05676   if (IsSigned)
05677     Value = DAG.getSExtOrTrunc(Value, getCurSDLoc(), VT);
05678   else
05679     Value = DAG.getZExtOrTrunc(Value, getCurSDLoc(), VT);
05680   setValue(&I, Value);
05681 }
05682 
05683 /// visitMemCmpCall - See if we can lower a call to memcmp in an optimized form.
05684 /// If so, return true and lower it, otherwise return false and it will be
05685 /// lowered like a normal call.
05686 bool SelectionDAGBuilder::visitMemCmpCall(const CallInst &I) {
05687   // Verify that the prototype makes sense.  int memcmp(void*,void*,size_t)
05688   if (I.getNumArgOperands() != 3)
05689     return false;
05690 
05691   const Value *LHS = I.getArgOperand(0), *RHS = I.getArgOperand(1);
05692   if (!LHS->getType()->isPointerTy() || !RHS->getType()->isPointerTy() ||
05693       !I.getArgOperand(2)->getType()->isIntegerTy() ||
05694       !I.getType()->isIntegerTy())
05695     return false;
05696 
05697   const Value *Size = I.getArgOperand(2);
05698   const ConstantInt *CSize = dyn_cast<ConstantInt>(Size);
05699   if (CSize && CSize->getZExtValue() == 0) {
05700     EVT CallVT = TM.getSubtargetImpl()->getTargetLowering()->getValueType(
05701         I.getType(), true);
05702     setValue(&I, DAG.getConstant(0, CallVT));
05703     return true;
05704   }
05705 
05706   const TargetSelectionDAGInfo &TSI = DAG.getSelectionDAGInfo();
05707   std::pair<SDValue, SDValue> Res =
05708     TSI.EmitTargetCodeForMemcmp(DAG, getCurSDLoc(), DAG.getRoot(),
05709                                 getValue(LHS), getValue(RHS), getValue(Size),
05710                                 MachinePointerInfo(LHS),
05711                                 MachinePointerInfo(RHS));
05712   if (Res.first.getNode()) {
05713     processIntegerCallValue(I, Res.first, true);
05714     PendingLoads.push_back(Res.second);
05715     return true;
05716   }
05717 
05718   // memcmp(S1,S2,2) != 0 -> (*(short*)LHS != *(short*)RHS)  != 0
05719   // memcmp(S1,S2,4) != 0 -> (*(int*)LHS != *(int*)RHS)  != 0
05720   if (CSize && IsOnlyUsedInZeroEqualityComparison(&I)) {
05721     bool ActuallyDoIt = true;
05722     MVT LoadVT;
05723     Type *LoadTy;
05724     switch (CSize->getZExtValue()) {
05725     default:
05726       LoadVT = MVT::Other;
05727       LoadTy = nullptr;
05728       ActuallyDoIt = false;
05729       break;
05730     case 2:
05731       LoadVT = MVT::i16;
05732       LoadTy = Type::getInt16Ty(CSize->getContext());
05733       break;
05734     case 4:
05735       LoadVT = MVT::i32;
05736       LoadTy = Type::getInt32Ty(CSize->getContext());
05737       break;
05738     case 8:
05739       LoadVT = MVT::i64;
05740       LoadTy = Type::getInt64Ty(CSize->getContext());
05741       break;
05742         /*
05743     case 16:
05744       LoadVT = MVT::v4i32;
05745       LoadTy = Type::getInt32Ty(CSize->getContext());
05746       LoadTy = VectorType::get(LoadTy, 4);
05747       break;
05748          */
05749     }
05750 
05751     // This turns into unaligned loads.  We only do this if the target natively
05752     // supports the MVT we'll be loading or if it is small enough (<= 4) that
05753     // we'll only produce a small number of byte loads.
05754 
05755     // Require that we can find a legal MVT, and only do this if the target
05756     // supports unaligned loads of that type.  Expanding into byte loads would
05757     // bloat the code.
05758     const TargetLowering *TLI = TM.getSubtargetImpl()->getTargetLowering();
05759     if (ActuallyDoIt && CSize->getZExtValue() > 4) {
05760       unsigned DstAS = LHS->getType()->getPointerAddressSpace();
05761       unsigned SrcAS = RHS->getType()->getPointerAddressSpace();
05762       // TODO: Handle 5 byte compare as 4-byte + 1 byte.
05763       // TODO: Handle 8 byte compare on x86-32 as two 32-bit loads.
05764       // TODO: Check alignment of src and dest ptrs.
05765       if (!TLI->isTypeLegal(LoadVT) ||
05766           !TLI->allowsMisalignedMemoryAccesses(LoadVT, SrcAS) ||
05767           !TLI->allowsMisalignedMemoryAccesses(LoadVT, DstAS))
05768         ActuallyDoIt = false;
05769     }
05770 
05771     if (ActuallyDoIt) {
05772       SDValue LHSVal = getMemCmpLoad(LHS, LoadVT, LoadTy, *this);
05773       SDValue RHSVal = getMemCmpLoad(RHS, LoadVT, LoadTy, *this);
05774 
05775       SDValue Res = DAG.getSetCC(getCurSDLoc(), MVT::i1, LHSVal, RHSVal,
05776                                  ISD::SETNE);
05777       processIntegerCallValue(I, Res, false);
05778       return true;
05779     }
05780   }
05781 
05782 
05783   return false;
05784 }
05785 
05786 /// visitMemChrCall -- See if we can lower a memchr call into an optimized
05787 /// form.  If so, return true and lower it, otherwise return false and it
05788 /// will be lowered like a normal call.
05789 bool SelectionDAGBuilder::visitMemChrCall(const CallInst &I) {
05790   // Verify that the prototype makes sense.  void *memchr(void *, int, size_t)
05791   if (I.getNumArgOperands() != 3)
05792     return false;
05793 
05794   const Value *Src = I.getArgOperand(0);
05795   const Value *Char = I.getArgOperand(1);
05796   const Value *Length = I.getArgOperand(2);
05797   if (!Src->getType()->isPointerTy() ||
05798       !Char->getType()->isIntegerTy() ||
05799       !Length->getType()->isIntegerTy() ||
05800       !I.getType()->isPointerTy())
05801     return false;
05802 
05803   const TargetSelectionDAGInfo &TSI = DAG.getSelectionDAGInfo();
05804   std::pair<SDValue, SDValue> Res =
05805     TSI.EmitTargetCodeForMemchr(DAG, getCurSDLoc(), DAG.getRoot(),
05806                                 getValue(Src), getValue(Char), getValue(Length),
05807                                 MachinePointerInfo(Src));
05808   if (Res.first.getNode()) {
05809     setValue(&I, Res.first);
05810     PendingLoads.push_back(Res.second);
05811     return true;
05812   }
05813 
05814   return false;
05815 }
05816 
05817 /// visitStrCpyCall -- See if we can lower a strcpy or stpcpy call into an
05818 /// optimized form.  If so, return true and lower it, otherwise return false
05819 /// and it will be lowered like a normal call.
05820 bool SelectionDAGBuilder::visitStrCpyCall(const CallInst &I, bool isStpcpy) {
05821   // Verify that the prototype makes sense.  char *strcpy(char *, char *)
05822   if (I.getNumArgOperands() != 2)
05823     return false;
05824 
05825   const Value *Arg0 = I.getArgOperand(0), *Arg1 = I.getArgOperand(1);
05826   if (!Arg0->getType()->isPointerTy() ||
05827       !Arg1->getType()->isPointerTy() ||
05828       !I.getType()->isPointerTy())
05829     return false;
05830 
05831   const TargetSelectionDAGInfo &TSI = DAG.getSelectionDAGInfo();
05832   std::pair<SDValue, SDValue> Res =
05833     TSI.EmitTargetCodeForStrcpy(DAG, getCurSDLoc(), getRoot(),
05834                                 getValue(Arg0), getValue(Arg1),
05835                                 MachinePointerInfo(Arg0),
05836                                 MachinePointerInfo(Arg1), isStpcpy);
05837   if (Res.first.getNode()) {
05838     setValue(&I, Res.first);
05839     DAG.setRoot(Res.second);
05840     return true;
05841   }
05842 
05843   return false;
05844 }
05845 
05846 /// visitStrCmpCall - See if we can lower a call to strcmp in an optimized form.
05847 /// If so, return true and lower it, otherwise return false and it will be
05848 /// lowered like a normal call.
05849 bool SelectionDAGBuilder::visitStrCmpCall(const CallInst &I) {
05850   // Verify that the prototype makes sense.  int strcmp(void*,void*)
05851   if (I.getNumArgOperands() != 2)
05852     return false;
05853 
05854   const Value *Arg0 = I.getArgOperand(0), *Arg1 = I.getArgOperand(1);
05855   if (!Arg0->getType()->isPointerTy() ||
05856       !Arg1->getType()->isPointerTy() ||
05857       !I.getType()->isIntegerTy())
05858     return false;
05859 
05860   const TargetSelectionDAGInfo &TSI = DAG.getSelectionDAGInfo();
05861   std::pair<SDValue, SDValue> Res =
05862     TSI.EmitTargetCodeForStrcmp(DAG, getCurSDLoc(), DAG.getRoot(),
05863                                 getValue(Arg0), getValue(Arg1),
05864                                 MachinePointerInfo(Arg0),
05865                                 MachinePointerInfo(Arg1));
05866   if (Res.first.getNode()) {
05867     processIntegerCallValue(I, Res.first, true);
05868     PendingLoads.push_back(Res.second);
05869     return true;
05870   }
05871 
05872   return false;
05873 }
05874 
05875 /// visitStrLenCall -- See if we can lower a strlen call into an optimized
05876 /// form.  If so, return true and lower it, otherwise return false and it
05877 /// will be lowered like a normal call.
05878 bool SelectionDAGBuilder::visitStrLenCall(const CallInst &I) {
05879   // Verify that the prototype makes sense.  size_t strlen(char *)
05880   if (I.getNumArgOperands() != 1)
05881     return false;
05882 
05883   const Value *Arg0 = I.getArgOperand(0);
05884   if (!Arg0->getType()->isPointerTy() || !I.getType()->isIntegerTy())
05885     return false;
05886 
05887   const TargetSelectionDAGInfo &TSI = DAG.getSelectionDAGInfo();
05888   std::pair<SDValue, SDValue> Res =
05889     TSI.EmitTargetCodeForStrlen(DAG, getCurSDLoc(), DAG.getRoot(),
05890                                 getValue(Arg0), MachinePointerInfo(Arg0));
05891   if (Res.first.getNode()) {
05892     processIntegerCallValue(I, Res.first, false);
05893     PendingLoads.push_back(Res.second);
05894     return true;
05895   }
05896 
05897   return false;
05898 }
05899 
05900 /// visitStrNLenCall -- See if we can lower a strnlen call into an optimized
05901 /// form.  If so, return true and lower it, otherwise return false and it
05902 /// will be lowered like a normal call.
05903 bool SelectionDAGBuilder::visitStrNLenCall(const CallInst &I) {
05904   // Verify that the prototype makes sense.  size_t strnlen(char *, size_t)
05905   if (I.getNumArgOperands() != 2)
05906     return false;
05907 
05908   const Value *Arg0 = I.getArgOperand(0), *Arg1 = I.getArgOperand(1);
05909   if (!Arg0->getType()->isPointerTy() ||
05910       !Arg1->getType()->isIntegerTy() ||
05911       !I.getType()->isIntegerTy())
05912     return false;
05913 
05914   const TargetSelectionDAGInfo &TSI = DAG.getSelectionDAGInfo();
05915   std::pair<SDValue, SDValue> Res =
05916     TSI.EmitTargetCodeForStrnlen(DAG, getCurSDLoc(), DAG.getRoot(),
05917                                  getValue(Arg0), getValue(Arg1),
05918                                  MachinePointerInfo(Arg0));
05919   if (Res.first.getNode()) {
05920     processIntegerCallValue(I, Res.first, false);
05921     PendingLoads.push_back(Res.second);
05922     return true;
05923   }
05924 
05925   return false;
05926 }
05927 
05928 /// visitUnaryFloatCall - If a call instruction is a unary floating-point
05929 /// operation (as expected), translate it to an SDNode with the specified opcode
05930 /// and return true.
05931 bool SelectionDAGBuilder::visitUnaryFloatCall(const CallInst &I,
05932                                               unsigned Opcode) {
05933   // Sanity check that it really is a unary floating-point call.
05934   if (I.getNumArgOperands() != 1 ||
05935       !I.getArgOperand(0)->getType()->isFloatingPointTy() ||
05936       I.getType() != I.getArgOperand(0)->getType() ||
05937       !I.onlyReadsMemory())
05938     return false;
05939 
05940   SDValue Tmp = getValue(I.getArgOperand(0));
05941   setValue(&I, DAG.getNode(Opcode, getCurSDLoc(), Tmp.getValueType(), Tmp));
05942   return true;
05943 }
05944 
05945 void SelectionDAGBuilder::visitCall(const CallInst &I) {
05946   // Handle inline assembly differently.
05947   if (isa<InlineAsm>(I.getCalledValue())) {
05948     visitInlineAsm(&I);
05949     return;
05950   }
05951 
05952   MachineModuleInfo &MMI = DAG.getMachineFunction().getMMI();
05953   ComputeUsesVAFloatArgument(I, &MMI);
05954 
05955   const char *RenameFn = nullptr;
05956   if (Function *F = I.getCalledFunction()) {
05957     if (F->isDeclaration()) {
05958       if (const TargetIntrinsicInfo *II = TM.getIntrinsicInfo()) {
05959         if (unsigned IID = II->getIntrinsicID(F)) {
05960           RenameFn = visitIntrinsicCall(I, IID);
05961           if (!RenameFn)
05962             return;
05963         }
05964       }
05965       if (unsigned IID = F->getIntrinsicID()) {
05966         RenameFn = visitIntrinsicCall(I, IID);
05967         if (!RenameFn)
05968           return;
05969       }
05970     }
05971 
05972     // Check for well-known libc/libm calls.  If the function is internal, it
05973     // can't be a library call.
05974     LibFunc::Func Func;
05975     if (!F->hasLocalLinkage() && F->hasName() &&
05976         LibInfo->getLibFunc(F->getName(), Func) &&
05977         LibInfo->hasOptimizedCodeGen(Func)) {
05978       switch (Func) {
05979       default: break;
05980       case LibFunc::copysign:
05981       case LibFunc::copysignf:
05982       case LibFunc::copysignl:
05983         if (I.getNumArgOperands() == 2 &&   // Basic sanity checks.
05984             I.getArgOperand(0)->getType()->isFloatingPointTy() &&
05985             I.getType() == I.getArgOperand(0)->getType() &&
05986             I.getType() == I.getArgOperand(1)->getType() &&
05987             I.onlyReadsMemory()) {
05988           SDValue LHS = getValue(I.getArgOperand(0));
05989           SDValue RHS = getValue(I.getArgOperand(1));
05990           setValue(&I, DAG.getNode(ISD::FCOPYSIGN, getCurSDLoc(),
05991                                    LHS.getValueType(), LHS, RHS));
05992           return;
05993         }
05994         break;
05995       case LibFunc::fabs:
05996       case LibFunc::fabsf:
05997       case LibFunc::fabsl:
05998         if (visitUnaryFloatCall(I, ISD::FABS))
05999           return;
06000         break;
06001       case LibFunc::sin:
06002       case LibFunc::sinf:
06003       case LibFunc::sinl:
06004         if (visitUnaryFloatCall(I, ISD::FSIN))
06005           return;
06006         break;
06007       case LibFunc::cos:
06008       case LibFunc::cosf:
06009       case LibFunc::cosl:
06010         if (visitUnaryFloatCall(I, ISD::FCOS))
06011           return;
06012         break;
06013       case LibFunc::sqrt:
06014       case LibFunc::sqrtf:
06015       case LibFunc::sqrtl:
06016       case LibFunc::sqrt_finite:
06017       case LibFunc::sqrtf_finite:
06018       case LibFunc::sqrtl_finite:
06019         if (visitUnaryFloatCall(I, ISD::FSQRT))
06020           return;
06021         break;
06022       case LibFunc::floor:
06023       case LibFunc::floorf:
06024       case LibFunc::floorl:
06025         if (visitUnaryFloatCall(I, ISD::FFLOOR))
06026           return;
06027         break;
06028       case LibFunc::nearbyint:
06029       case LibFunc::nearbyintf:
06030       case LibFunc::nearbyintl:
06031         if (visitUnaryFloatCall(I, ISD::FNEARBYINT))
06032           return;
06033         break;
06034       case LibFunc::ceil:
06035       case LibFunc::ceilf:
06036       case LibFunc::ceill:
06037         if (visitUnaryFloatCall(I, ISD::FCEIL))
06038           return;
06039         break;
06040       case LibFunc::rint:
06041       case LibFunc::rintf:
06042       case LibFunc::rintl:
06043         if (visitUnaryFloatCall(I, ISD::FRINT))
06044           return;
06045         break;
06046       case LibFunc::round:
06047       case LibFunc::roundf:
06048       case LibFunc::roundl:
06049         if (visitUnaryFloatCall(I, ISD::FROUND))
06050           return;
06051         break;
06052       case LibFunc::trunc:
06053       case LibFunc::truncf:
06054       case LibFunc::truncl:
06055         if (visitUnaryFloatCall(I, ISD::FTRUNC))
06056           return;
06057         break;
06058       case LibFunc::log2:
06059       case LibFunc::log2f:
06060       case LibFunc::log2l:
06061         if (visitUnaryFloatCall(I, ISD::FLOG2))
06062           return;
06063         break;
06064       case LibFunc::exp2:
06065       case LibFunc::exp2f:
06066       case LibFunc::exp2l:
06067         if (visitUnaryFloatCall(I, ISD::FEXP2))
06068           return;
06069         break;
06070       case LibFunc::memcmp:
06071         if (visitMemCmpCall(I))
06072           return;
06073         break;
06074       case LibFunc::memchr:
06075         if (visitMemChrCall(I))
06076           return;
06077         break;
06078       case LibFunc::strcpy:
06079         if (visitStrCpyCall(I, false))
06080           return;
06081         break;
06082       case LibFunc::stpcpy:
06083         if (visitStrCpyCall(I, true))
06084           return;
06085         break;
06086       case LibFunc::strcmp:
06087         if (visitStrCmpCall(I))
06088           return;
06089         break;
06090       case LibFunc::strlen:
06091         if (visitStrLenCall(I))
06092           return;
06093         break;
06094       case LibFunc::strnlen:
06095         if (visitStrNLenCall(I))
06096           return;
06097         break;
06098       }
06099     }
06100   }
06101 
06102   SDValue Callee;
06103   if (!RenameFn)
06104     Callee = getValue(I.getCalledValue());
06105   else
06106     Callee = DAG.getExternalSymbol(
06107         RenameFn, TM.getSubtargetImpl()->getTargetLowering()->getPointerTy());
06108 
06109   // Check if we can potentially perform a tail call. More detailed checking is
06110   // be done within LowerCallTo, after more information about the call is known.
06111   LowerCallTo(&I, Callee, I.isTailCall());
06112 }
06113 
06114 namespace {
06115 
06116 /// AsmOperandInfo - This contains information for each constraint that we are
06117 /// lowering.
06118 class SDISelAsmOperandInfo : public TargetLowering::AsmOperandInfo {
06119 public:
06120   /// CallOperand - If this is the result output operand or a clobber
06121   /// this is null, otherwise it is the incoming operand to the CallInst.
06122   /// This gets modified as the asm is processed.
06123   SDValue CallOperand;
06124 
06125   /// AssignedRegs - If this is a register or register class operand, this
06126   /// contains the set of register corresponding to the operand.
06127   RegsForValue AssignedRegs;
06128 
06129   explicit SDISelAsmOperandInfo(const TargetLowering::AsmOperandInfo &info)
06130     : TargetLowering::AsmOperandInfo(info), CallOperand(nullptr,0) {
06131   }
06132 
06133   /// getCallOperandValEVT - Return the EVT of the Value* that this operand
06134   /// corresponds to.  If there is no Value* for this operand, it returns
06135   /// MVT::Other.
06136   EVT getCallOperandValEVT(LLVMContext &Context,
06137                            const TargetLowering &TLI,
06138                            const DataLayout *DL) const {
06139     if (!CallOperandVal) return MVT::Other;
06140 
06141     if (isa<BasicBlock>(CallOperandVal))
06142       return TLI.getPointerTy();
06143 
06144     llvm::Type *OpTy = CallOperandVal->getType();
06145 
06146     // FIXME: code duplicated from TargetLowering::ParseConstraints().
06147     // If this is an indirect operand, the operand is a pointer to the
06148     // accessed type.
06149     if (isIndirect) {
06150       llvm::PointerType *PtrTy = dyn_cast<PointerType>(OpTy);
06151       if (!PtrTy)
06152         report_fatal_error("Indirect operand for inline asm not a pointer!");
06153       OpTy = PtrTy->getElementType();
06154     }
06155 
06156     // Look for vector wrapped in a struct. e.g. { <16 x i8> }.
06157     if (StructType *STy = dyn_cast<StructType>(OpTy))
06158       if (STy->getNumElements() == 1)
06159         OpTy = STy->getElementType(0);
06160 
06161     // If OpTy is not a single value, it may be a struct/union that we
06162     // can tile with integers.
06163     if (!OpTy->isSingleValueType() && OpTy->isSized()) {
06164       unsigned BitSize = DL->getTypeSizeInBits(OpTy);
06165       switch (BitSize) {
06166       default: break;
06167       case 1:
06168       case 8:
06169       case 16:
06170       case 32:
06171       case 64:
06172       case 128:
06173         OpTy = IntegerType::get(Context, BitSize);
06174         break;
06175       }
06176     }
06177 
06178     return TLI.getValueType(OpTy, true);
06179   }
06180 };
06181 
06182 typedef SmallVector<SDISelAsmOperandInfo,16> SDISelAsmOperandInfoVector;
06183 
06184 } // end anonymous namespace
06185 
06186 /// GetRegistersForValue - Assign registers (virtual or physical) for the
06187 /// specified operand.  We prefer to assign virtual registers, to allow the
06188 /// register allocator to handle the assignment process.  However, if the asm
06189 /// uses features that we can't model on machineinstrs, we have SDISel do the
06190 /// allocation.  This produces generally horrible, but correct, code.
06191 ///
06192 ///   OpInfo describes the operand.
06193 ///
06194 static void GetRegistersForValue(SelectionDAG &DAG,
06195                                  const TargetLowering &TLI,
06196                                  SDLoc DL,