LLVM API Documentation

SelectionDAGBuilder.cpp
Go to the documentation of this file.
00001 //===-- SelectionDAGBuilder.cpp - Selection-DAG building ------------------===//
00002 //
00003 //                     The LLVM Compiler Infrastructure
00004 //
00005 // This file is distributed under the University of Illinois Open Source
00006 // License. See LICENSE.TXT for details.
00007 //
00008 //===----------------------------------------------------------------------===//
00009 //
00010 // This implements routines for translating from LLVM IR into SelectionDAG IR.
00011 //
00012 //===----------------------------------------------------------------------===//
00013 
00014 #include "SelectionDAGBuilder.h"
00015 #include "SDNodeDbgValue.h"
00016 #include "llvm/ADT/BitVector.h"
00017 #include "llvm/ADT/Optional.h"
00018 #include "llvm/ADT/SmallSet.h"
00019 #include "llvm/Analysis/AliasAnalysis.h"
00020 #include "llvm/Analysis/BranchProbabilityInfo.h"
00021 #include "llvm/Analysis/ConstantFolding.h"
00022 #include "llvm/Analysis/ValueTracking.h"
00023 #include "llvm/CodeGen/Analysis.h"
00024 #include "llvm/CodeGen/FastISel.h"
00025 #include "llvm/CodeGen/FunctionLoweringInfo.h"
00026 #include "llvm/CodeGen/GCMetadata.h"
00027 #include "llvm/CodeGen/GCStrategy.h"
00028 #include "llvm/CodeGen/MachineFrameInfo.h"
00029 #include "llvm/CodeGen/MachineFunction.h"
00030 #include "llvm/CodeGen/MachineInstrBuilder.h"
00031 #include "llvm/CodeGen/MachineJumpTableInfo.h"
00032 #include "llvm/CodeGen/MachineModuleInfo.h"
00033 #include "llvm/CodeGen/MachineRegisterInfo.h"
00034 #include "llvm/CodeGen/SelectionDAG.h"
00035 #include "llvm/CodeGen/StackMaps.h"
00036 #include "llvm/IR/CallingConv.h"
00037 #include "llvm/IR/Constants.h"
00038 #include "llvm/IR/DataLayout.h"
00039 #include "llvm/IR/DebugInfo.h"
00040 #include "llvm/IR/DerivedTypes.h"
00041 #include "llvm/IR/Function.h"
00042 #include "llvm/IR/GlobalVariable.h"
00043 #include "llvm/IR/InlineAsm.h"
00044 #include "llvm/IR/Instructions.h"
00045 #include "llvm/IR/IntrinsicInst.h"
00046 #include "llvm/IR/Intrinsics.h"
00047 #include "llvm/IR/LLVMContext.h"
00048 #include "llvm/IR/Module.h"
00049 #include "llvm/Support/CommandLine.h"
00050 #include "llvm/Support/Debug.h"
00051 #include "llvm/Support/ErrorHandling.h"
00052 #include "llvm/Support/MathExtras.h"
00053 #include "llvm/Support/raw_ostream.h"
00054 #include "llvm/Target/TargetFrameLowering.h"
00055 #include "llvm/Target/TargetInstrInfo.h"
00056 #include "llvm/Target/TargetIntrinsicInfo.h"
00057 #include "llvm/Target/TargetLibraryInfo.h"
00058 #include "llvm/Target/TargetLowering.h"
00059 #include "llvm/Target/TargetOptions.h"
00060 #include "llvm/Target/TargetSelectionDAGInfo.h"
00061 #include "llvm/Target/TargetSubtargetInfo.h"
00062 #include <algorithm>
00063 using namespace llvm;
00064 
00065 #define DEBUG_TYPE "isel"
00066 
00067 /// LimitFloatPrecision - Generate low-precision inline sequences for
00068 /// some float libcalls (6, 8 or 12 bits).
00069 static unsigned LimitFloatPrecision;
00070 
00071 static cl::opt<unsigned, true>
00072 LimitFPPrecision("limit-float-precision",
00073                  cl::desc("Generate low-precision inline sequences "
00074                           "for some float libcalls"),
00075                  cl::location(LimitFloatPrecision),
00076                  cl::init(0));
00077 
00078 // Limit the width of DAG chains. This is important in general to prevent
00079 // prevent DAG-based analysis from blowing up. For example, alias analysis and
00080 // load clustering may not complete in reasonable time. It is difficult to
00081 // recognize and avoid this situation within each individual analysis, and
00082 // future analyses are likely to have the same behavior. Limiting DAG width is
00083 // the safe approach, and will be especially important with global DAGs.
00084 //
00085 // MaxParallelChains default is arbitrarily high to avoid affecting
00086 // optimization, but could be lowered to improve compile time. Any ld-ld-st-st
00087 // sequence over this should have been converted to llvm.memcpy by the
00088 // frontend. It easy to induce this behavior with .ll code such as:
00089 // %buffer = alloca [4096 x i8]
00090 // %data = load [4096 x i8]* %argPtr
00091 // store [4096 x i8] %data, [4096 x i8]* %buffer
00092 static const unsigned MaxParallelChains = 64;
00093 
00094 static SDValue getCopyFromPartsVector(SelectionDAG &DAG, SDLoc DL,
00095                                       const SDValue *Parts, unsigned NumParts,
00096                                       MVT PartVT, EVT ValueVT, const Value *V);
00097 
00098 /// getCopyFromParts - Create a value that contains the specified legal parts
00099 /// combined into the value they represent.  If the parts combine to a type
00100 /// larger then ValueVT then AssertOp can be used to specify whether the extra
00101 /// bits are known to be zero (ISD::AssertZext) or sign extended from ValueVT
00102 /// (ISD::AssertSext).
00103 static SDValue getCopyFromParts(SelectionDAG &DAG, SDLoc DL,
00104                                 const SDValue *Parts,
00105                                 unsigned NumParts, MVT PartVT, EVT ValueVT,
00106                                 const Value *V,
00107                                 ISD::NodeType AssertOp = ISD::DELETED_NODE) {
00108   if (ValueVT.isVector())
00109     return getCopyFromPartsVector(DAG, DL, Parts, NumParts,
00110                                   PartVT, ValueVT, V);
00111 
00112   assert(NumParts > 0 && "No parts to assemble!");
00113   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
00114   SDValue Val = Parts[0];
00115 
00116   if (NumParts > 1) {
00117     // Assemble the value from multiple parts.
00118     if (ValueVT.isInteger()) {
00119       unsigned PartBits = PartVT.getSizeInBits();
00120       unsigned ValueBits = ValueVT.getSizeInBits();
00121 
00122       // Assemble the power of 2 part.
00123       unsigned RoundParts = NumParts & (NumParts - 1) ?
00124         1 << Log2_32(NumParts) : NumParts;
00125       unsigned RoundBits = PartBits * RoundParts;
00126       EVT RoundVT = RoundBits == ValueBits ?
00127         ValueVT : EVT::getIntegerVT(*DAG.getContext(), RoundBits);
00128       SDValue Lo, Hi;
00129 
00130       EVT HalfVT = EVT::getIntegerVT(*DAG.getContext(), RoundBits/2);
00131 
00132       if (RoundParts > 2) {
00133         Lo = getCopyFromParts(DAG, DL, Parts, RoundParts / 2,
00134                               PartVT, HalfVT, V);
00135         Hi = getCopyFromParts(DAG, DL, Parts + RoundParts / 2,
00136                               RoundParts / 2, PartVT, HalfVT, V);
00137       } else {
00138         Lo = DAG.getNode(ISD::BITCAST, DL, HalfVT, Parts[0]);
00139         Hi = DAG.getNode(ISD::BITCAST, DL, HalfVT, Parts[1]);
00140       }
00141 
00142       if (TLI.isBigEndian())
00143         std::swap(Lo, Hi);
00144 
00145       Val = DAG.getNode(ISD::BUILD_PAIR, DL, RoundVT, Lo, Hi);
00146 
00147       if (RoundParts < NumParts) {
00148         // Assemble the trailing non-power-of-2 part.
00149         unsigned OddParts = NumParts - RoundParts;
00150         EVT OddVT = EVT::getIntegerVT(*DAG.getContext(), OddParts * PartBits);
00151         Hi = getCopyFromParts(DAG, DL,
00152                               Parts + RoundParts, OddParts, PartVT, OddVT, V);
00153 
00154         // Combine the round and odd parts.
00155         Lo = Val;
00156         if (TLI.isBigEndian())
00157           std::swap(Lo, Hi);
00158         EVT TotalVT = EVT::getIntegerVT(*DAG.getContext(), NumParts * PartBits);
00159         Hi = DAG.getNode(ISD::ANY_EXTEND, DL, TotalVT, Hi);
00160         Hi = DAG.getNode(ISD::SHL, DL, TotalVT, Hi,
00161                          DAG.getConstant(Lo.getValueType().getSizeInBits(),
00162                                          TLI.getPointerTy()));
00163         Lo = DAG.getNode(ISD::ZERO_EXTEND, DL, TotalVT, Lo);
00164         Val = DAG.getNode(ISD::OR, DL, TotalVT, Lo, Hi);
00165       }
00166     } else if (PartVT.isFloatingPoint()) {
00167       // FP split into multiple FP parts (for ppcf128)
00168       assert(ValueVT == EVT(MVT::ppcf128) && PartVT == MVT::f64 &&
00169              "Unexpected split");
00170       SDValue Lo, Hi;
00171       Lo = DAG.getNode(ISD::BITCAST, DL, EVT(MVT::f64), Parts[0]);
00172       Hi = DAG.getNode(ISD::BITCAST, DL, EVT(MVT::f64), Parts[1]);
00173       if (TLI.hasBigEndianPartOrdering(ValueVT))
00174         std::swap(Lo, Hi);
00175       Val = DAG.getNode(ISD::BUILD_PAIR, DL, ValueVT, Lo, Hi);
00176     } else {
00177       // FP split into integer parts (soft fp)
00178       assert(ValueVT.isFloatingPoint() && PartVT.isInteger() &&
00179              !PartVT.isVector() && "Unexpected split");
00180       EVT IntVT = EVT::getIntegerVT(*DAG.getContext(), ValueVT.getSizeInBits());
00181       Val = getCopyFromParts(DAG, DL, Parts, NumParts, PartVT, IntVT, V);
00182     }
00183   }
00184 
00185   // There is now one part, held in Val.  Correct it to match ValueVT.
00186   EVT PartEVT = Val.getValueType();
00187 
00188   if (PartEVT == ValueVT)
00189     return Val;
00190 
00191   if (PartEVT.isInteger() && ValueVT.isInteger()) {
00192     if (ValueVT.bitsLT(PartEVT)) {
00193       // For a truncate, see if we have any information to
00194       // indicate whether the truncated bits will always be
00195       // zero or sign-extension.
00196       if (AssertOp != ISD::DELETED_NODE)
00197         Val = DAG.getNode(AssertOp, DL, PartEVT, Val,
00198                           DAG.getValueType(ValueVT));
00199       return DAG.getNode(ISD::TRUNCATE, DL, ValueVT, Val);
00200     }
00201     return DAG.getNode(ISD::ANY_EXTEND, DL, ValueVT, Val);
00202   }
00203 
00204   if (PartEVT.isFloatingPoint() && ValueVT.isFloatingPoint()) {
00205     // FP_ROUND's are always exact here.
00206     if (ValueVT.bitsLT(Val.getValueType()))
00207       return DAG.getNode(ISD::FP_ROUND, DL, ValueVT, Val,
00208                          DAG.getTargetConstant(1, TLI.getPointerTy()));
00209 
00210     return DAG.getNode(ISD::FP_EXTEND, DL, ValueVT, Val);
00211   }
00212 
00213   if (PartEVT.getSizeInBits() == ValueVT.getSizeInBits())
00214     return DAG.getNode(ISD::BITCAST, DL, ValueVT, Val);
00215 
00216   llvm_unreachable("Unknown mismatch!");
00217 }
00218 
00219 static void diagnosePossiblyInvalidConstraint(LLVMContext &Ctx, const Value *V,
00220                                               const Twine &ErrMsg) {
00221   const Instruction *I = dyn_cast_or_null<Instruction>(V);
00222   if (!V)
00223     return Ctx.emitError(ErrMsg);
00224 
00225   const char *AsmError = ", possible invalid constraint for vector type";
00226   if (const CallInst *CI = dyn_cast<CallInst>(I))
00227     if (isa<InlineAsm>(CI->getCalledValue()))
00228       return Ctx.emitError(I, ErrMsg + AsmError);
00229 
00230   return Ctx.emitError(I, ErrMsg);
00231 }
00232 
00233 /// getCopyFromPartsVector - Create a value that contains the specified legal
00234 /// parts combined into the value they represent.  If the parts combine to a
00235 /// type larger then ValueVT then AssertOp can be used to specify whether the
00236 /// extra bits are known to be zero (ISD::AssertZext) or sign extended from
00237 /// ValueVT (ISD::AssertSext).
00238 static SDValue getCopyFromPartsVector(SelectionDAG &DAG, SDLoc DL,
00239                                       const SDValue *Parts, unsigned NumParts,
00240                                       MVT PartVT, EVT ValueVT, const Value *V) {
00241   assert(ValueVT.isVector() && "Not a vector value");
00242   assert(NumParts > 0 && "No parts to assemble!");
00243   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
00244   SDValue Val = Parts[0];
00245 
00246   // Handle a multi-element vector.
00247   if (NumParts > 1) {
00248     EVT IntermediateVT;
00249     MVT RegisterVT;
00250     unsigned NumIntermediates;
00251     unsigned NumRegs =
00252     TLI.getVectorTypeBreakdown(*DAG.getContext(), ValueVT, IntermediateVT,
00253                                NumIntermediates, RegisterVT);
00254     assert(NumRegs == NumParts && "Part count doesn't match vector breakdown!");
00255     NumParts = NumRegs; // Silence a compiler warning.
00256     assert(RegisterVT == PartVT && "Part type doesn't match vector breakdown!");
00257     assert(RegisterVT == Parts[0].getSimpleValueType() &&
00258            "Part type doesn't match part!");
00259 
00260     // Assemble the parts into intermediate operands.
00261     SmallVector<SDValue, 8> Ops(NumIntermediates);
00262     if (NumIntermediates == NumParts) {
00263       // If the register was not expanded, truncate or copy the value,
00264       // as appropriate.
00265       for (unsigned i = 0; i != NumParts; ++i)
00266         Ops[i] = getCopyFromParts(DAG, DL, &Parts[i], 1,
00267                                   PartVT, IntermediateVT, V);
00268     } else if (NumParts > 0) {
00269       // If the intermediate type was expanded, build the intermediate
00270       // operands from the parts.
00271       assert(NumParts % NumIntermediates == 0 &&
00272              "Must expand into a divisible number of parts!");
00273       unsigned Factor = NumParts / NumIntermediates;
00274       for (unsigned i = 0; i != NumIntermediates; ++i)
00275         Ops[i] = getCopyFromParts(DAG, DL, &Parts[i * Factor], Factor,
00276                                   PartVT, IntermediateVT, V);
00277     }
00278 
00279     // Build a vector with BUILD_VECTOR or CONCAT_VECTORS from the
00280     // intermediate operands.
00281     Val = DAG.getNode(IntermediateVT.isVector() ? ISD::CONCAT_VECTORS
00282                                                 : ISD::BUILD_VECTOR,
00283                       DL, ValueVT, Ops);
00284   }
00285 
00286   // There is now one part, held in Val.  Correct it to match ValueVT.
00287   EVT PartEVT = Val.getValueType();
00288 
00289   if (PartEVT == ValueVT)
00290     return Val;
00291 
00292   if (PartEVT.isVector()) {
00293     // If the element type of the source/dest vectors are the same, but the
00294     // parts vector has more elements than the value vector, then we have a
00295     // vector widening case (e.g. <2 x float> -> <4 x float>).  Extract the
00296     // elements we want.
00297     if (PartEVT.getVectorElementType() == ValueVT.getVectorElementType()) {
00298       assert(PartEVT.getVectorNumElements() > ValueVT.getVectorNumElements() &&
00299              "Cannot narrow, it would be a lossy transformation");
00300       return DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, ValueVT, Val,
00301                          DAG.getConstant(0, TLI.getVectorIdxTy()));
00302     }
00303 
00304     // Vector/Vector bitcast.
00305     if (ValueVT.getSizeInBits() == PartEVT.getSizeInBits())
00306       return DAG.getNode(ISD::BITCAST, DL, ValueVT, Val);
00307 
00308     assert(PartEVT.getVectorNumElements() == ValueVT.getVectorNumElements() &&
00309       "Cannot handle this kind of promotion");
00310     // Promoted vector extract
00311     bool Smaller = ValueVT.bitsLE(PartEVT);
00312     return DAG.getNode((Smaller ? ISD::TRUNCATE : ISD::ANY_EXTEND),
00313                        DL, ValueVT, Val);
00314 
00315   }
00316 
00317   // Trivial bitcast if the types are the same size and the destination
00318   // vector type is legal.
00319   if (PartEVT.getSizeInBits() == ValueVT.getSizeInBits() &&
00320       TLI.isTypeLegal(ValueVT))
00321     return DAG.getNode(ISD::BITCAST, DL, ValueVT, Val);
00322 
00323   // Handle cases such as i8 -> <1 x i1>
00324   if (ValueVT.getVectorNumElements() != 1) {
00325     diagnosePossiblyInvalidConstraint(*DAG.getContext(), V,
00326                                       "non-trivial scalar-to-vector conversion");
00327     return DAG.getUNDEF(ValueVT);
00328   }
00329 
00330   if (ValueVT.getVectorNumElements() == 1 &&
00331       ValueVT.getVectorElementType() != PartEVT) {
00332     bool Smaller = ValueVT.bitsLE(PartEVT);
00333     Val = DAG.getNode((Smaller ? ISD::TRUNCATE : ISD::ANY_EXTEND),
00334                        DL, ValueVT.getScalarType(), Val);
00335   }
00336 
00337   return DAG.getNode(ISD::BUILD_VECTOR, DL, ValueVT, Val);
00338 }
00339 
00340 static void getCopyToPartsVector(SelectionDAG &DAG, SDLoc dl,
00341                                  SDValue Val, SDValue *Parts, unsigned NumParts,
00342                                  MVT PartVT, const Value *V);
00343 
00344 /// getCopyToParts - Create a series of nodes that contain the specified value
00345 /// split into legal parts.  If the parts contain more bits than Val, then, for
00346 /// integers, ExtendKind can be used to specify how to generate the extra bits.
00347 static void getCopyToParts(SelectionDAG &DAG, SDLoc DL,
00348                            SDValue Val, SDValue *Parts, unsigned NumParts,
00349                            MVT PartVT, const Value *V,
00350                            ISD::NodeType ExtendKind = ISD::ANY_EXTEND) {
00351   EVT ValueVT = Val.getValueType();
00352 
00353   // Handle the vector case separately.
00354   if (ValueVT.isVector())
00355     return getCopyToPartsVector(DAG, DL, Val, Parts, NumParts, PartVT, V);
00356 
00357   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
00358   unsigned PartBits = PartVT.getSizeInBits();
00359   unsigned OrigNumParts = NumParts;
00360   assert(TLI.isTypeLegal(PartVT) && "Copying to an illegal type!");
00361 
00362   if (NumParts == 0)
00363     return;
00364 
00365   assert(!ValueVT.isVector() && "Vector case handled elsewhere");
00366   EVT PartEVT = PartVT;
00367   if (PartEVT == ValueVT) {
00368     assert(NumParts == 1 && "No-op copy with multiple parts!");
00369     Parts[0] = Val;
00370     return;
00371   }
00372 
00373   if (NumParts * PartBits > ValueVT.getSizeInBits()) {
00374     // If the parts cover more bits than the value has, promote the value.
00375     if (PartVT.isFloatingPoint() && ValueVT.isFloatingPoint()) {
00376       assert(NumParts == 1 && "Do not know what to promote to!");
00377       Val = DAG.getNode(ISD::FP_EXTEND, DL, PartVT, Val);
00378     } else {
00379       assert((PartVT.isInteger() || PartVT == MVT::x86mmx) &&
00380              ValueVT.isInteger() &&
00381              "Unknown mismatch!");
00382       ValueVT = EVT::getIntegerVT(*DAG.getContext(), NumParts * PartBits);
00383       Val = DAG.getNode(ExtendKind, DL, ValueVT, Val);
00384       if (PartVT == MVT::x86mmx)
00385         Val = DAG.getNode(ISD::BITCAST, DL, PartVT, Val);
00386     }
00387   } else if (PartBits == ValueVT.getSizeInBits()) {
00388     // Different types of the same size.
00389     assert(NumParts == 1 && PartEVT != ValueVT);
00390     Val = DAG.getNode(ISD::BITCAST, DL, PartVT, Val);
00391   } else if (NumParts * PartBits < ValueVT.getSizeInBits()) {
00392     // If the parts cover less bits than value has, truncate the value.
00393     assert((PartVT.isInteger() || PartVT == MVT::x86mmx) &&
00394            ValueVT.isInteger() &&
00395            "Unknown mismatch!");
00396     ValueVT = EVT::getIntegerVT(*DAG.getContext(), NumParts * PartBits);
00397     Val = DAG.getNode(ISD::TRUNCATE, DL, ValueVT, Val);
00398     if (PartVT == MVT::x86mmx)
00399       Val = DAG.getNode(ISD::BITCAST, DL, PartVT, Val);
00400   }
00401 
00402   // The value may have changed - recompute ValueVT.
00403   ValueVT = Val.getValueType();
00404   assert(NumParts * PartBits == ValueVT.getSizeInBits() &&
00405          "Failed to tile the value with PartVT!");
00406 
00407   if (NumParts == 1) {
00408     if (PartEVT != ValueVT)
00409       diagnosePossiblyInvalidConstraint(*DAG.getContext(), V,
00410                                         "scalar-to-vector conversion failed");
00411 
00412     Parts[0] = Val;
00413     return;
00414   }
00415 
00416   // Expand the value into multiple parts.
00417   if (NumParts & (NumParts - 1)) {
00418     // The number of parts is not a power of 2.  Split off and copy the tail.
00419     assert(PartVT.isInteger() && ValueVT.isInteger() &&
00420            "Do not know what to expand to!");
00421     unsigned RoundParts = 1 << Log2_32(NumParts);
00422     unsigned RoundBits = RoundParts * PartBits;
00423     unsigned OddParts = NumParts - RoundParts;
00424     SDValue OddVal = DAG.getNode(ISD::SRL, DL, ValueVT, Val,
00425                                  DAG.getIntPtrConstant(RoundBits));
00426     getCopyToParts(DAG, DL, OddVal, Parts + RoundParts, OddParts, PartVT, V);
00427 
00428     if (TLI.isBigEndian())
00429       // The odd parts were reversed by getCopyToParts - unreverse them.
00430       std::reverse(Parts + RoundParts, Parts + NumParts);
00431 
00432     NumParts = RoundParts;
00433     ValueVT = EVT::getIntegerVT(*DAG.getContext(), NumParts * PartBits);
00434     Val = DAG.getNode(ISD::TRUNCATE, DL, ValueVT, Val);
00435   }
00436 
00437   // The number of parts is a power of 2.  Repeatedly bisect the value using
00438   // EXTRACT_ELEMENT.
00439   Parts[0] = DAG.getNode(ISD::BITCAST, DL,
00440                          EVT::getIntegerVT(*DAG.getContext(),
00441                                            ValueVT.getSizeInBits()),
00442                          Val);
00443 
00444   for (unsigned StepSize = NumParts; StepSize > 1; StepSize /= 2) {
00445     for (unsigned i = 0; i < NumParts; i += StepSize) {
00446       unsigned ThisBits = StepSize * PartBits / 2;
00447       EVT ThisVT = EVT::getIntegerVT(*DAG.getContext(), ThisBits);
00448       SDValue &Part0 = Parts[i];
00449       SDValue &Part1 = Parts[i+StepSize/2];
00450 
00451       Part1 = DAG.getNode(ISD::EXTRACT_ELEMENT, DL,
00452                           ThisVT, Part0, DAG.getIntPtrConstant(1));
00453       Part0 = DAG.getNode(ISD::EXTRACT_ELEMENT, DL,
00454                           ThisVT, Part0, DAG.getIntPtrConstant(0));
00455 
00456       if (ThisBits == PartBits && ThisVT != PartVT) {
00457         Part0 = DAG.getNode(ISD::BITCAST, DL, PartVT, Part0);
00458         Part1 = DAG.getNode(ISD::BITCAST, DL, PartVT, Part1);
00459       }
00460     }
00461   }
00462 
00463   if (TLI.isBigEndian())
00464     std::reverse(Parts, Parts + OrigNumParts);
00465 }
00466 
00467 
00468 /// getCopyToPartsVector - Create a series of nodes that contain the specified
00469 /// value split into legal parts.
00470 static void getCopyToPartsVector(SelectionDAG &DAG, SDLoc DL,
00471                                  SDValue Val, SDValue *Parts, unsigned NumParts,
00472                                  MVT PartVT, const Value *V) {
00473   EVT ValueVT = Val.getValueType();
00474   assert(ValueVT.isVector() && "Not a vector");
00475   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
00476 
00477   if (NumParts == 1) {
00478     EVT PartEVT = PartVT;
00479     if (PartEVT == ValueVT) {
00480       // Nothing to do.
00481     } else if (PartVT.getSizeInBits() == ValueVT.getSizeInBits()) {
00482       // Bitconvert vector->vector case.
00483       Val = DAG.getNode(ISD::BITCAST, DL, PartVT, Val);
00484     } else if (PartVT.isVector() &&
00485                PartEVT.getVectorElementType() == ValueVT.getVectorElementType() &&
00486                PartEVT.getVectorNumElements() > ValueVT.getVectorNumElements()) {
00487       EVT ElementVT = PartVT.getVectorElementType();
00488       // Vector widening case, e.g. <2 x float> -> <4 x float>.  Shuffle in
00489       // undef elements.
00490       SmallVector<SDValue, 16> Ops;
00491       for (unsigned i = 0, e = ValueVT.getVectorNumElements(); i != e; ++i)
00492         Ops.push_back(DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL,
00493                                   ElementVT, Val, DAG.getConstant(i,
00494                                                   TLI.getVectorIdxTy())));
00495 
00496       for (unsigned i = ValueVT.getVectorNumElements(),
00497            e = PartVT.getVectorNumElements(); i != e; ++i)
00498         Ops.push_back(DAG.getUNDEF(ElementVT));
00499 
00500       Val = DAG.getNode(ISD::BUILD_VECTOR, DL, PartVT, Ops);
00501 
00502       // FIXME: Use CONCAT for 2x -> 4x.
00503 
00504       //SDValue UndefElts = DAG.getUNDEF(VectorTy);
00505       //Val = DAG.getNode(ISD::CONCAT_VECTORS, DL, PartVT, Val, UndefElts);
00506     } else if (PartVT.isVector() &&
00507                PartEVT.getVectorElementType().bitsGE(
00508                  ValueVT.getVectorElementType()) &&
00509                PartEVT.getVectorNumElements() == ValueVT.getVectorNumElements()) {
00510 
00511       // Promoted vector extract
00512       bool Smaller = PartEVT.bitsLE(ValueVT);
00513       Val = DAG.getNode((Smaller ? ISD::TRUNCATE : ISD::ANY_EXTEND),
00514                         DL, PartVT, Val);
00515     } else{
00516       // Vector -> scalar conversion.
00517       assert(ValueVT.getVectorNumElements() == 1 &&
00518              "Only trivial vector-to-scalar conversions should get here!");
00519       Val = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL,
00520                         PartVT, Val, DAG.getConstant(0, TLI.getVectorIdxTy()));
00521 
00522       bool Smaller = ValueVT.bitsLE(PartVT);
00523       Val = DAG.getNode((Smaller ? ISD::TRUNCATE : ISD::ANY_EXTEND),
00524                          DL, PartVT, Val);
00525     }
00526 
00527     Parts[0] = Val;
00528     return;
00529   }
00530 
00531   // Handle a multi-element vector.
00532   EVT IntermediateVT;
00533   MVT RegisterVT;
00534   unsigned NumIntermediates;
00535   unsigned NumRegs = TLI.getVectorTypeBreakdown(*DAG.getContext(), ValueVT,
00536                                                 IntermediateVT,
00537                                                 NumIntermediates, RegisterVT);
00538   unsigned NumElements = ValueVT.getVectorNumElements();
00539 
00540   assert(NumRegs == NumParts && "Part count doesn't match vector breakdown!");
00541   NumParts = NumRegs; // Silence a compiler warning.
00542   assert(RegisterVT == PartVT && "Part type doesn't match vector breakdown!");
00543 
00544   // Split the vector into intermediate operands.
00545   SmallVector<SDValue, 8> Ops(NumIntermediates);
00546   for (unsigned i = 0; i != NumIntermediates; ++i) {
00547     if (IntermediateVT.isVector())
00548       Ops[i] = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL,
00549                            IntermediateVT, Val,
00550                    DAG.getConstant(i * (NumElements / NumIntermediates),
00551                                    TLI.getVectorIdxTy()));
00552     else
00553       Ops[i] = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL,
00554                            IntermediateVT, Val,
00555                            DAG.getConstant(i, TLI.getVectorIdxTy()));
00556   }
00557 
00558   // Split the intermediate operands into legal parts.
00559   if (NumParts == NumIntermediates) {
00560     // If the register was not expanded, promote or copy the value,
00561     // as appropriate.
00562     for (unsigned i = 0; i != NumParts; ++i)
00563       getCopyToParts(DAG, DL, Ops[i], &Parts[i], 1, PartVT, V);
00564   } else if (NumParts > 0) {
00565     // If the intermediate type was expanded, split each the value into
00566     // legal parts.
00567     assert(NumParts % NumIntermediates == 0 &&
00568            "Must expand into a divisible number of parts!");
00569     unsigned Factor = NumParts / NumIntermediates;
00570     for (unsigned i = 0; i != NumIntermediates; ++i)
00571       getCopyToParts(DAG, DL, Ops[i], &Parts[i*Factor], Factor, PartVT, V);
00572   }
00573 }
00574 
00575 namespace {
00576   /// RegsForValue - This struct represents the registers (physical or virtual)
00577   /// that a particular set of values is assigned, and the type information
00578   /// about the value. The most common situation is to represent one value at a
00579   /// time, but struct or array values are handled element-wise as multiple
00580   /// values.  The splitting of aggregates is performed recursively, so that we
00581   /// never have aggregate-typed registers. The values at this point do not
00582   /// necessarily have legal types, so each value may require one or more
00583   /// registers of some legal type.
00584   ///
00585   struct RegsForValue {
00586     /// ValueVTs - The value types of the values, which may not be legal, and
00587     /// may need be promoted or synthesized from one or more registers.
00588     ///
00589     SmallVector<EVT, 4> ValueVTs;
00590 
00591     /// RegVTs - The value types of the registers. This is the same size as
00592     /// ValueVTs and it records, for each value, what the type of the assigned
00593     /// register or registers are. (Individual values are never synthesized
00594     /// from more than one type of register.)
00595     ///
00596     /// With virtual registers, the contents of RegVTs is redundant with TLI's
00597     /// getRegisterType member function, however when with physical registers
00598     /// it is necessary to have a separate record of the types.
00599     ///
00600     SmallVector<MVT, 4> RegVTs;
00601 
00602     /// Regs - This list holds the registers assigned to the values.
00603     /// Each legal or promoted value requires one register, and each
00604     /// expanded value requires multiple registers.
00605     ///
00606     SmallVector<unsigned, 4> Regs;
00607 
00608     RegsForValue() {}
00609 
00610     RegsForValue(const SmallVector<unsigned, 4> &regs,
00611                  MVT regvt, EVT valuevt)
00612       : ValueVTs(1, valuevt), RegVTs(1, regvt), Regs(regs) {}
00613 
00614     RegsForValue(LLVMContext &Context, const TargetLowering &tli,
00615                  unsigned Reg, Type *Ty) {
00616       ComputeValueVTs(tli, Ty, ValueVTs);
00617 
00618       for (unsigned Value = 0, e = ValueVTs.size(); Value != e; ++Value) {
00619         EVT ValueVT = ValueVTs[Value];
00620         unsigned NumRegs = tli.getNumRegisters(Context, ValueVT);
00621         MVT RegisterVT = tli.getRegisterType(Context, ValueVT);
00622         for (unsigned i = 0; i != NumRegs; ++i)
00623           Regs.push_back(Reg + i);
00624         RegVTs.push_back(RegisterVT);
00625         Reg += NumRegs;
00626       }
00627     }
00628 
00629     /// append - Add the specified values to this one.
00630     void append(const RegsForValue &RHS) {
00631       ValueVTs.append(RHS.ValueVTs.begin(), RHS.ValueVTs.end());
00632       RegVTs.append(RHS.RegVTs.begin(), RHS.RegVTs.end());
00633       Regs.append(RHS.Regs.begin(), RHS.Regs.end());
00634     }
00635 
00636     /// getCopyFromRegs - Emit a series of CopyFromReg nodes that copies from
00637     /// this value and returns the result as a ValueVTs value.  This uses
00638     /// Chain/Flag as the input and updates them for the output Chain/Flag.
00639     /// If the Flag pointer is NULL, no flag is used.
00640     SDValue getCopyFromRegs(SelectionDAG &DAG, FunctionLoweringInfo &FuncInfo,
00641                             SDLoc dl,
00642                             SDValue &Chain, SDValue *Flag,
00643                             const Value *V = nullptr) const;
00644 
00645     /// getCopyToRegs - Emit a series of CopyToReg nodes that copies the
00646     /// specified value into the registers specified by this object.  This uses
00647     /// Chain/Flag as the input and updates them for the output Chain/Flag.
00648     /// If the Flag pointer is NULL, no flag is used.
00649     void
00650     getCopyToRegs(SDValue Val, SelectionDAG &DAG, SDLoc dl, SDValue &Chain,
00651                   SDValue *Flag, const Value *V,
00652                   ISD::NodeType PreferredExtendType = ISD::ANY_EXTEND) const;
00653 
00654     /// AddInlineAsmOperands - Add this value to the specified inlineasm node
00655     /// operand list.  This adds the code marker, matching input operand index
00656     /// (if applicable), and includes the number of values added into it.
00657     void AddInlineAsmOperands(unsigned Kind,
00658                               bool HasMatching, unsigned MatchingIdx,
00659                               SelectionDAG &DAG,
00660                               std::vector<SDValue> &Ops) const;
00661   };
00662 }
00663 
00664 /// getCopyFromRegs - Emit a series of CopyFromReg nodes that copies from
00665 /// this value and returns the result as a ValueVT value.  This uses
00666 /// Chain/Flag as the input and updates them for the output Chain/Flag.
00667 /// If the Flag pointer is NULL, no flag is used.
00668 SDValue RegsForValue::getCopyFromRegs(SelectionDAG &DAG,
00669                                       FunctionLoweringInfo &FuncInfo,
00670                                       SDLoc dl,
00671                                       SDValue &Chain, SDValue *Flag,
00672                                       const Value *V) const {
00673   // A Value with type {} or [0 x %t] needs no registers.
00674   if (ValueVTs.empty())
00675     return SDValue();
00676 
00677   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
00678 
00679   // Assemble the legal parts into the final values.
00680   SmallVector<SDValue, 4> Values(ValueVTs.size());
00681   SmallVector<SDValue, 8> Parts;
00682   for (unsigned Value = 0, Part = 0, e = ValueVTs.size(); Value != e; ++Value) {
00683     // Copy the legal parts from the registers.
00684     EVT ValueVT = ValueVTs[Value];
00685     unsigned NumRegs = TLI.getNumRegisters(*DAG.getContext(), ValueVT);
00686     MVT RegisterVT = RegVTs[Value];
00687 
00688     Parts.resize(NumRegs);
00689     for (unsigned i = 0; i != NumRegs; ++i) {
00690       SDValue P;
00691       if (!Flag) {
00692         P = DAG.getCopyFromReg(Chain, dl, Regs[Part+i], RegisterVT);
00693       } else {
00694         P = DAG.getCopyFromReg(Chain, dl, Regs[Part+i], RegisterVT, *Flag);
00695         *Flag = P.getValue(2);
00696       }
00697 
00698       Chain = P.getValue(1);
00699       Parts[i] = P;
00700 
00701       // If the source register was virtual and if we know something about it,
00702       // add an assert node.
00703       if (!TargetRegisterInfo::isVirtualRegister(Regs[Part+i]) ||
00704           !RegisterVT.isInteger() || RegisterVT.isVector())
00705         continue;
00706 
00707       const FunctionLoweringInfo::LiveOutInfo *LOI =
00708         FuncInfo.GetLiveOutRegInfo(Regs[Part+i]);
00709       if (!LOI)
00710         continue;
00711 
00712       unsigned RegSize = RegisterVT.getSizeInBits();
00713       unsigned NumSignBits = LOI->NumSignBits;
00714       unsigned NumZeroBits = LOI->KnownZero.countLeadingOnes();
00715 
00716       if (NumZeroBits == RegSize) {
00717         // The current value is a zero.
00718         // Explicitly express that as it would be easier for
00719         // optimizations to kick in.
00720         Parts[i] = DAG.getConstant(0, RegisterVT);
00721         continue;
00722       }
00723 
00724       // FIXME: We capture more information than the dag can represent.  For
00725       // now, just use the tightest assertzext/assertsext possible.
00726       bool isSExt = true;
00727       EVT FromVT(MVT::Other);
00728       if (NumSignBits == RegSize)
00729         isSExt = true, FromVT = MVT::i1;   // ASSERT SEXT 1
00730       else if (NumZeroBits >= RegSize-1)
00731         isSExt = false, FromVT = MVT::i1;  // ASSERT ZEXT 1
00732       else if (NumSignBits > RegSize-8)
00733         isSExt = true, FromVT = MVT::i8;   // ASSERT SEXT 8
00734       else if (NumZeroBits >= RegSize-8)
00735         isSExt = false, FromVT = MVT::i8;  // ASSERT ZEXT 8
00736       else if (NumSignBits > RegSize-16)
00737         isSExt = true, FromVT = MVT::i16;  // ASSERT SEXT 16
00738       else if (NumZeroBits >= RegSize-16)
00739         isSExt = false, FromVT = MVT::i16; // ASSERT ZEXT 16
00740       else if (NumSignBits > RegSize-32)
00741         isSExt = true, FromVT = MVT::i32;  // ASSERT SEXT 32
00742       else if (NumZeroBits >= RegSize-32)
00743         isSExt = false, FromVT = MVT::i32; // ASSERT ZEXT 32
00744       else
00745         continue;
00746 
00747       // Add an assertion node.
00748       assert(FromVT != MVT::Other);
00749       Parts[i] = DAG.getNode(isSExt ? ISD::AssertSext : ISD::AssertZext, dl,
00750                              RegisterVT, P, DAG.getValueType(FromVT));
00751     }
00752 
00753     Values[Value] = getCopyFromParts(DAG, dl, Parts.begin(),
00754                                      NumRegs, RegisterVT, ValueVT, V);
00755     Part += NumRegs;
00756     Parts.clear();
00757   }
00758 
00759   return DAG.getNode(ISD::MERGE_VALUES, dl, DAG.getVTList(ValueVTs), Values);
00760 }
00761 
00762 /// getCopyToRegs - Emit a series of CopyToReg nodes that copies the
00763 /// specified value into the registers specified by this object.  This uses
00764 /// Chain/Flag as the input and updates them for the output Chain/Flag.
00765 /// If the Flag pointer is NULL, no flag is used.
00766 void RegsForValue::getCopyToRegs(SDValue Val, SelectionDAG &DAG, SDLoc dl,
00767                                  SDValue &Chain, SDValue *Flag, const Value *V,
00768                                  ISD::NodeType PreferredExtendType) const {
00769   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
00770   ISD::NodeType ExtendKind = PreferredExtendType;
00771 
00772   // Get the list of the values's legal parts.
00773   unsigned NumRegs = Regs.size();
00774   SmallVector<SDValue, 8> Parts(NumRegs);
00775   for (unsigned Value = 0, Part = 0, e = ValueVTs.size(); Value != e; ++Value) {
00776     EVT ValueVT = ValueVTs[Value];
00777     unsigned NumParts = TLI.getNumRegisters(*DAG.getContext(), ValueVT);
00778     MVT RegisterVT = RegVTs[Value];
00779 
00780     if (ExtendKind == ISD::ANY_EXTEND && TLI.isZExtFree(Val, RegisterVT))
00781       ExtendKind = ISD::ZERO_EXTEND;
00782 
00783     getCopyToParts(DAG, dl, Val.getValue(Val.getResNo() + Value),
00784                    &Parts[Part], NumParts, RegisterVT, V, ExtendKind);
00785     Part += NumParts;
00786   }
00787 
00788   // Copy the parts into the registers.
00789   SmallVector<SDValue, 8> Chains(NumRegs);
00790   for (unsigned i = 0; i != NumRegs; ++i) {
00791     SDValue Part;
00792     if (!Flag) {
00793       Part = DAG.getCopyToReg(Chain, dl, Regs[i], Parts[i]);
00794     } else {
00795       Part = DAG.getCopyToReg(Chain, dl, Regs[i], Parts[i], *Flag);
00796       *Flag = Part.getValue(1);
00797     }
00798 
00799     Chains[i] = Part.getValue(0);
00800   }
00801 
00802   if (NumRegs == 1 || Flag)
00803     // If NumRegs > 1 && Flag is used then the use of the last CopyToReg is
00804     // flagged to it. That is the CopyToReg nodes and the user are considered
00805     // a single scheduling unit. If we create a TokenFactor and return it as
00806     // chain, then the TokenFactor is both a predecessor (operand) of the
00807     // user as well as a successor (the TF operands are flagged to the user).
00808     // c1, f1 = CopyToReg
00809     // c2, f2 = CopyToReg
00810     // c3     = TokenFactor c1, c2
00811     // ...
00812     //        = op c3, ..., f2
00813     Chain = Chains[NumRegs-1];
00814   else
00815     Chain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Chains);
00816 }
00817 
00818 /// AddInlineAsmOperands - Add this value to the specified inlineasm node
00819 /// operand list.  This adds the code marker and includes the number of
00820 /// values added into it.
00821 void RegsForValue::AddInlineAsmOperands(unsigned Code, bool HasMatching,
00822                                         unsigned MatchingIdx,
00823                                         SelectionDAG &DAG,
00824                                         std::vector<SDValue> &Ops) const {
00825   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
00826 
00827   unsigned Flag = InlineAsm::getFlagWord(Code, Regs.size());
00828   if (HasMatching)
00829     Flag = InlineAsm::getFlagWordForMatchingOp(Flag, MatchingIdx);
00830   else if (!Regs.empty() &&
00831            TargetRegisterInfo::isVirtualRegister(Regs.front())) {
00832     // Put the register class of the virtual registers in the flag word.  That
00833     // way, later passes can recompute register class constraints for inline
00834     // assembly as well as normal instructions.
00835     // Don't do this for tied operands that can use the regclass information
00836     // from the def.
00837     const MachineRegisterInfo &MRI = DAG.getMachineFunction().getRegInfo();
00838     const TargetRegisterClass *RC = MRI.getRegClass(Regs.front());
00839     Flag = InlineAsm::getFlagWordForRegClass(Flag, RC->getID());
00840   }
00841 
00842   SDValue Res = DAG.getTargetConstant(Flag, MVT::i32);
00843   Ops.push_back(Res);
00844 
00845   unsigned SP = TLI.getStackPointerRegisterToSaveRestore();
00846   for (unsigned Value = 0, Reg = 0, e = ValueVTs.size(); Value != e; ++Value) {
00847     unsigned NumRegs = TLI.getNumRegisters(*DAG.getContext(), ValueVTs[Value]);
00848     MVT RegisterVT = RegVTs[Value];
00849     for (unsigned i = 0; i != NumRegs; ++i) {
00850       assert(Reg < Regs.size() && "Mismatch in # registers expected");
00851       unsigned TheReg = Regs[Reg++];
00852       Ops.push_back(DAG.getRegister(TheReg, RegisterVT));
00853 
00854       if (TheReg == SP && Code == InlineAsm::Kind_Clobber) {
00855         // If we clobbered the stack pointer, MFI should know about it.
00856         assert(DAG.getMachineFunction().getFrameInfo()->
00857             hasInlineAsmWithSPAdjust());
00858       }
00859     }
00860   }
00861 }
00862 
00863 void SelectionDAGBuilder::init(GCFunctionInfo *gfi, AliasAnalysis &aa,
00864                                const TargetLibraryInfo *li) {
00865   AA = &aa;
00866   GFI = gfi;
00867   LibInfo = li;
00868   DL = DAG.getSubtarget().getDataLayout();
00869   Context = DAG.getContext();
00870   LPadToCallSiteMap.clear();
00871 }
00872 
00873 /// clear - Clear out the current SelectionDAG and the associated
00874 /// state and prepare this SelectionDAGBuilder object to be used
00875 /// for a new block. This doesn't clear out information about
00876 /// additional blocks that are needed to complete switch lowering
00877 /// or PHI node updating; that information is cleared out as it is
00878 /// consumed.
00879 void SelectionDAGBuilder::clear() {
00880   NodeMap.clear();
00881   UnusedArgNodeMap.clear();
00882   PendingLoads.clear();
00883   PendingExports.clear();
00884   CurInst = nullptr;
00885   HasTailCall = false;
00886   SDNodeOrder = LowestSDNodeOrder;
00887 }
00888 
00889 /// clearDanglingDebugInfo - Clear the dangling debug information
00890 /// map. This function is separated from the clear so that debug
00891 /// information that is dangling in a basic block can be properly
00892 /// resolved in a different basic block. This allows the
00893 /// SelectionDAG to resolve dangling debug information attached
00894 /// to PHI nodes.
00895 void SelectionDAGBuilder::clearDanglingDebugInfo() {
00896   DanglingDebugInfoMap.clear();
00897 }
00898 
00899 /// getRoot - Return the current virtual root of the Selection DAG,
00900 /// flushing any PendingLoad items. This must be done before emitting
00901 /// a store or any other node that may need to be ordered after any
00902 /// prior load instructions.
00903 ///
00904 SDValue SelectionDAGBuilder::getRoot() {
00905   if (PendingLoads.empty())
00906     return DAG.getRoot();
00907 
00908   if (PendingLoads.size() == 1) {
00909     SDValue Root = PendingLoads[0];
00910     DAG.setRoot(Root);
00911     PendingLoads.clear();
00912     return Root;
00913   }
00914 
00915   // Otherwise, we have to make a token factor node.
00916   SDValue Root = DAG.getNode(ISD::TokenFactor, getCurSDLoc(), MVT::Other,
00917                              PendingLoads);
00918   PendingLoads.clear();
00919   DAG.setRoot(Root);
00920   return Root;
00921 }
00922 
00923 /// getControlRoot - Similar to getRoot, but instead of flushing all the
00924 /// PendingLoad items, flush all the PendingExports items. It is necessary
00925 /// to do this before emitting a terminator instruction.
00926 ///
00927 SDValue SelectionDAGBuilder::getControlRoot() {
00928   SDValue Root = DAG.getRoot();
00929 
00930   if (PendingExports.empty())
00931     return Root;
00932 
00933   // Turn all of the CopyToReg chains into one factored node.
00934   if (Root.getOpcode() != ISD::EntryToken) {
00935     unsigned i = 0, e = PendingExports.size();
00936     for (; i != e; ++i) {
00937       assert(PendingExports[i].getNode()->getNumOperands() > 1);
00938       if (PendingExports[i].getNode()->getOperand(0) == Root)
00939         break;  // Don't add the root if we already indirectly depend on it.
00940     }
00941 
00942     if (i == e)
00943       PendingExports.push_back(Root);
00944   }
00945 
00946   Root = DAG.getNode(ISD::TokenFactor, getCurSDLoc(), MVT::Other,
00947                      PendingExports);
00948   PendingExports.clear();
00949   DAG.setRoot(Root);
00950   return Root;
00951 }
00952 
00953 void SelectionDAGBuilder::visit(const Instruction &I) {
00954   // Set up outgoing PHI node register values before emitting the terminator.
00955   if (isa<TerminatorInst>(&I))
00956     HandlePHINodesInSuccessorBlocks(I.getParent());
00957 
00958   ++SDNodeOrder;
00959 
00960   CurInst = &I;
00961 
00962   visit(I.getOpcode(), I);
00963 
00964   if (!isa<TerminatorInst>(&I) && !HasTailCall)
00965     CopyToExportRegsIfNeeded(&I);
00966 
00967   CurInst = nullptr;
00968 }
00969 
00970 void SelectionDAGBuilder::visitPHI(const PHINode &) {
00971   llvm_unreachable("SelectionDAGBuilder shouldn't visit PHI nodes!");
00972 }
00973 
00974 void SelectionDAGBuilder::visit(unsigned Opcode, const User &I) {
00975   // Note: this doesn't use InstVisitor, because it has to work with
00976   // ConstantExpr's in addition to instructions.
00977   switch (Opcode) {
00978   default: llvm_unreachable("Unknown instruction type encountered!");
00979     // Build the switch statement using the Instruction.def file.
00980 #define HANDLE_INST(NUM, OPCODE, CLASS) \
00981     case Instruction::OPCODE: visit##OPCODE((const CLASS&)I); break;
00982 #include "llvm/IR/Instruction.def"
00983   }
00984 }
00985 
00986 // resolveDanglingDebugInfo - if we saw an earlier dbg_value referring to V,
00987 // generate the debug data structures now that we've seen its definition.
00988 void SelectionDAGBuilder::resolveDanglingDebugInfo(const Value *V,
00989                                                    SDValue Val) {
00990   DanglingDebugInfo &DDI = DanglingDebugInfoMap[V];
00991   if (DDI.getDI()) {
00992     const DbgValueInst *DI = DDI.getDI();
00993     DebugLoc dl = DDI.getdl();
00994     unsigned DbgSDNodeOrder = DDI.getSDNodeOrder();
00995     MDNode *Variable = DI->getVariable();
00996     MDNode *Expr = DI->getExpression();
00997     uint64_t Offset = DI->getOffset();
00998     // A dbg.value for an alloca is always indirect.
00999     bool IsIndirect = isa<AllocaInst>(V) || Offset != 0;
01000     SDDbgValue *SDV;
01001     if (Val.getNode()) {
01002       if (!EmitFuncArgumentDbgValue(V, Variable, Expr, Offset, IsIndirect,
01003                                     Val)) {
01004         SDV = DAG.getDbgValue(Variable, Expr, Val.getNode(), Val.getResNo(),
01005                               IsIndirect, Offset, dl, DbgSDNodeOrder);
01006         DAG.AddDbgValue(SDV, Val.getNode(), false);
01007       }
01008     } else
01009       DEBUG(dbgs() << "Dropping debug info for " << *DI << "\n");
01010     DanglingDebugInfoMap[V] = DanglingDebugInfo();
01011   }
01012 }
01013 
01014 /// getValue - Return an SDValue for the given Value.
01015 SDValue SelectionDAGBuilder::getValue(const Value *V) {
01016   // If we already have an SDValue for this value, use it. It's important
01017   // to do this first, so that we don't create a CopyFromReg if we already
01018   // have a regular SDValue.
01019   SDValue &N = NodeMap[V];
01020   if (N.getNode()) return N;
01021 
01022   // If there's a virtual register allocated and initialized for this
01023   // value, use it.
01024   DenseMap<const Value *, unsigned>::iterator It = FuncInfo.ValueMap.find(V);
01025   if (It != FuncInfo.ValueMap.end()) {
01026     unsigned InReg = It->second;
01027     RegsForValue RFV(*DAG.getContext(), DAG.getTargetLoweringInfo(), InReg,
01028                      V->getType());
01029     SDValue Chain = DAG.getEntryNode();
01030     N = RFV.getCopyFromRegs(DAG, FuncInfo, getCurSDLoc(), Chain, nullptr, V);
01031     resolveDanglingDebugInfo(V, N);
01032     return N;
01033   }
01034 
01035   // Otherwise create a new SDValue and remember it.
01036   SDValue Val = getValueImpl(V);
01037   NodeMap[V] = Val;
01038   resolveDanglingDebugInfo(V, Val);
01039   return Val;
01040 }
01041 
01042 /// getNonRegisterValue - Return an SDValue for the given Value, but
01043 /// don't look in FuncInfo.ValueMap for a virtual register.
01044 SDValue SelectionDAGBuilder::getNonRegisterValue(const Value *V) {
01045   // If we already have an SDValue for this value, use it.
01046   SDValue &N = NodeMap[V];
01047   if (N.getNode()) return N;
01048 
01049   // Otherwise create a new SDValue and remember it.
01050   SDValue Val = getValueImpl(V);
01051   NodeMap[V] = Val;
01052   resolveDanglingDebugInfo(V, Val);
01053   return Val;
01054 }
01055 
01056 /// getValueImpl - Helper function for getValue and getNonRegisterValue.
01057 /// Create an SDValue for the given value.
01058 SDValue SelectionDAGBuilder::getValueImpl(const Value *V) {
01059   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
01060 
01061   if (const Constant *C = dyn_cast<Constant>(V)) {
01062     EVT VT = TLI.getValueType(V->getType(), true);
01063 
01064     if (const ConstantInt *CI = dyn_cast<ConstantInt>(C))
01065       return DAG.getConstant(*CI, VT);
01066 
01067     if (const GlobalValue *GV = dyn_cast<GlobalValue>(C))
01068       return DAG.getGlobalAddress(GV, getCurSDLoc(), VT);
01069 
01070     if (isa<ConstantPointerNull>(C)) {
01071       unsigned AS = V->getType()->getPointerAddressSpace();
01072       return DAG.getConstant(0, TLI.getPointerTy(AS));
01073     }
01074 
01075     if (const ConstantFP *CFP = dyn_cast<ConstantFP>(C))
01076       return DAG.getConstantFP(*CFP, VT);
01077 
01078     if (isa<UndefValue>(C) && !V->getType()->isAggregateType())
01079       return DAG.getUNDEF(VT);
01080 
01081     if (const ConstantExpr *CE = dyn_cast<ConstantExpr>(C)) {
01082       visit(CE->getOpcode(), *CE);
01083       SDValue N1 = NodeMap[V];
01084       assert(N1.getNode() && "visit didn't populate the NodeMap!");
01085       return N1;
01086     }
01087 
01088     if (isa<ConstantStruct>(C) || isa<ConstantArray>(C)) {
01089       SmallVector<SDValue, 4> Constants;
01090       for (User::const_op_iterator OI = C->op_begin(), OE = C->op_end();
01091            OI != OE; ++OI) {
01092         SDNode *Val = getValue(*OI).getNode();
01093         // If the operand is an empty aggregate, there are no values.
01094         if (!Val) continue;
01095         // Add each leaf value from the operand to the Constants list
01096         // to form a flattened list of all the values.
01097         for (unsigned i = 0, e = Val->getNumValues(); i != e; ++i)
01098           Constants.push_back(SDValue(Val, i));
01099       }
01100 
01101       return DAG.getMergeValues(Constants, getCurSDLoc());
01102     }
01103 
01104     if (const ConstantDataSequential *CDS =
01105           dyn_cast<ConstantDataSequential>(C)) {
01106       SmallVector<SDValue, 4> Ops;
01107       for (unsigned i = 0, e = CDS->getNumElements(); i != e; ++i) {
01108         SDNode *Val = getValue(CDS->getElementAsConstant(i)).getNode();
01109         // Add each leaf value from the operand to the Constants list
01110         // to form a flattened list of all the values.
01111         for (unsigned i = 0, e = Val->getNumValues(); i != e; ++i)
01112           Ops.push_back(SDValue(Val, i));
01113       }
01114 
01115       if (isa<ArrayType>(CDS->getType()))
01116         return DAG.getMergeValues(Ops, getCurSDLoc());
01117       return NodeMap[V] = DAG.getNode(ISD::BUILD_VECTOR, getCurSDLoc(),
01118                                       VT, Ops);
01119     }
01120 
01121     if (C->getType()->isStructTy() || C->getType()->isArrayTy()) {
01122       assert((isa<ConstantAggregateZero>(C) || isa<UndefValue>(C)) &&
01123              "Unknown struct or array constant!");
01124 
01125       SmallVector<EVT, 4> ValueVTs;
01126       ComputeValueVTs(TLI, C->getType(), ValueVTs);
01127       unsigned NumElts = ValueVTs.size();
01128       if (NumElts == 0)
01129         return SDValue(); // empty struct
01130       SmallVector<SDValue, 4> Constants(NumElts);
01131       for (unsigned i = 0; i != NumElts; ++i) {
01132         EVT EltVT = ValueVTs[i];
01133         if (isa<UndefValue>(C))
01134           Constants[i] = DAG.getUNDEF(EltVT);
01135         else if (EltVT.isFloatingPoint())
01136           Constants[i] = DAG.getConstantFP(0, EltVT);
01137         else
01138           Constants[i] = DAG.getConstant(0, EltVT);
01139       }
01140 
01141       return DAG.getMergeValues(Constants, getCurSDLoc());
01142     }
01143 
01144     if (const BlockAddress *BA = dyn_cast<BlockAddress>(C))
01145       return DAG.getBlockAddress(BA, VT);
01146 
01147     VectorType *VecTy = cast<VectorType>(V->getType());
01148     unsigned NumElements = VecTy->getNumElements();
01149 
01150     // Now that we know the number and type of the elements, get that number of
01151     // elements into the Ops array based on what kind of constant it is.
01152     SmallVector<SDValue, 16> Ops;
01153     if (const ConstantVector *CV = dyn_cast<ConstantVector>(C)) {
01154       for (unsigned i = 0; i != NumElements; ++i)
01155         Ops.push_back(getValue(CV->getOperand(i)));
01156     } else {
01157       assert(isa<ConstantAggregateZero>(C) && "Unknown vector constant!");
01158       EVT EltVT = TLI.getValueType(VecTy->getElementType());
01159 
01160       SDValue Op;
01161       if (EltVT.isFloatingPoint())
01162         Op = DAG.getConstantFP(0, EltVT);
01163       else
01164         Op = DAG.getConstant(0, EltVT);
01165       Ops.assign(NumElements, Op);
01166     }
01167 
01168     // Create a BUILD_VECTOR node.
01169     return NodeMap[V] = DAG.getNode(ISD::BUILD_VECTOR, getCurSDLoc(), VT, Ops);
01170   }
01171 
01172   // If this is a static alloca, generate it as the frameindex instead of
01173   // computation.
01174   if (const AllocaInst *AI = dyn_cast<AllocaInst>(V)) {
01175     DenseMap<const AllocaInst*, int>::iterator SI =
01176       FuncInfo.StaticAllocaMap.find(AI);
01177     if (SI != FuncInfo.StaticAllocaMap.end())
01178       return DAG.getFrameIndex(SI->second, TLI.getPointerTy());
01179   }
01180 
01181   // If this is an instruction which fast-isel has deferred, select it now.
01182   if (const Instruction *Inst = dyn_cast<Instruction>(V)) {
01183     unsigned InReg = FuncInfo.InitializeRegForValue(Inst);
01184     RegsForValue RFV(*DAG.getContext(), TLI, InReg, Inst->getType());
01185     SDValue Chain = DAG.getEntryNode();
01186     return RFV.getCopyFromRegs(DAG, FuncInfo, getCurSDLoc(), Chain, nullptr, V);
01187   }
01188 
01189   llvm_unreachable("Can't get register for value!");
01190 }
01191 
01192 void SelectionDAGBuilder::visitRet(const ReturnInst &I) {
01193   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
01194   SDValue Chain = getControlRoot();
01195   SmallVector<ISD::OutputArg, 8> Outs;
01196   SmallVector<SDValue, 8> OutVals;
01197 
01198   if (!FuncInfo.CanLowerReturn) {
01199     unsigned DemoteReg = FuncInfo.DemoteRegister;
01200     const Function *F = I.getParent()->getParent();
01201 
01202     // Emit a store of the return value through the virtual register.
01203     // Leave Outs empty so that LowerReturn won't try to load return
01204     // registers the usual way.
01205     SmallVector<EVT, 1> PtrValueVTs;
01206     ComputeValueVTs(TLI, PointerType::getUnqual(F->getReturnType()),
01207                     PtrValueVTs);
01208 
01209     SDValue RetPtr = DAG.getRegister(DemoteReg, PtrValueVTs[0]);
01210     SDValue RetOp = getValue(I.getOperand(0));
01211 
01212     SmallVector<EVT, 4> ValueVTs;
01213     SmallVector<uint64_t, 4> Offsets;
01214     ComputeValueVTs(TLI, I.getOperand(0)->getType(), ValueVTs, &Offsets);
01215     unsigned NumValues = ValueVTs.size();
01216 
01217     SmallVector<SDValue, 4> Chains(NumValues);
01218     for (unsigned i = 0; i != NumValues; ++i) {
01219       SDValue Add = DAG.getNode(ISD::ADD, getCurSDLoc(),
01220                                 RetPtr.getValueType(), RetPtr,
01221                                 DAG.getIntPtrConstant(Offsets[i]));
01222       Chains[i] =
01223         DAG.getStore(Chain, getCurSDLoc(),
01224                      SDValue(RetOp.getNode(), RetOp.getResNo() + i),
01225                      // FIXME: better loc info would be nice.
01226                      Add, MachinePointerInfo(), false, false, 0);
01227     }
01228 
01229     Chain = DAG.getNode(ISD::TokenFactor, getCurSDLoc(),
01230                         MVT::Other, Chains);
01231   } else if (I.getNumOperands() != 0) {
01232     SmallVector<EVT, 4> ValueVTs;
01233     ComputeValueVTs(TLI, I.getOperand(0)->getType(), ValueVTs);
01234     unsigned NumValues = ValueVTs.size();
01235     if (NumValues) {
01236       SDValue RetOp = getValue(I.getOperand(0));
01237       for (unsigned j = 0, f = NumValues; j != f; ++j) {
01238         EVT VT = ValueVTs[j];
01239 
01240         ISD::NodeType ExtendKind = ISD::ANY_EXTEND;
01241 
01242         const Function *F = I.getParent()->getParent();
01243         if (F->getAttributes().hasAttribute(AttributeSet::ReturnIndex,
01244                                             Attribute::SExt))
01245           ExtendKind = ISD::SIGN_EXTEND;
01246         else if (F->getAttributes().hasAttribute(AttributeSet::ReturnIndex,
01247                                                  Attribute::ZExt))
01248           ExtendKind = ISD::ZERO_EXTEND;
01249 
01250         if (ExtendKind != ISD::ANY_EXTEND && VT.isInteger())
01251           VT = TLI.getTypeForExtArgOrReturn(*DAG.getContext(), VT, ExtendKind);
01252 
01253         unsigned NumParts = TLI.getNumRegisters(*DAG.getContext(), VT);
01254         MVT PartVT = TLI.getRegisterType(*DAG.getContext(), VT);
01255         SmallVector<SDValue, 4> Parts(NumParts);
01256         getCopyToParts(DAG, getCurSDLoc(),
01257                        SDValue(RetOp.getNode(), RetOp.getResNo() + j),
01258                        &Parts[0], NumParts, PartVT, &I, ExtendKind);
01259 
01260         // 'inreg' on function refers to return value
01261         ISD::ArgFlagsTy Flags = ISD::ArgFlagsTy();
01262         if (F->getAttributes().hasAttribute(AttributeSet::ReturnIndex,
01263                                             Attribute::InReg))
01264           Flags.setInReg();
01265 
01266         // Propagate extension type if any
01267         if (ExtendKind == ISD::SIGN_EXTEND)
01268           Flags.setSExt();
01269         else if (ExtendKind == ISD::ZERO_EXTEND)
01270           Flags.setZExt();
01271 
01272         for (unsigned i = 0; i < NumParts; ++i) {
01273           Outs.push_back(ISD::OutputArg(Flags, Parts[i].getValueType(),
01274                                         VT, /*isfixed=*/true, 0, 0));
01275           OutVals.push_back(Parts[i]);
01276         }
01277       }
01278     }
01279   }
01280 
01281   bool isVarArg = DAG.getMachineFunction().getFunction()->isVarArg();
01282   CallingConv::ID CallConv =
01283     DAG.getMachineFunction().getFunction()->getCallingConv();
01284   Chain = DAG.getTargetLoweringInfo().LowerReturn(
01285       Chain, CallConv, isVarArg, Outs, OutVals, getCurSDLoc(), DAG);
01286 
01287   // Verify that the target's LowerReturn behaved as expected.
01288   assert(Chain.getNode() && Chain.getValueType() == MVT::Other &&
01289          "LowerReturn didn't return a valid chain!");
01290 
01291   // Update the DAG with the new chain value resulting from return lowering.
01292   DAG.setRoot(Chain);
01293 }
01294 
01295 /// CopyToExportRegsIfNeeded - If the given value has virtual registers
01296 /// created for it, emit nodes to copy the value into the virtual
01297 /// registers.
01298 void SelectionDAGBuilder::CopyToExportRegsIfNeeded(const Value *V) {
01299   // Skip empty types
01300   if (V->getType()->isEmptyTy())
01301     return;
01302 
01303   DenseMap<const Value *, unsigned>::iterator VMI = FuncInfo.ValueMap.find(V);
01304   if (VMI != FuncInfo.ValueMap.end()) {
01305     assert(!V->use_empty() && "Unused value assigned virtual registers!");
01306     CopyValueToVirtualRegister(V, VMI->second);
01307   }
01308 }
01309 
01310 /// ExportFromCurrentBlock - If this condition isn't known to be exported from
01311 /// the current basic block, add it to ValueMap now so that we'll get a
01312 /// CopyTo/FromReg.
01313 void SelectionDAGBuilder::ExportFromCurrentBlock(const Value *V) {
01314   // No need to export constants.
01315   if (!isa<Instruction>(V) && !isa<Argument>(V)) return;
01316 
01317   // Already exported?
01318   if (FuncInfo.isExportedInst(V)) return;
01319 
01320   unsigned Reg = FuncInfo.InitializeRegForValue(V);
01321   CopyValueToVirtualRegister(V, Reg);
01322 }
01323 
01324 bool SelectionDAGBuilder::isExportableFromCurrentBlock(const Value *V,
01325                                                      const BasicBlock *FromBB) {
01326   // The operands of the setcc have to be in this block.  We don't know
01327   // how to export them from some other block.
01328   if (const Instruction *VI = dyn_cast<Instruction>(V)) {
01329     // Can export from current BB.
01330     if (VI->getParent() == FromBB)
01331       return true;
01332 
01333     // Is already exported, noop.
01334     return FuncInfo.isExportedInst(V);
01335   }
01336 
01337   // If this is an argument, we can export it if the BB is the entry block or
01338   // if it is already exported.
01339   if (isa<Argument>(V)) {
01340     if (FromBB == &FromBB->getParent()->getEntryBlock())
01341       return true;
01342 
01343     // Otherwise, can only export this if it is already exported.
01344     return FuncInfo.isExportedInst(V);
01345   }
01346 
01347   // Otherwise, constants can always be exported.
01348   return true;
01349 }
01350 
01351 /// Return branch probability calculated by BranchProbabilityInfo for IR blocks.
01352 uint32_t SelectionDAGBuilder::getEdgeWeight(const MachineBasicBlock *Src,
01353                                             const MachineBasicBlock *Dst) const {
01354   BranchProbabilityInfo *BPI = FuncInfo.BPI;
01355   if (!BPI)
01356     return 0;
01357   const BasicBlock *SrcBB = Src->getBasicBlock();
01358   const BasicBlock *DstBB = Dst->getBasicBlock();
01359   return BPI->getEdgeWeight(SrcBB, DstBB);
01360 }
01361 
01362 void SelectionDAGBuilder::
01363 addSuccessorWithWeight(MachineBasicBlock *Src, MachineBasicBlock *Dst,
01364                        uint32_t Weight /* = 0 */) {
01365   if (!Weight)
01366     Weight = getEdgeWeight(Src, Dst);
01367   Src->addSuccessor(Dst, Weight);
01368 }
01369 
01370 
01371 static bool InBlock(const Value *V, const BasicBlock *BB) {
01372   if (const Instruction *I = dyn_cast<Instruction>(V))
01373     return I->getParent() == BB;
01374   return true;
01375 }
01376 
01377 /// EmitBranchForMergedCondition - Helper method for FindMergedConditions.
01378 /// This function emits a branch and is used at the leaves of an OR or an
01379 /// AND operator tree.
01380 ///
01381 void
01382 SelectionDAGBuilder::EmitBranchForMergedCondition(const Value *Cond,
01383                                                   MachineBasicBlock *TBB,
01384                                                   MachineBasicBlock *FBB,
01385                                                   MachineBasicBlock *CurBB,
01386                                                   MachineBasicBlock *SwitchBB,
01387                                                   uint32_t TWeight,
01388                                                   uint32_t FWeight) {
01389   const BasicBlock *BB = CurBB->getBasicBlock();
01390 
01391   // If the leaf of the tree is a comparison, merge the condition into
01392   // the caseblock.
01393   if (const CmpInst *BOp = dyn_cast<CmpInst>(Cond)) {
01394     // The operands of the cmp have to be in this block.  We don't know
01395     // how to export them from some other block.  If this is the first block
01396     // of the sequence, no exporting is needed.
01397     if (CurBB == SwitchBB ||
01398         (isExportableFromCurrentBlock(BOp->getOperand(0), BB) &&
01399          isExportableFromCurrentBlock(BOp->getOperand(1), BB))) {
01400       ISD::CondCode Condition;
01401       if (const ICmpInst *IC = dyn_cast<ICmpInst>(Cond)) {
01402         Condition = getICmpCondCode(IC->getPredicate());
01403       } else if (const FCmpInst *FC = dyn_cast<FCmpInst>(Cond)) {
01404         Condition = getFCmpCondCode(FC->getPredicate());
01405         if (TM.Options.NoNaNsFPMath)
01406           Condition = getFCmpCodeWithoutNaN(Condition);
01407       } else {
01408         Condition = ISD::SETEQ; // silence warning.
01409         llvm_unreachable("Unknown compare instruction");
01410       }
01411 
01412       CaseBlock CB(Condition, BOp->getOperand(0), BOp->getOperand(1), nullptr,
01413                    TBB, FBB, CurBB, TWeight, FWeight);
01414       SwitchCases.push_back(CB);
01415       return;
01416     }
01417   }
01418 
01419   // Create a CaseBlock record representing this branch.
01420   CaseBlock CB(ISD::SETEQ, Cond, ConstantInt::getTrue(*DAG.getContext()),
01421                nullptr, TBB, FBB, CurBB, TWeight, FWeight);
01422   SwitchCases.push_back(CB);
01423 }
01424 
01425 /// Scale down both weights to fit into uint32_t.
01426 static void ScaleWeights(uint64_t &NewTrue, uint64_t &NewFalse) {
01427   uint64_t NewMax = (NewTrue > NewFalse) ? NewTrue : NewFalse;
01428   uint32_t Scale = (NewMax / UINT32_MAX) + 1;
01429   NewTrue = NewTrue / Scale;
01430   NewFalse = NewFalse / Scale;
01431 }
01432 
01433 /// FindMergedConditions - If Cond is an expression like
01434 void SelectionDAGBuilder::FindMergedConditions(const Value *Cond,
01435                                                MachineBasicBlock *TBB,
01436                                                MachineBasicBlock *FBB,
01437                                                MachineBasicBlock *CurBB,
01438                                                MachineBasicBlock *SwitchBB,
01439                                                unsigned Opc, uint32_t TWeight,
01440                                                uint32_t FWeight) {
01441   // If this node is not part of the or/and tree, emit it as a branch.
01442   const Instruction *BOp = dyn_cast<Instruction>(Cond);
01443   if (!BOp || !(isa<BinaryOperator>(BOp) || isa<CmpInst>(BOp)) ||
01444       (unsigned)BOp->getOpcode() != Opc || !BOp->hasOneUse() ||
01445       BOp->getParent() != CurBB->getBasicBlock() ||
01446       !InBlock(BOp->getOperand(0), CurBB->getBasicBlock()) ||
01447       !InBlock(BOp->getOperand(1), CurBB->getBasicBlock())) {
01448     EmitBranchForMergedCondition(Cond, TBB, FBB, CurBB, SwitchBB,
01449                                  TWeight, FWeight);
01450     return;
01451   }
01452 
01453   //  Create TmpBB after CurBB.
01454   MachineFunction::iterator BBI = CurBB;
01455   MachineFunction &MF = DAG.getMachineFunction();
01456   MachineBasicBlock *TmpBB = MF.CreateMachineBasicBlock(CurBB->getBasicBlock());
01457   CurBB->getParent()->insert(++BBI, TmpBB);
01458 
01459   if (Opc == Instruction::Or) {
01460     // Codegen X | Y as:
01461     // BB1:
01462     //   jmp_if_X TBB
01463     //   jmp TmpBB
01464     // TmpBB:
01465     //   jmp_if_Y TBB
01466     //   jmp FBB
01467     //
01468 
01469     // We have flexibility in setting Prob for BB1 and Prob for TmpBB.
01470     // The requirement is that
01471     //   TrueProb for BB1 + (FalseProb for BB1 * TrueProb for TmpBB)
01472     //     = TrueProb for orignal BB.
01473     // Assuming the orignal weights are A and B, one choice is to set BB1's
01474     // weights to A and A+2B, and set TmpBB's weights to A and 2B. This choice
01475     // assumes that
01476     //   TrueProb for BB1 == FalseProb for BB1 * TrueProb for TmpBB.
01477     // Another choice is to assume TrueProb for BB1 equals to TrueProb for
01478     // TmpBB, but the math is more complicated.
01479 
01480     uint64_t NewTrueWeight = TWeight;
01481     uint64_t NewFalseWeight = (uint64_t)TWeight + 2 * (uint64_t)FWeight;
01482     ScaleWeights(NewTrueWeight, NewFalseWeight);
01483     // Emit the LHS condition.
01484     FindMergedConditions(BOp->getOperand(0), TBB, TmpBB, CurBB, SwitchBB, Opc,
01485                          NewTrueWeight, NewFalseWeight);
01486 
01487     NewTrueWeight = TWeight;
01488     NewFalseWeight = 2 * (uint64_t)FWeight;
01489     ScaleWeights(NewTrueWeight, NewFalseWeight);
01490     // Emit the RHS condition into TmpBB.
01491     FindMergedConditions(BOp->getOperand(1), TBB, FBB, TmpBB, SwitchBB, Opc,
01492                          NewTrueWeight, NewFalseWeight);
01493   } else {
01494     assert(Opc == Instruction::And && "Unknown merge op!");
01495     // Codegen X & Y as:
01496     // BB1:
01497     //   jmp_if_X TmpBB
01498     //   jmp FBB
01499     // TmpBB:
01500     //   jmp_if_Y TBB
01501     //   jmp FBB
01502     //
01503     //  This requires creation of TmpBB after CurBB.
01504 
01505     // We have flexibility in setting Prob for BB1 and Prob for TmpBB.
01506     // The requirement is that
01507     //   FalseProb for BB1 + (TrueProb for BB1 * FalseProb for TmpBB)
01508     //     = FalseProb for orignal BB.
01509     // Assuming the orignal weights are A and B, one choice is to set BB1's
01510     // weights to 2A+B and B, and set TmpBB's weights to 2A and B. This choice
01511     // assumes that
01512     //   FalseProb for BB1 == TrueProb for BB1 * FalseProb for TmpBB.
01513 
01514     uint64_t NewTrueWeight = 2 * (uint64_t)TWeight + (uint64_t)FWeight;
01515     uint64_t NewFalseWeight = FWeight;
01516     ScaleWeights(NewTrueWeight, NewFalseWeight);
01517     // Emit the LHS condition.
01518     FindMergedConditions(BOp->getOperand(0), TmpBB, FBB, CurBB, SwitchBB, Opc,
01519                          NewTrueWeight, NewFalseWeight);
01520 
01521     NewTrueWeight = 2 * (uint64_t)TWeight;
01522     NewFalseWeight = FWeight;
01523     ScaleWeights(NewTrueWeight, NewFalseWeight);
01524     // Emit the RHS condition into TmpBB.
01525     FindMergedConditions(BOp->getOperand(1), TBB, FBB, TmpBB, SwitchBB, Opc,
01526                          NewTrueWeight, NewFalseWeight);
01527   }
01528 }
01529 
01530 /// If the set of cases should be emitted as a series of branches, return true.
01531 /// If we should emit this as a bunch of and/or'd together conditions, return
01532 /// false.
01533 bool
01534 SelectionDAGBuilder::ShouldEmitAsBranches(const std::vector<CaseBlock> &Cases) {
01535   if (Cases.size() != 2) return true;
01536 
01537   // If this is two comparisons of the same values or'd or and'd together, they
01538   // will get folded into a single comparison, so don't emit two blocks.
01539   if ((Cases[0].CmpLHS == Cases[1].CmpLHS &&
01540        Cases[0].CmpRHS == Cases[1].CmpRHS) ||
01541       (Cases[0].CmpRHS == Cases[1].CmpLHS &&
01542        Cases[0].CmpLHS == Cases[1].CmpRHS)) {
01543     return false;
01544   }
01545 
01546   // Handle: (X != null) | (Y != null) --> (X|Y) != 0
01547   // Handle: (X == null) & (Y == null) --> (X|Y) == 0
01548   if (Cases[0].CmpRHS == Cases[1].CmpRHS &&
01549       Cases[0].CC == Cases[1].CC &&
01550       isa<Constant>(Cases[0].CmpRHS) &&
01551       cast<Constant>(Cases[0].CmpRHS)->isNullValue()) {
01552     if (Cases[0].CC == ISD::SETEQ && Cases[0].TrueBB == Cases[1].ThisBB)
01553       return false;
01554     if (Cases[0].CC == ISD::SETNE && Cases[0].FalseBB == Cases[1].ThisBB)
01555       return false;
01556   }
01557 
01558   return true;
01559 }
01560 
01561 void SelectionDAGBuilder::visitBr(const BranchInst &I) {
01562   MachineBasicBlock *BrMBB = FuncInfo.MBB;
01563 
01564   // Update machine-CFG edges.
01565   MachineBasicBlock *Succ0MBB = FuncInfo.MBBMap[I.getSuccessor(0)];
01566 
01567   // Figure out which block is immediately after the current one.
01568   MachineBasicBlock *NextBlock = nullptr;
01569   MachineFunction::iterator BBI = BrMBB;
01570   if (++BBI != FuncInfo.MF->end())
01571     NextBlock = BBI;
01572 
01573   if (I.isUnconditional()) {
01574     // Update machine-CFG edges.
01575     BrMBB->addSuccessor(Succ0MBB);
01576 
01577     // If this is not a fall-through branch or optimizations are switched off,
01578     // emit the branch.
01579     if (Succ0MBB != NextBlock || TM.getOptLevel() == CodeGenOpt::None)
01580       DAG.setRoot(DAG.getNode(ISD::BR, getCurSDLoc(),
01581                               MVT::Other, getControlRoot(),
01582                               DAG.getBasicBlock(Succ0MBB)));
01583 
01584     return;
01585   }
01586 
01587   // If this condition is one of the special cases we handle, do special stuff
01588   // now.
01589   const Value *CondVal = I.getCondition();
01590   MachineBasicBlock *Succ1MBB = FuncInfo.MBBMap[I.getSuccessor(1)];
01591 
01592   // If this is a series of conditions that are or'd or and'd together, emit
01593   // this as a sequence of branches instead of setcc's with and/or operations.
01594   // As long as jumps are not expensive, this should improve performance.
01595   // For example, instead of something like:
01596   //     cmp A, B
01597   //     C = seteq
01598   //     cmp D, E
01599   //     F = setle
01600   //     or C, F
01601   //     jnz foo
01602   // Emit:
01603   //     cmp A, B
01604   //     je foo
01605   //     cmp D, E
01606   //     jle foo
01607   //
01608   if (const BinaryOperator *BOp = dyn_cast<BinaryOperator>(CondVal)) {
01609     if (!DAG.getTargetLoweringInfo().isJumpExpensive() &&
01610         BOp->hasOneUse() && (BOp->getOpcode() == Instruction::And ||
01611                              BOp->getOpcode() == Instruction::Or)) {
01612       FindMergedConditions(BOp, Succ0MBB, Succ1MBB, BrMBB, BrMBB,
01613                            BOp->getOpcode(), getEdgeWeight(BrMBB, Succ0MBB),
01614                            getEdgeWeight(BrMBB, Succ1MBB));
01615       // If the compares in later blocks need to use values not currently
01616       // exported from this block, export them now.  This block should always
01617       // be the first entry.
01618       assert(SwitchCases[0].ThisBB == BrMBB && "Unexpected lowering!");
01619 
01620       // Allow some cases to be rejected.
01621       if (ShouldEmitAsBranches(SwitchCases)) {
01622         for (unsigned i = 1, e = SwitchCases.size(); i != e; ++i) {
01623           ExportFromCurrentBlock(SwitchCases[i].CmpLHS);
01624           ExportFromCurrentBlock(SwitchCases[i].CmpRHS);
01625         }
01626 
01627         // Emit the branch for this block.
01628         visitSwitchCase(SwitchCases[0], BrMBB);
01629         SwitchCases.erase(SwitchCases.begin());
01630         return;
01631       }
01632 
01633       // Okay, we decided not to do this, remove any inserted MBB's and clear
01634       // SwitchCases.
01635       for (unsigned i = 1, e = SwitchCases.size(); i != e; ++i)
01636         FuncInfo.MF->erase(SwitchCases[i].ThisBB);
01637 
01638       SwitchCases.clear();
01639     }
01640   }
01641 
01642   // Create a CaseBlock record representing this branch.
01643   CaseBlock CB(ISD::SETEQ, CondVal, ConstantInt::getTrue(*DAG.getContext()),
01644                nullptr, Succ0MBB, Succ1MBB, BrMBB);
01645 
01646   // Use visitSwitchCase to actually insert the fast branch sequence for this
01647   // cond branch.
01648   visitSwitchCase(CB, BrMBB);
01649 }
01650 
01651 /// visitSwitchCase - Emits the necessary code to represent a single node in
01652 /// the binary search tree resulting from lowering a switch instruction.
01653 void SelectionDAGBuilder::visitSwitchCase(CaseBlock &CB,
01654                                           MachineBasicBlock *SwitchBB) {
01655   SDValue Cond;
01656   SDValue CondLHS = getValue(CB.CmpLHS);
01657   SDLoc dl = getCurSDLoc();
01658 
01659   // Build the setcc now.
01660   if (!CB.CmpMHS) {
01661     // Fold "(X == true)" to X and "(X == false)" to !X to
01662     // handle common cases produced by branch lowering.
01663     if (CB.CmpRHS == ConstantInt::getTrue(*DAG.getContext()) &&
01664         CB.CC == ISD::SETEQ)
01665       Cond = CondLHS;
01666     else if (CB.CmpRHS == ConstantInt::getFalse(*DAG.getContext()) &&
01667              CB.CC == ISD::SETEQ) {
01668       SDValue True = DAG.getConstant(1, CondLHS.getValueType());
01669       Cond = DAG.getNode(ISD::XOR, dl, CondLHS.getValueType(), CondLHS, True);
01670     } else
01671       Cond = DAG.getSetCC(dl, MVT::i1, CondLHS, getValue(CB.CmpRHS), CB.CC);
01672   } else {
01673     assert(CB.CC == ISD::SETLE && "Can handle only LE ranges now");
01674 
01675     const APInt& Low = cast<ConstantInt>(CB.CmpLHS)->getValue();
01676     const APInt& High  = cast<ConstantInt>(CB.CmpRHS)->getValue();
01677 
01678     SDValue CmpOp = getValue(CB.CmpMHS);
01679     EVT VT = CmpOp.getValueType();
01680 
01681     if (cast<ConstantInt>(CB.CmpLHS)->isMinValue(true)) {
01682       Cond = DAG.getSetCC(dl, MVT::i1, CmpOp, DAG.getConstant(High, VT),
01683                           ISD::SETLE);
01684     } else {
01685       SDValue SUB = DAG.getNode(ISD::SUB, dl,
01686                                 VT, CmpOp, DAG.getConstant(Low, VT));
01687       Cond = DAG.getSetCC(dl, MVT::i1, SUB,
01688                           DAG.getConstant(High-Low, VT), ISD::SETULE);
01689     }
01690   }
01691 
01692   // Update successor info
01693   addSuccessorWithWeight(SwitchBB, CB.TrueBB, CB.TrueWeight);
01694   // TrueBB and FalseBB are always different unless the incoming IR is
01695   // degenerate. This only happens when running llc on weird IR.
01696   if (CB.TrueBB != CB.FalseBB)
01697     addSuccessorWithWeight(SwitchBB, CB.FalseBB, CB.FalseWeight);
01698 
01699   // Set NextBlock to be the MBB immediately after the current one, if any.
01700   // This is used to avoid emitting unnecessary branches to the next block.
01701   MachineBasicBlock *NextBlock = nullptr;
01702   MachineFunction::iterator BBI = SwitchBB;
01703   if (++BBI != FuncInfo.MF->end())
01704     NextBlock = BBI;
01705 
01706   // If the lhs block is the next block, invert the condition so that we can
01707   // fall through to the lhs instead of the rhs block.
01708   if (CB.TrueBB == NextBlock) {
01709     std::swap(CB.TrueBB, CB.FalseBB);
01710     SDValue True = DAG.getConstant(1, Cond.getValueType());
01711     Cond = DAG.getNode(ISD::XOR, dl, Cond.getValueType(), Cond, True);
01712   }
01713 
01714   SDValue BrCond = DAG.getNode(ISD::BRCOND, dl,
01715                                MVT::Other, getControlRoot(), Cond,
01716                                DAG.getBasicBlock(CB.TrueBB));
01717 
01718   // Insert the false branch. Do this even if it's a fall through branch,
01719   // this makes it easier to do DAG optimizations which require inverting
01720   // the branch condition.
01721   BrCond = DAG.getNode(ISD::BR, dl, MVT::Other, BrCond,
01722                        DAG.getBasicBlock(CB.FalseBB));
01723 
01724   DAG.setRoot(BrCond);
01725 }
01726 
01727 /// visitJumpTable - Emit JumpTable node in the current MBB
01728 void SelectionDAGBuilder::visitJumpTable(JumpTable &JT) {
01729   // Emit the code for the jump table
01730   assert(JT.Reg != -1U && "Should lower JT Header first!");
01731   EVT PTy = DAG.getTargetLoweringInfo().getPointerTy();
01732   SDValue Index = DAG.getCopyFromReg(getControlRoot(), getCurSDLoc(),
01733                                      JT.Reg, PTy);
01734   SDValue Table = DAG.getJumpTable(JT.JTI, PTy);
01735   SDValue BrJumpTable = DAG.getNode(ISD::BR_JT, getCurSDLoc(),
01736                                     MVT::Other, Index.getValue(1),
01737                                     Table, Index);
01738   DAG.setRoot(BrJumpTable);
01739 }
01740 
01741 /// visitJumpTableHeader - This function emits necessary code to produce index
01742 /// in the JumpTable from switch case.
01743 void SelectionDAGBuilder::visitJumpTableHeader(JumpTable &JT,
01744                                                JumpTableHeader &JTH,
01745                                                MachineBasicBlock *SwitchBB) {
01746   // Subtract the lowest switch case value from the value being switched on and
01747   // conditional branch to default mbb if the result is greater than the
01748   // difference between smallest and largest cases.
01749   SDValue SwitchOp = getValue(JTH.SValue);
01750   EVT VT = SwitchOp.getValueType();
01751   SDValue Sub = DAG.getNode(ISD::SUB, getCurSDLoc(), VT, SwitchOp,
01752                             DAG.getConstant(JTH.First, VT));
01753 
01754   // The SDNode we just created, which holds the value being switched on minus
01755   // the smallest case value, needs to be copied to a virtual register so it
01756   // can be used as an index into the jump table in a subsequent basic block.
01757   // This value may be smaller or larger than the target's pointer type, and
01758   // therefore require extension or truncating.
01759   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
01760   SwitchOp = DAG.getZExtOrTrunc(Sub, getCurSDLoc(), TLI.getPointerTy());
01761 
01762   unsigned JumpTableReg = FuncInfo.CreateReg(TLI.getPointerTy());
01763   SDValue CopyTo = DAG.getCopyToReg(getControlRoot(), getCurSDLoc(),
01764                                     JumpTableReg, SwitchOp);
01765   JT.Reg = JumpTableReg;
01766 
01767   // Emit the range check for the jump table, and branch to the default block
01768   // for the switch statement if the value being switched on exceeds the largest
01769   // case in the switch.
01770   SDValue CMP =
01771       DAG.getSetCC(getCurSDLoc(), TLI.getSetCCResultType(*DAG.getContext(),
01772                                                          Sub.getValueType()),
01773                    Sub, DAG.getConstant(JTH.Last - JTH.First, VT), ISD::SETUGT);
01774 
01775   // Set NextBlock to be the MBB immediately after the current one, if any.
01776   // This is used to avoid emitting unnecessary branches to the next block.
01777   MachineBasicBlock *NextBlock = nullptr;
01778   MachineFunction::iterator BBI = SwitchBB;
01779 
01780   if (++BBI != FuncInfo.MF->end())
01781     NextBlock = BBI;
01782 
01783   SDValue BrCond = DAG.getNode(ISD::BRCOND, getCurSDLoc(),
01784                                MVT::Other, CopyTo, CMP,
01785                                DAG.getBasicBlock(JT.Default));
01786 
01787   if (JT.MBB != NextBlock)
01788     BrCond = DAG.getNode(ISD::BR, getCurSDLoc(), MVT::Other, BrCond,
01789                          DAG.getBasicBlock(JT.MBB));
01790 
01791   DAG.setRoot(BrCond);
01792 }
01793 
01794 /// Codegen a new tail for a stack protector check ParentMBB which has had its
01795 /// tail spliced into a stack protector check success bb.
01796 ///
01797 /// For a high level explanation of how this fits into the stack protector
01798 /// generation see the comment on the declaration of class
01799 /// StackProtectorDescriptor.
01800 void SelectionDAGBuilder::visitSPDescriptorParent(StackProtectorDescriptor &SPD,
01801                                                   MachineBasicBlock *ParentBB) {
01802 
01803   // First create the loads to the guard/stack slot for the comparison.
01804   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
01805   EVT PtrTy = TLI.getPointerTy();
01806 
01807   MachineFrameInfo *MFI = ParentBB->getParent()->getFrameInfo();
01808   int FI = MFI->getStackProtectorIndex();
01809 
01810   const Value *IRGuard = SPD.getGuard();
01811   SDValue GuardPtr = getValue(IRGuard);
01812   SDValue StackSlotPtr = DAG.getFrameIndex(FI, PtrTy);
01813 
01814   unsigned Align =
01815     TLI.getDataLayout()->getPrefTypeAlignment(IRGuard->getType());
01816 
01817   SDValue Guard;
01818 
01819   // If GuardReg is set and useLoadStackGuardNode returns true, retrieve the
01820   // guard value from the virtual register holding the value. Otherwise, emit a
01821   // volatile load to retrieve the stack guard value.
01822   unsigned GuardReg = SPD.getGuardReg();
01823 
01824   if (GuardReg && TLI.useLoadStackGuardNode())
01825     Guard = DAG.getCopyFromReg(DAG.getEntryNode(), getCurSDLoc(), GuardReg,
01826                                PtrTy);
01827   else
01828     Guard = DAG.getLoad(PtrTy, getCurSDLoc(), DAG.getEntryNode(),
01829                         GuardPtr, MachinePointerInfo(IRGuard, 0),
01830                         true, false, false, Align);
01831 
01832   SDValue StackSlot = DAG.getLoad(PtrTy, getCurSDLoc(), DAG.getEntryNode(),
01833                                   StackSlotPtr,
01834                                   MachinePointerInfo::getFixedStack(FI),
01835                                   true, false, false, Align);
01836 
01837   // Perform the comparison via a subtract/getsetcc.
01838   EVT VT = Guard.getValueType();
01839   SDValue Sub = DAG.getNode(ISD::SUB, getCurSDLoc(), VT, Guard, StackSlot);
01840 
01841   SDValue Cmp =
01842       DAG.getSetCC(getCurSDLoc(), TLI.getSetCCResultType(*DAG.getContext(),
01843                                                          Sub.getValueType()),
01844                    Sub, DAG.getConstant(0, VT), ISD::SETNE);
01845 
01846   // If the sub is not 0, then we know the guard/stackslot do not equal, so
01847   // branch to failure MBB.
01848   SDValue BrCond = DAG.getNode(ISD::BRCOND, getCurSDLoc(),
01849                                MVT::Other, StackSlot.getOperand(0),
01850                                Cmp, DAG.getBasicBlock(SPD.getFailureMBB()));
01851   // Otherwise branch to success MBB.
01852   SDValue Br = DAG.getNode(ISD::BR, getCurSDLoc(),
01853                            MVT::Other, BrCond,
01854                            DAG.getBasicBlock(SPD.getSuccessMBB()));
01855 
01856   DAG.setRoot(Br);
01857 }
01858 
01859 /// Codegen the failure basic block for a stack protector check.
01860 ///
01861 /// A failure stack protector machine basic block consists simply of a call to
01862 /// __stack_chk_fail().
01863 ///
01864 /// For a high level explanation of how this fits into the stack protector
01865 /// generation see the comment on the declaration of class
01866 /// StackProtectorDescriptor.
01867 void
01868 SelectionDAGBuilder::visitSPDescriptorFailure(StackProtectorDescriptor &SPD) {
01869   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
01870   SDValue Chain =
01871       TLI.makeLibCall(DAG, RTLIB::STACKPROTECTOR_CHECK_FAIL, MVT::isVoid,
01872                       nullptr, 0, false, getCurSDLoc(), false, false).second;
01873   DAG.setRoot(Chain);
01874 }
01875 
01876 /// visitBitTestHeader - This function emits necessary code to produce value
01877 /// suitable for "bit tests"
01878 void SelectionDAGBuilder::visitBitTestHeader(BitTestBlock &B,
01879                                              MachineBasicBlock *SwitchBB) {
01880   // Subtract the minimum value
01881   SDValue SwitchOp = getValue(B.SValue);
01882   EVT VT = SwitchOp.getValueType();
01883   SDValue Sub = DAG.getNode(ISD::SUB, getCurSDLoc(), VT, SwitchOp,
01884                             DAG.getConstant(B.First, VT));
01885 
01886   // Check range
01887   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
01888   SDValue RangeCmp =
01889       DAG.getSetCC(getCurSDLoc(), TLI.getSetCCResultType(*DAG.getContext(),
01890                                                          Sub.getValueType()),
01891                    Sub, DAG.getConstant(B.Range, VT), ISD::SETUGT);
01892 
01893   // Determine the type of the test operands.
01894   bool UsePtrType = false;
01895   if (!TLI.isTypeLegal(VT))
01896     UsePtrType = true;
01897   else {
01898     for (unsigned i = 0, e = B.Cases.size(); i != e; ++i)
01899       if (!isUIntN(VT.getSizeInBits(), B.Cases[i].Mask)) {
01900         // Switch table case range are encoded into series of masks.
01901         // Just use pointer type, it's guaranteed to fit.
01902         UsePtrType = true;
01903         break;
01904       }
01905   }
01906   if (UsePtrType) {
01907     VT = TLI.getPointerTy();
01908     Sub = DAG.getZExtOrTrunc(Sub, getCurSDLoc(), VT);
01909   }
01910 
01911   B.RegVT = VT.getSimpleVT();
01912   B.Reg = FuncInfo.CreateReg(B.RegVT);
01913   SDValue CopyTo = DAG.getCopyToReg(getControlRoot(), getCurSDLoc(),
01914                                     B.Reg, Sub);
01915 
01916   // Set NextBlock to be the MBB immediately after the current one, if any.
01917   // This is used to avoid emitting unnecessary branches to the next block.
01918   MachineBasicBlock *NextBlock = nullptr;
01919   MachineFunction::iterator BBI = SwitchBB;
01920   if (++BBI != FuncInfo.MF->end())
01921     NextBlock = BBI;
01922 
01923   MachineBasicBlock* MBB = B.Cases[0].ThisBB;
01924 
01925   addSuccessorWithWeight(SwitchBB, B.Default);
01926   addSuccessorWithWeight(SwitchBB, MBB);
01927 
01928   SDValue BrRange = DAG.getNode(ISD::BRCOND, getCurSDLoc(),
01929                                 MVT::Other, CopyTo, RangeCmp,
01930                                 DAG.getBasicBlock(B.Default));
01931 
01932   if (MBB != NextBlock)
01933     BrRange = DAG.getNode(ISD::BR, getCurSDLoc(), MVT::Other, CopyTo,
01934                           DAG.getBasicBlock(MBB));
01935 
01936   DAG.setRoot(BrRange);
01937 }
01938 
01939 /// visitBitTestCase - this function produces one "bit test"
01940 void SelectionDAGBuilder::visitBitTestCase(BitTestBlock &BB,
01941                                            MachineBasicBlock* NextMBB,
01942                                            uint32_t BranchWeightToNext,
01943                                            unsigned Reg,
01944                                            BitTestCase &B,
01945                                            MachineBasicBlock *SwitchBB) {
01946   MVT VT = BB.RegVT;
01947   SDValue ShiftOp = DAG.getCopyFromReg(getControlRoot(), getCurSDLoc(),
01948                                        Reg, VT);
01949   SDValue Cmp;
01950   unsigned PopCount = CountPopulation_64(B.Mask);
01951   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
01952   if (PopCount == 1) {
01953     // Testing for a single bit; just compare the shift count with what it
01954     // would need to be to shift a 1 bit in that position.
01955     Cmp = DAG.getSetCC(
01956         getCurSDLoc(), TLI.getSetCCResultType(*DAG.getContext(), VT), ShiftOp,
01957         DAG.getConstant(countTrailingZeros(B.Mask), VT), ISD::SETEQ);
01958   } else if (PopCount == BB.Range) {
01959     // There is only one zero bit in the range, test for it directly.
01960     Cmp = DAG.getSetCC(
01961         getCurSDLoc(), TLI.getSetCCResultType(*DAG.getContext(), VT), ShiftOp,
01962         DAG.getConstant(CountTrailingOnes_64(B.Mask), VT), ISD::SETNE);
01963   } else {
01964     // Make desired shift
01965     SDValue SwitchVal = DAG.getNode(ISD::SHL, getCurSDLoc(), VT,
01966                                     DAG.getConstant(1, VT), ShiftOp);
01967 
01968     // Emit bit tests and jumps
01969     SDValue AndOp = DAG.getNode(ISD::AND, getCurSDLoc(),
01970                                 VT, SwitchVal, DAG.getConstant(B.Mask, VT));
01971     Cmp = DAG.getSetCC(getCurSDLoc(),
01972                        TLI.getSetCCResultType(*DAG.getContext(), VT), AndOp,
01973                        DAG.getConstant(0, VT), ISD::SETNE);
01974   }
01975 
01976   // The branch weight from SwitchBB to B.TargetBB is B.ExtraWeight.
01977   addSuccessorWithWeight(SwitchBB, B.TargetBB, B.ExtraWeight);
01978   // The branch weight from SwitchBB to NextMBB is BranchWeightToNext.
01979   addSuccessorWithWeight(SwitchBB, NextMBB, BranchWeightToNext);
01980 
01981   SDValue BrAnd = DAG.getNode(ISD::BRCOND, getCurSDLoc(),
01982                               MVT::Other, getControlRoot(),
01983                               Cmp, DAG.getBasicBlock(B.TargetBB));
01984 
01985   // Set NextBlock to be the MBB immediately after the current one, if any.
01986   // This is used to avoid emitting unnecessary branches to the next block.
01987   MachineBasicBlock *NextBlock = nullptr;
01988   MachineFunction::iterator BBI = SwitchBB;
01989   if (++BBI != FuncInfo.MF->end())
01990     NextBlock = BBI;
01991 
01992   if (NextMBB != NextBlock)
01993     BrAnd = DAG.getNode(ISD::BR, getCurSDLoc(), MVT::Other, BrAnd,
01994                         DAG.getBasicBlock(NextMBB));
01995 
01996   DAG.setRoot(BrAnd);
01997 }
01998 
01999 void SelectionDAGBuilder::visitInvoke(const InvokeInst &I) {
02000   MachineBasicBlock *InvokeMBB = FuncInfo.MBB;
02001 
02002   // Retrieve successors.
02003   MachineBasicBlock *Return = FuncInfo.MBBMap[I.getSuccessor(0)];
02004   MachineBasicBlock *LandingPad = FuncInfo.MBBMap[I.getSuccessor(1)];
02005 
02006   const Value *Callee(I.getCalledValue());
02007   const Function *Fn = dyn_cast<Function>(Callee);
02008   if (isa<InlineAsm>(Callee))
02009     visitInlineAsm(&I);
02010   else if (Fn && Fn->isIntrinsic()) {
02011     switch (Fn->getIntrinsicID()) {
02012     default:
02013       llvm_unreachable("Cannot invoke this intrinsic");
02014     case Intrinsic::donothing:
02015       // Ignore invokes to @llvm.donothing: jump directly to the next BB.
02016       break;
02017     case Intrinsic::experimental_patchpoint_void:
02018     case Intrinsic::experimental_patchpoint_i64:
02019       visitPatchpoint(&I, LandingPad);
02020       break;
02021     }
02022   } else
02023     LowerCallTo(&I, getValue(Callee), false, LandingPad);
02024 
02025   // If the value of the invoke is used outside of its defining block, make it
02026   // available as a virtual register.
02027   CopyToExportRegsIfNeeded(&I);
02028 
02029   // Update successor info
02030   addSuccessorWithWeight(InvokeMBB, Return);
02031   addSuccessorWithWeight(InvokeMBB, LandingPad);
02032 
02033   // Drop into normal successor.
02034   DAG.setRoot(DAG.getNode(ISD::BR, getCurSDLoc(),
02035                           MVT::Other, getControlRoot(),
02036                           DAG.getBasicBlock(Return)));
02037 }
02038 
02039 void SelectionDAGBuilder::visitResume(const ResumeInst &RI) {
02040   llvm_unreachable("SelectionDAGBuilder shouldn't visit resume instructions!");
02041 }
02042 
02043 void SelectionDAGBuilder::visitLandingPad(const LandingPadInst &LP) {
02044   assert(FuncInfo.MBB->isLandingPad() &&
02045          "Call to landingpad not in landing pad!");
02046 
02047   MachineBasicBlock *MBB = FuncInfo.MBB;
02048   MachineModuleInfo &MMI = DAG.getMachineFunction().getMMI();
02049   AddLandingPadInfo(LP, MMI, MBB);
02050 
02051   // If there aren't registers to copy the values into (e.g., during SjLj
02052   // exceptions), then don't bother to create these DAG nodes.
02053   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
02054   if (TLI.getExceptionPointerRegister() == 0 &&
02055       TLI.getExceptionSelectorRegister() == 0)
02056     return;
02057 
02058   SmallVector<EVT, 2> ValueVTs;
02059   ComputeValueVTs(TLI, LP.getType(), ValueVTs);
02060   assert(ValueVTs.size() == 2 && "Only two-valued landingpads are supported");
02061 
02062   // Get the two live-in registers as SDValues. The physregs have already been
02063   // copied into virtual registers.
02064   SDValue Ops[2];
02065   Ops[0] = DAG.getZExtOrTrunc(
02066       DAG.getCopyFromReg(DAG.getEntryNode(), getCurSDLoc(),
02067                          FuncInfo.ExceptionPointerVirtReg, TLI.getPointerTy()),
02068       getCurSDLoc(), ValueVTs[0]);
02069   Ops[1] = DAG.getZExtOrTrunc(
02070       DAG.getCopyFromReg(DAG.getEntryNode(), getCurSDLoc(),
02071                          FuncInfo.ExceptionSelectorVirtReg, TLI.getPointerTy()),
02072       getCurSDLoc(), ValueVTs[1]);
02073 
02074   // Merge into one.
02075   SDValue Res = DAG.getNode(ISD::MERGE_VALUES, getCurSDLoc(),
02076                             DAG.getVTList(ValueVTs), Ops);
02077   setValue(&LP, Res);
02078 }
02079 
02080 /// handleSmallSwitchCaseRange - Emit a series of specific tests (suitable for
02081 /// small case ranges).
02082 bool SelectionDAGBuilder::handleSmallSwitchRange(CaseRec& CR,
02083                                                  CaseRecVector& WorkList,
02084                                                  const Value* SV,
02085                                                  MachineBasicBlock *Default,
02086                                                  MachineBasicBlock *SwitchBB) {
02087   // Size is the number of Cases represented by this range.
02088   size_t Size = CR.Range.second - CR.Range.first;
02089   if (Size > 3)
02090     return false;
02091 
02092   // Get the MachineFunction which holds the current MBB.  This is used when
02093   // inserting any additional MBBs necessary to represent the switch.
02094   MachineFunction *CurMF = FuncInfo.MF;
02095 
02096   // Figure out which block is immediately after the current one.
02097   MachineBasicBlock *NextBlock = nullptr;
02098   MachineFunction::iterator BBI = CR.CaseBB;
02099 
02100   if (++BBI != FuncInfo.MF->end())
02101     NextBlock = BBI;
02102 
02103   BranchProbabilityInfo *BPI = FuncInfo.BPI;
02104   // If any two of the cases has the same destination, and if one value
02105   // is the same as the other, but has one bit unset that the other has set,
02106   // use bit manipulation to do two compares at once.  For example:
02107   // "if (X == 6 || X == 4)" -> "if ((X|2) == 6)"
02108   // TODO: This could be extended to merge any 2 cases in switches with 3 cases.
02109   // TODO: Handle cases where CR.CaseBB != SwitchBB.
02110   if (Size == 2 && CR.CaseBB == SwitchBB) {
02111     Case &Small = *CR.Range.first;
02112     Case &Big = *(CR.Range.second-1);
02113 
02114     if (Small.Low == Small.High && Big.Low == Big.High && Small.BB == Big.BB) {
02115       const APInt& SmallValue = cast<ConstantInt>(Small.Low)->getValue();
02116       const APInt& BigValue = cast<ConstantInt>(Big.Low)->getValue();
02117 
02118       // Check that there is only one bit different.
02119       if (BigValue.countPopulation() == SmallValue.countPopulation() + 1 &&
02120           (SmallValue | BigValue) == BigValue) {
02121         // Isolate the common bit.
02122         APInt CommonBit = BigValue & ~SmallValue;
02123         assert((SmallValue | CommonBit) == BigValue &&
02124                CommonBit.countPopulation() == 1 && "Not a common bit?");
02125 
02126         SDValue CondLHS = getValue(SV);
02127         EVT VT = CondLHS.getValueType();
02128         SDLoc DL = getCurSDLoc();
02129 
02130         SDValue Or = DAG.getNode(ISD::OR, DL, VT, CondLHS,
02131                                  DAG.getConstant(CommonBit, VT));
02132         SDValue Cond = DAG.getSetCC(DL, MVT::i1,
02133                                     Or, DAG.getConstant(BigValue, VT),
02134                                     ISD::SETEQ);
02135 
02136         // Update successor info.
02137         // Both Small and Big will jump to Small.BB, so we sum up the weights.
02138         addSuccessorWithWeight(SwitchBB, Small.BB,
02139                                Small.ExtraWeight + Big.ExtraWeight);
02140         addSuccessorWithWeight(SwitchBB, Default,
02141           // The default destination is the first successor in IR.
02142           BPI ? BPI->getEdgeWeight(SwitchBB->getBasicBlock(), (unsigned)0) : 0);
02143 
02144         // Insert the true branch.
02145         SDValue BrCond = DAG.getNode(ISD::BRCOND, DL, MVT::Other,
02146                                      getControlRoot(), Cond,
02147                                      DAG.getBasicBlock(Small.BB));
02148 
02149         // Insert the false branch.
02150         BrCond = DAG.getNode(ISD::BR, DL, MVT::Other, BrCond,
02151                              DAG.getBasicBlock(Default));
02152 
02153         DAG.setRoot(BrCond);
02154         return true;
02155       }
02156     }
02157   }
02158 
02159   // Order cases by weight so the most likely case will be checked first.
02160   uint32_t UnhandledWeights = 0;
02161   if (BPI) {
02162     for (CaseItr I = CR.Range.first, IE = CR.Range.second; I != IE; ++I) {
02163       uint32_t IWeight = I->ExtraWeight;
02164       UnhandledWeights += IWeight;
02165       for (CaseItr J = CR.Range.first; J < I; ++J) {
02166         uint32_t JWeight = J->ExtraWeight;
02167         if (IWeight > JWeight)
02168           std::swap(*I, *J);
02169       }
02170     }
02171   }
02172   // Rearrange the case blocks so that the last one falls through if possible.
02173   Case &BackCase = *(CR.Range.second-1);
02174   if (Size > 1 &&
02175       NextBlock && Default != NextBlock && BackCase.BB != NextBlock) {
02176     // The last case block won't fall through into 'NextBlock' if we emit the
02177     // branches in this order.  See if rearranging a case value would help.
02178     // We start at the bottom as it's the case with the least weight.
02179     for (Case *I = &*(CR.Range.second-2), *E = &*CR.Range.first-1; I != E; --I)
02180       if (I->BB == NextBlock) {
02181         std::swap(*I, BackCase);
02182         break;
02183       }
02184   }
02185 
02186   // Create a CaseBlock record representing a conditional branch to
02187   // the Case's target mbb if the value being switched on SV is equal
02188   // to C.
02189   MachineBasicBlock *CurBlock = CR.CaseBB;
02190   for (CaseItr I = CR.Range.first, E = CR.Range.second; I != E; ++I) {
02191     MachineBasicBlock *FallThrough;
02192     if (I != E-1) {
02193       FallThrough = CurMF->CreateMachineBasicBlock(CurBlock->getBasicBlock());
02194       CurMF->insert(BBI, FallThrough);
02195 
02196       // Put SV in a virtual register to make it available from the new blocks.
02197       ExportFromCurrentBlock(SV);
02198     } else {
02199       // If the last case doesn't match, go to the default block.
02200       FallThrough = Default;
02201     }
02202 
02203     const Value *RHS, *LHS, *MHS;
02204     ISD::CondCode CC;
02205     if (I->High == I->Low) {
02206       // This is just small small case range :) containing exactly 1 case
02207       CC = ISD::SETEQ;
02208       LHS = SV; RHS = I->High; MHS = nullptr;
02209     } else {
02210       CC = ISD::SETLE;
02211       LHS = I->Low; MHS = SV; RHS = I->High;
02212     }
02213 
02214     // The false weight should be sum of all un-handled cases.
02215     UnhandledWeights -= I->ExtraWeight;
02216     CaseBlock CB(CC, LHS, RHS, MHS, /* truebb */ I->BB, /* falsebb */ FallThrough,
02217                  /* me */ CurBlock,
02218                  /* trueweight */ I->ExtraWeight,
02219                  /* falseweight */ UnhandledWeights);
02220 
02221     // If emitting the first comparison, just call visitSwitchCase to emit the
02222     // code into the current block.  Otherwise, push the CaseBlock onto the
02223     // vector to be later processed by SDISel, and insert the node's MBB
02224     // before the next MBB.
02225     if (CurBlock == SwitchBB)
02226       visitSwitchCase(CB, SwitchBB);
02227     else
02228       SwitchCases.push_back(CB);
02229 
02230     CurBlock = FallThrough;
02231   }
02232 
02233   return true;
02234 }
02235 
02236 static inline bool areJTsAllowed(const TargetLowering &TLI) {
02237   return TLI.isOperationLegalOrCustom(ISD::BR_JT, MVT::Other) ||
02238          TLI.isOperationLegalOrCustom(ISD::BRIND, MVT::Other);
02239 }
02240 
02241 static APInt ComputeRange(const APInt &First, const APInt &Last) {
02242   uint32_t BitWidth = std::max(Last.getBitWidth(), First.getBitWidth()) + 1;
02243   APInt LastExt = Last.sext(BitWidth), FirstExt = First.sext(BitWidth);
02244   return (LastExt - FirstExt + 1ULL);
02245 }
02246 
02247 /// handleJTSwitchCase - Emit jumptable for current switch case range
02248 bool SelectionDAGBuilder::handleJTSwitchCase(CaseRec &CR,
02249                                              CaseRecVector &WorkList,
02250                                              const Value *SV,
02251                                              MachineBasicBlock *Default,
02252                                              MachineBasicBlock *SwitchBB) {
02253   Case& FrontCase = *CR.Range.first;
02254   Case& BackCase  = *(CR.Range.second-1);
02255 
02256   const APInt &First = cast<ConstantInt>(FrontCase.Low)->getValue();
02257   const APInt &Last  = cast<ConstantInt>(BackCase.High)->getValue();
02258 
02259   APInt TSize(First.getBitWidth(), 0);
02260   for (CaseItr I = CR.Range.first, E = CR.Range.second; I != E; ++I)
02261     TSize += I->size();
02262 
02263   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
02264   if (!areJTsAllowed(TLI) || TSize.ult(TLI.getMinimumJumpTableEntries()))
02265     return false;
02266 
02267   APInt Range = ComputeRange(First, Last);
02268   // The density is TSize / Range. Require at least 40%.
02269   // It should not be possible for IntTSize to saturate for sane code, but make
02270   // sure we handle Range saturation correctly.
02271   uint64_t IntRange = Range.getLimitedValue(UINT64_MAX/10);
02272   uint64_t IntTSize = TSize.getLimitedValue(UINT64_MAX/10);
02273   if (IntTSize * 10 < IntRange * 4)
02274     return false;
02275 
02276   DEBUG(dbgs() << "Lowering jump table\n"
02277                << "First entry: " << First << ". Last entry: " << Last << '\n'
02278                << "Range: " << Range << ". Size: " << TSize << ".\n\n");
02279 
02280   // Get the MachineFunction which holds the current MBB.  This is used when
02281   // inserting any additional MBBs necessary to represent the switch.
02282   MachineFunction *CurMF = FuncInfo.MF;
02283 
02284   // Figure out which block is immediately after the current one.
02285   MachineFunction::iterator BBI = CR.CaseBB;
02286   ++BBI;
02287 
02288   const BasicBlock *LLVMBB = CR.CaseBB->getBasicBlock();
02289 
02290   // Create a new basic block to hold the code for loading the address
02291   // of the jump table, and jumping to it.  Update successor information;
02292   // we will either branch to the default case for the switch, or the jump
02293   // table.
02294   MachineBasicBlock *JumpTableBB = CurMF->CreateMachineBasicBlock(LLVMBB);
02295   CurMF->insert(BBI, JumpTableBB);
02296 
02297   addSuccessorWithWeight(CR.CaseBB, Default);
02298   addSuccessorWithWeight(CR.CaseBB, JumpTableBB);
02299 
02300   // Build a vector of destination BBs, corresponding to each target
02301   // of the jump table. If the value of the jump table slot corresponds to
02302   // a case statement, push the case's BB onto the vector, otherwise, push
02303   // the default BB.
02304   std::vector<MachineBasicBlock*> DestBBs;
02305   APInt TEI = First;
02306   for (CaseItr I = CR.Range.first, E = CR.Range.second; I != E; ++TEI) {
02307     const APInt &Low = cast<ConstantInt>(I->Low)->getValue();
02308     const APInt &High = cast<ConstantInt>(I->High)->getValue();
02309 
02310     if (Low.sle(TEI) && TEI.sle(High)) {
02311       DestBBs.push_back(I->BB);
02312       if (TEI==High)
02313         ++I;
02314     } else {
02315       DestBBs.push_back(Default);
02316     }
02317   }
02318 
02319   // Calculate weight for each unique destination in CR.
02320   DenseMap<MachineBasicBlock*, uint32_t> DestWeights;
02321   if (FuncInfo.BPI)
02322     for (CaseItr I = CR.Range.first, E = CR.Range.second; I != E; ++I) {
02323       DenseMap<MachineBasicBlock*, uint32_t>::iterator Itr =
02324           DestWeights.find(I->BB);
02325       if (Itr != DestWeights.end())
02326         Itr->second += I->ExtraWeight;
02327       else
02328         DestWeights[I->BB] = I->ExtraWeight;
02329     }
02330 
02331   // Update successor info. Add one edge to each unique successor.
02332   BitVector SuccsHandled(CR.CaseBB->getParent()->getNumBlockIDs());
02333   for (std::vector<MachineBasicBlock*>::iterator I = DestBBs.begin(),
02334          E = DestBBs.end(); I != E; ++I) {
02335     if (!SuccsHandled[(*I)->getNumber()]) {
02336       SuccsHandled[(*I)->getNumber()] = true;
02337       DenseMap<MachineBasicBlock*, uint32_t>::iterator Itr =
02338           DestWeights.find(*I);
02339       addSuccessorWithWeight(JumpTableBB, *I,
02340                              Itr != DestWeights.end() ? Itr->second : 0);
02341     }
02342   }
02343 
02344   // Create a jump table index for this jump table.
02345   unsigned JTEncoding = TLI.getJumpTableEncoding();
02346   unsigned JTI = CurMF->getOrCreateJumpTableInfo(JTEncoding)
02347                        ->createJumpTableIndex(DestBBs);
02348 
02349   // Set the jump table information so that we can codegen it as a second
02350   // MachineBasicBlock
02351   JumpTable JT(-1U, JTI, JumpTableBB, Default);
02352   JumpTableHeader JTH(First, Last, SV, CR.CaseBB, (CR.CaseBB == SwitchBB));
02353   if (CR.CaseBB == SwitchBB)
02354     visitJumpTableHeader(JT, JTH, SwitchBB);
02355 
02356   JTCases.push_back(JumpTableBlock(JTH, JT));
02357   return true;
02358 }
02359 
02360 /// handleBTSplitSwitchCase - emit comparison and split binary search tree into
02361 /// 2 subtrees.
02362 bool SelectionDAGBuilder::handleBTSplitSwitchCase(CaseRec& CR,
02363                                                   CaseRecVector& WorkList,
02364                                                   const Value* SV,
02365                                                   MachineBasicBlock* SwitchBB) {
02366   // Get the MachineFunction which holds the current MBB.  This is used when
02367   // inserting any additional MBBs necessary to represent the switch.
02368   MachineFunction *CurMF = FuncInfo.MF;
02369 
02370   // Figure out which block is immediately after the current one.
02371   MachineFunction::iterator BBI = CR.CaseBB;
02372   ++BBI;
02373 
02374   Case& FrontCase = *CR.Range.first;
02375   Case& BackCase  = *(CR.Range.second-1);
02376   const BasicBlock *LLVMBB = CR.CaseBB->getBasicBlock();
02377 
02378   // Size is the number of Cases represented by this range.
02379   unsigned Size = CR.Range.second - CR.Range.first;
02380 
02381   const APInt &First = cast<ConstantInt>(FrontCase.Low)->getValue();
02382   const APInt &Last  = cast<ConstantInt>(BackCase.High)->getValue();
02383   double FMetric = 0;
02384   CaseItr Pivot = CR.Range.first + Size/2;
02385 
02386   // Select optimal pivot, maximizing sum density of LHS and RHS. This will
02387   // (heuristically) allow us to emit JumpTable's later.
02388   APInt TSize(First.getBitWidth(), 0);
02389   for (CaseItr I = CR.Range.first, E = CR.Range.second;
02390        I!=E; ++I)
02391     TSize += I->size();
02392 
02393   APInt LSize = FrontCase.size();
02394   APInt RSize = TSize-LSize;
02395   DEBUG(dbgs() << "Selecting best pivot: \n"
02396                << "First: " << First << ", Last: " << Last <<'\n'
02397                << "LSize: " << LSize << ", RSize: " << RSize << '\n');
02398   for (CaseItr I = CR.Range.first, J=I+1, E = CR.Range.second;
02399        J!=E; ++I, ++J) {
02400     const APInt &LEnd = cast<ConstantInt>(I->High)->getValue();
02401     const APInt &RBegin = cast<ConstantInt>(J->Low)->getValue();
02402     APInt Range = ComputeRange(LEnd, RBegin);
02403     assert((Range - 2ULL).isNonNegative() &&
02404            "Invalid case distance");
02405     // Use volatile double here to avoid excess precision issues on some hosts,
02406     // e.g. that use 80-bit X87 registers.
02407     volatile double LDensity =
02408        (double)LSize.roundToDouble() /
02409                            (LEnd - First + 1ULL).roundToDouble();
02410     volatile double RDensity =
02411       (double)RSize.roundToDouble() /
02412                            (Last - RBegin + 1ULL).roundToDouble();
02413     volatile double Metric = Range.logBase2()*(LDensity+RDensity);
02414     // Should always split in some non-trivial place
02415     DEBUG(dbgs() <<"=>Step\n"
02416                  << "LEnd: " << LEnd << ", RBegin: " << RBegin << '\n'
02417                  << "LDensity: " << LDensity
02418                  << ", RDensity: " << RDensity << '\n'
02419                  << "Metric: " << Metric << '\n');
02420     if (FMetric < Metric) {
02421       Pivot = J;
02422       FMetric = Metric;
02423       DEBUG(dbgs() << "Current metric set to: " << FMetric << '\n');
02424     }
02425 
02426     LSize += J->size();
02427     RSize -= J->size();
02428   }
02429 
02430   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
02431   if (areJTsAllowed(TLI)) {
02432     // If our case is dense we *really* should handle it earlier!
02433     assert((FMetric > 0) && "Should handle dense range earlier!");
02434   } else {
02435     Pivot = CR.Range.first + Size/2;
02436   }
02437 
02438   CaseRange LHSR(CR.Range.first, Pivot);
02439   CaseRange RHSR(Pivot, CR.Range.second);
02440   const Constant *C = Pivot->Low;
02441   MachineBasicBlock *FalseBB = nullptr, *TrueBB = nullptr;
02442 
02443   // We know that we branch to the LHS if the Value being switched on is
02444   // less than the Pivot value, C.  We use this to optimize our binary
02445   // tree a bit, by recognizing that if SV is greater than or equal to the
02446   // LHS's Case Value, and that Case Value is exactly one less than the
02447   // Pivot's Value, then we can branch directly to the LHS's Target,
02448   // rather than creating a leaf node for it.
02449   if ((LHSR.second - LHSR.first) == 1 &&
02450       LHSR.first->High == CR.GE &&
02451       cast<ConstantInt>(C)->getValue() ==
02452       (cast<ConstantInt>(CR.GE)->getValue() + 1LL)) {
02453     TrueBB = LHSR.first->BB;
02454   } else {
02455     TrueBB = CurMF->CreateMachineBasicBlock(LLVMBB);
02456     CurMF->insert(BBI, TrueBB);
02457     WorkList.push_back(CaseRec(TrueBB, C, CR.GE, LHSR));
02458 
02459     // Put SV in a virtual register to make it available from the new blocks.
02460     ExportFromCurrentBlock(SV);
02461   }
02462 
02463   // Similar to the optimization above, if the Value being switched on is
02464   // known to be less than the Constant CR.LT, and the current Case Value
02465   // is CR.LT - 1, then we can branch directly to the target block for
02466   // the current Case Value, rather than emitting a RHS leaf node for it.
02467   if ((RHSR.second - RHSR.first) == 1 && CR.LT &&
02468       cast<ConstantInt>(RHSR.first->Low)->getValue() ==
02469       (cast<ConstantInt>(CR.LT)->getValue() - 1LL)) {
02470     FalseBB = RHSR.first->BB;
02471   } else {
02472     FalseBB = CurMF->CreateMachineBasicBlock(LLVMBB);
02473     CurMF->insert(BBI, FalseBB);
02474     WorkList.push_back(CaseRec(FalseBB,CR.LT,C,RHSR));
02475 
02476     // Put SV in a virtual register to make it available from the new blocks.
02477     ExportFromCurrentBlock(SV);
02478   }
02479 
02480   // Create a CaseBlock record representing a conditional branch to
02481   // the LHS node if the value being switched on SV is less than C.
02482   // Otherwise, branch to LHS.
02483   CaseBlock CB(ISD::SETLT, SV, C, nullptr, TrueBB, FalseBB, CR.CaseBB);
02484 
02485   if (CR.CaseBB == SwitchBB)
02486     visitSwitchCase(CB, SwitchBB);
02487   else
02488     SwitchCases.push_back(CB);
02489 
02490   return true;
02491 }
02492 
02493 /// handleBitTestsSwitchCase - if current case range has few destination and
02494 /// range span less, than machine word bitwidth, encode case range into series
02495 /// of masks and emit bit tests with these masks.
02496 bool SelectionDAGBuilder::handleBitTestsSwitchCase(CaseRec& CR,
02497                                                    CaseRecVector& WorkList,
02498                                                    const Value* SV,
02499                                                    MachineBasicBlock* Default,
02500                                                    MachineBasicBlock* SwitchBB) {
02501   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
02502   EVT PTy = TLI.getPointerTy();
02503   unsigned IntPtrBits = PTy.getSizeInBits();
02504 
02505   Case& FrontCase = *CR.Range.first;
02506   Case& BackCase  = *(CR.Range.second-1);
02507 
02508   // Get the MachineFunction which holds the current MBB.  This is used when
02509   // inserting any additional MBBs necessary to represent the switch.
02510   MachineFunction *CurMF = FuncInfo.MF;
02511 
02512   // If target does not have legal shift left, do not emit bit tests at all.
02513   if (!TLI.isOperationLegal(ISD::SHL, PTy))
02514     return false;
02515 
02516   size_t numCmps = 0;
02517   for (CaseItr I = CR.Range.first, E = CR.Range.second;
02518        I!=E; ++I) {
02519     // Single case counts one, case range - two.
02520     numCmps += (I->Low == I->High ? 1 : 2);
02521   }
02522 
02523   // Count unique destinations
02524   SmallSet<MachineBasicBlock*, 4> Dests;
02525   for (CaseItr I = CR.Range.first, E = CR.Range.second; I!=E; ++I) {
02526     Dests.insert(I->BB);
02527     if (Dests.size() > 3)
02528       // Don't bother the code below, if there are too much unique destinations
02529       return false;
02530   }
02531   DEBUG(dbgs() << "Total number of unique destinations: "
02532         << Dests.size() << '\n'
02533         << "Total number of comparisons: " << numCmps << '\n');
02534 
02535   // Compute span of values.
02536   const APInt& minValue = cast<ConstantInt>(FrontCase.Low)->getValue();
02537   const APInt& maxValue = cast<ConstantInt>(BackCase.High)->getValue();
02538   APInt cmpRange = maxValue - minValue;
02539 
02540   DEBUG(dbgs() << "Compare range: " << cmpRange << '\n'
02541                << "Low bound: " << minValue << '\n'
02542                << "High bound: " << maxValue << '\n');
02543 
02544   if (cmpRange.uge(IntPtrBits) ||
02545       (!(Dests.size() == 1 && numCmps >= 3) &&
02546        !(Dests.size() == 2 && numCmps >= 5) &&
02547        !(Dests.size() >= 3 && numCmps >= 6)))
02548     return false;
02549 
02550   DEBUG(dbgs() << "Emitting bit tests\n");
02551   APInt lowBound = APInt::getNullValue(cmpRange.getBitWidth());
02552 
02553   // Optimize the case where all the case values fit in a
02554   // word without having to subtract minValue. In this case,
02555   // we can optimize away the subtraction.
02556   if (minValue.isNonNegative() && maxValue.slt(IntPtrBits)) {
02557     cmpRange = maxValue;
02558   } else {
02559     lowBound = minValue;
02560   }
02561 
02562   CaseBitsVector CasesBits;
02563   unsigned i, count = 0;
02564 
02565   for (CaseItr I = CR.Range.first, E = CR.Range.second; I!=E; ++I) {
02566     MachineBasicBlock* Dest = I->BB;
02567     for (i = 0; i < count; ++i)
02568       if (Dest == CasesBits[i].BB)
02569         break;
02570 
02571     if (i == count) {
02572       assert((count < 3) && "Too much destinations to test!");
02573       CasesBits.push_back(CaseBits(0, Dest, 0, 0/*Weight*/));
02574       count++;
02575     }
02576 
02577     const APInt& lowValue = cast<ConstantInt>(I->Low)->getValue();
02578     const APInt& highValue = cast<ConstantInt>(I->High)->getValue();
02579 
02580     uint64_t lo = (lowValue - lowBound).getZExtValue();
02581     uint64_t hi = (highValue - lowBound).getZExtValue();
02582     CasesBits[i].ExtraWeight += I->ExtraWeight;
02583 
02584     for (uint64_t j = lo; j <= hi; j++) {
02585       CasesBits[i].Mask |=  1ULL << j;
02586       CasesBits[i].Bits++;
02587     }
02588 
02589   }
02590   std::sort(CasesBits.begin(), CasesBits.end(), CaseBitsCmp());
02591 
02592   BitTestInfo BTC;
02593 
02594   // Figure out which block is immediately after the current one.
02595   MachineFunction::iterator BBI = CR.CaseBB;
02596   ++BBI;
02597 
02598   const BasicBlock *LLVMBB = CR.CaseBB->getBasicBlock();
02599 
02600   DEBUG(dbgs() << "Cases:\n");
02601   for (unsigned i = 0, e = CasesBits.size(); i!=e; ++i) {
02602     DEBUG(dbgs() << "Mask: " << CasesBits[i].Mask
02603                  << ", Bits: " << CasesBits[i].Bits
02604                  << ", BB: " << CasesBits[i].BB << '\n');
02605 
02606     MachineBasicBlock *CaseBB = CurMF->CreateMachineBasicBlock(LLVMBB);
02607     CurMF->insert(BBI, CaseBB);
02608     BTC.push_back(BitTestCase(CasesBits[i].Mask,
02609                               CaseBB,
02610                               CasesBits[i].BB, CasesBits[i].ExtraWeight));
02611 
02612     // Put SV in a virtual register to make it available from the new blocks.
02613     ExportFromCurrentBlock(SV);
02614   }
02615 
02616   BitTestBlock BTB(lowBound, cmpRange, SV,
02617                    -1U, MVT::Other, (CR.CaseBB == SwitchBB),
02618                    CR.CaseBB, Default, std::move(BTC));
02619 
02620   if (CR.CaseBB == SwitchBB)
02621     visitBitTestHeader(BTB, SwitchBB);
02622 
02623   BitTestCases.push_back(std::move(BTB));
02624 
02625   return true;
02626 }
02627 
02628 /// Clusterify - Transform simple list of Cases into list of CaseRange's
02629 void SelectionDAGBuilder::Clusterify(CaseVector& Cases,
02630                                      const SwitchInst& SI) {
02631   BranchProbabilityInfo *BPI = FuncInfo.BPI;
02632   // Start with "simple" cases
02633   for (SwitchInst::ConstCaseIt i = SI.case_begin(), e = SI.case_end();
02634        i != e; ++i) {
02635     const BasicBlock *SuccBB = i.getCaseSuccessor();
02636     MachineBasicBlock *SMBB = FuncInfo.MBBMap[SuccBB];
02637 
02638     uint32_t ExtraWeight =
02639       BPI ? BPI->getEdgeWeight(SI.getParent(), i.getSuccessorIndex()) : 0;
02640 
02641     Cases.push_back(Case(i.getCaseValue(), i.getCaseValue(),
02642                          SMBB, ExtraWeight));
02643   }
02644   std::sort(Cases.begin(), Cases.end(), CaseCmp());
02645 
02646   // Merge case into clusters
02647   if (Cases.size() >= 2)
02648     // Must recompute end() each iteration because it may be
02649     // invalidated by erase if we hold on to it
02650     for (CaseItr I = Cases.begin(), J = std::next(Cases.begin());
02651          J != Cases.end(); ) {
02652       const APInt& nextValue = cast<ConstantInt>(J->Low)->getValue();
02653       const APInt& currentValue = cast<ConstantInt>(I->High)->getValue();
02654       MachineBasicBlock* nextBB = J->BB;
02655       MachineBasicBlock* currentBB = I->BB;
02656 
02657       // If the two neighboring cases go to the same destination, merge them
02658       // into a single case.
02659       if ((nextValue - currentValue == 1) && (currentBB == nextBB)) {
02660         I->High = J->High;
02661         I->ExtraWeight += J->ExtraWeight;
02662         J = Cases.erase(J);
02663       } else {
02664         I = J++;
02665       }
02666     }
02667 
02668   DEBUG({
02669       size_t numCmps = 0;
02670       for (auto &I : Cases)
02671         // A range counts double, since it requires two compares.
02672         numCmps += I.Low != I.High ? 2 : 1;
02673 
02674       dbgs() << "Clusterify finished. Total clusters: " << Cases.size()
02675              << ". Total compares: " << numCmps << '\n';
02676     });
02677 }
02678 
02679 void SelectionDAGBuilder::UpdateSplitBlock(MachineBasicBlock *First,
02680                                            MachineBasicBlock *Last) {
02681   // Update JTCases.
02682   for (unsigned i = 0, e = JTCases.size(); i != e; ++i)
02683     if (JTCases[i].first.HeaderBB == First)
02684       JTCases[i].first.HeaderBB = Last;
02685 
02686   // Update BitTestCases.
02687   for (unsigned i = 0, e = BitTestCases.size(); i != e; ++i)
02688     if (BitTestCases[i].Parent == First)
02689       BitTestCases[i].Parent = Last;
02690 }
02691 
02692 void SelectionDAGBuilder::visitSwitch(const SwitchInst &SI) {
02693   MachineBasicBlock *SwitchMBB = FuncInfo.MBB;
02694 
02695   // Figure out which block is immediately after the current one.
02696   MachineBasicBlock *NextBlock = nullptr;
02697   MachineBasicBlock *Default = FuncInfo.MBBMap[SI.getDefaultDest()];
02698 
02699   // If there is only the default destination, branch to it if it is not the
02700   // next basic block.  Otherwise, just fall through.
02701   if (!SI.getNumCases()) {
02702     // Update machine-CFG edges.
02703 
02704     // If this is not a fall-through branch, emit the branch.
02705     SwitchMBB->addSuccessor(Default);
02706     if (Default != NextBlock)
02707       DAG.setRoot(DAG.getNode(ISD::BR, getCurSDLoc(),
02708                               MVT::Other, getControlRoot(),
02709                               DAG.getBasicBlock(Default)));
02710 
02711     return;
02712   }
02713 
02714   // If there are any non-default case statements, create a vector of Cases
02715   // representing each one, and sort the vector so that we can efficiently
02716   // create a binary search tree from them.
02717   CaseVector Cases;
02718   Clusterify(Cases, SI);
02719 
02720   // Get the Value to be switched on and default basic blocks, which will be
02721   // inserted into CaseBlock records, representing basic blocks in the binary
02722   // search tree.
02723   const Value *SV = SI.getCondition();
02724 
02725   // Push the initial CaseRec onto the worklist
02726   CaseRecVector WorkList;
02727   WorkList.push_back(CaseRec(SwitchMBB,nullptr,nullptr,
02728                              CaseRange(Cases.begin(),Cases.end())));
02729 
02730   while (!WorkList.empty()) {
02731     // Grab a record representing a case range to process off the worklist
02732     CaseRec CR = WorkList.back();
02733     WorkList.pop_back();
02734 
02735     if (handleBitTestsSwitchCase(CR, WorkList, SV, Default, SwitchMBB))
02736       continue;
02737 
02738     // If the range has few cases (two or less) emit a series of specific
02739     // tests.
02740     if (handleSmallSwitchRange(CR, WorkList, SV, Default, SwitchMBB))
02741       continue;
02742 
02743     // If the switch has more than N blocks, and is at least 40% dense, and the
02744     // target supports indirect branches, then emit a jump table rather than
02745     // lowering the switch to a binary tree of conditional branches.
02746     // N defaults to 4 and is controlled via TLS.getMinimumJumpTableEntries().
02747     if (handleJTSwitchCase(CR, WorkList, SV, Default, SwitchMBB))
02748       continue;
02749 
02750     // Emit binary tree. We need to pick a pivot, and push left and right ranges
02751     // onto the worklist. Leafs are handled via handleSmallSwitchRange() call.
02752     handleBTSplitSwitchCase(CR, WorkList, SV, SwitchMBB);
02753   }
02754 }
02755 
02756 void SelectionDAGBuilder::visitIndirectBr(const IndirectBrInst &I) {
02757   MachineBasicBlock *IndirectBrMBB = FuncInfo.MBB;
02758 
02759   // Update machine-CFG edges with unique successors.
02760   SmallSet<BasicBlock*, 32> Done;
02761   for (unsigned i = 0, e = I.getNumSuccessors(); i != e; ++i) {
02762     BasicBlock *BB = I.getSuccessor(i);
02763     bool Inserted = Done.insert(BB).second;
02764     if (!Inserted)
02765         continue;
02766 
02767     MachineBasicBlock *Succ = FuncInfo.MBBMap[BB];
02768     addSuccessorWithWeight(IndirectBrMBB, Succ);
02769   }
02770 
02771   DAG.setRoot(DAG.getNode(ISD::BRIND, getCurSDLoc(),
02772                           MVT::Other, getControlRoot(),
02773                           getValue(I.getAddress())));
02774 }
02775 
02776 void SelectionDAGBuilder::visitUnreachable(const UnreachableInst &I) {
02777   if (DAG.getTarget().Options.TrapUnreachable)
02778     DAG.setRoot(DAG.getNode(ISD::TRAP, getCurSDLoc(), MVT::Other, DAG.getRoot()));
02779 }
02780 
02781 void SelectionDAGBuilder::visitFSub(const User &I) {
02782   // -0.0 - X --> fneg
02783   Type *Ty = I.getType();
02784   if (isa<Constant>(I.getOperand(0)) &&
02785       I.getOperand(0) == ConstantFP::getZeroValueForNegation(Ty)) {
02786     SDValue Op2 = getValue(I.getOperand(1));
02787     setValue(&I, DAG.getNode(ISD::FNEG, getCurSDLoc(),
02788                              Op2.getValueType(), Op2));
02789     return;
02790   }
02791 
02792   visitBinary(I, ISD::FSUB);
02793 }
02794 
02795 void SelectionDAGBuilder::visitBinary(const User &I, unsigned OpCode) {
02796   SDValue Op1 = getValue(I.getOperand(0));
02797   SDValue Op2 = getValue(I.getOperand(1));
02798 
02799   bool nuw = false;
02800   bool nsw = false;
02801   bool exact = false;
02802   if (const OverflowingBinaryOperator *OFBinOp =
02803           dyn_cast<const OverflowingBinaryOperator>(&I)) {
02804     nuw = OFBinOp->hasNoUnsignedWrap();
02805     nsw = OFBinOp->hasNoSignedWrap();
02806   }
02807   if (const PossiblyExactOperator *ExactOp =
02808           dyn_cast<const PossiblyExactOperator>(&I))
02809     exact = ExactOp->isExact();
02810 
02811   SDValue BinNodeValue = DAG.getNode(OpCode, getCurSDLoc(), Op1.getValueType(),
02812                                      Op1, Op2, nuw, nsw, exact);
02813   setValue(&I, BinNodeValue);
02814 }
02815 
02816 void SelectionDAGBuilder::visitShift(const User &I, unsigned Opcode) {
02817   SDValue Op1 = getValue(I.getOperand(0));
02818   SDValue Op2 = getValue(I.getOperand(1));
02819 
02820   EVT ShiftTy =
02821       DAG.getTargetLoweringInfo().getShiftAmountTy(Op2.getValueType());
02822 
02823   // Coerce the shift amount to the right type if we can.
02824   if (!I.getType()->isVectorTy() && Op2.getValueType() != ShiftTy) {
02825     unsigned ShiftSize = ShiftTy.getSizeInBits();
02826     unsigned Op2Size = Op2.getValueType().getSizeInBits();
02827     SDLoc DL = getCurSDLoc();
02828 
02829     // If the operand is smaller than the shift count type, promote it.
02830     if (ShiftSize > Op2Size)
02831       Op2 = DAG.getNode(ISD::ZERO_EXTEND, DL, ShiftTy, Op2);
02832 
02833     // If the operand is larger than the shift count type but the shift
02834     // count type has enough bits to represent any shift value, truncate
02835     // it now. This is a common case and it exposes the truncate to
02836     // optimization early.
02837     else if (ShiftSize >= Log2_32_Ceil(Op2.getValueType().getSizeInBits()))
02838       Op2 = DAG.getNode(ISD::TRUNCATE, DL, ShiftTy, Op2);
02839     // Otherwise we'll need to temporarily settle for some other convenient
02840     // type.  Type legalization will make adjustments once the shiftee is split.
02841     else
02842       Op2 = DAG.getZExtOrTrunc(Op2, DL, MVT::i32);
02843   }
02844 
02845   bool nuw = false;
02846   bool nsw = false;
02847   bool exact = false;
02848 
02849   if (Opcode == ISD::SRL || Opcode == ISD::SRA || Opcode == ISD::SHL) {
02850 
02851     if (const OverflowingBinaryOperator *OFBinOp =
02852             dyn_cast<const OverflowingBinaryOperator>(&I)) {
02853       nuw = OFBinOp->hasNoUnsignedWrap();
02854       nsw = OFBinOp->hasNoSignedWrap();
02855     }
02856     if (const PossiblyExactOperator *ExactOp =
02857             dyn_cast<const PossiblyExactOperator>(&I))
02858       exact = ExactOp->isExact();
02859   }
02860 
02861   SDValue Res = DAG.getNode(Opcode, getCurSDLoc(), Op1.getValueType(), Op1, Op2,
02862                             nuw, nsw, exact);
02863   setValue(&I, Res);
02864 }
02865 
02866 void SelectionDAGBuilder::visitSDiv(const User &I) {
02867   SDValue Op1 = getValue(I.getOperand(0));
02868   SDValue Op2 = getValue(I.getOperand(1));
02869 
02870   // Turn exact SDivs into multiplications.
02871   // FIXME: This should be in DAGCombiner, but it doesn't have access to the
02872   // exact bit.
02873   if (isa<BinaryOperator>(&I) && cast<BinaryOperator>(&I)->isExact() &&
02874       !isa<ConstantSDNode>(Op1) &&
02875       isa<ConstantSDNode>(Op2) && !cast<ConstantSDNode>(Op2)->isNullValue())
02876     setValue(&I, DAG.getTargetLoweringInfo()
02877                      .BuildExactSDIV(Op1, Op2, getCurSDLoc(), DAG));
02878   else
02879     setValue(&I, DAG.getNode(ISD::SDIV, getCurSDLoc(), Op1.getValueType(),
02880                              Op1, Op2));
02881 }
02882 
02883 void SelectionDAGBuilder::visitICmp(const User &I) {
02884   ICmpInst::Predicate predicate = ICmpInst::BAD_ICMP_PREDICATE;
02885   if (const ICmpInst *IC = dyn_cast<ICmpInst>(&I))
02886     predicate = IC->getPredicate();
02887   else if (const ConstantExpr *IC = dyn_cast<ConstantExpr>(&I))
02888     predicate = ICmpInst::Predicate(IC->getPredicate());
02889   SDValue Op1 = getValue(I.getOperand(0));
02890   SDValue Op2 = getValue(I.getOperand(1));
02891   ISD::CondCode Opcode = getICmpCondCode(predicate);
02892 
02893   EVT DestVT = DAG.getTargetLoweringInfo().getValueType(I.getType());
02894   setValue(&I, DAG.getSetCC(getCurSDLoc(), DestVT, Op1, Op2, Opcode));
02895 }
02896 
02897 void SelectionDAGBuilder::visitFCmp(const User &I) {
02898   FCmpInst::Predicate predicate = FCmpInst::BAD_FCMP_PREDICATE;
02899   if (const FCmpInst *FC = dyn_cast<FCmpInst>(&I))
02900     predicate = FC->getPredicate();
02901   else if (const ConstantExpr *FC = dyn_cast<ConstantExpr>(&I))
02902     predicate = FCmpInst::Predicate(FC->getPredicate());
02903   SDValue Op1 = getValue(I.getOperand(0));
02904   SDValue Op2 = getValue(I.getOperand(1));
02905   ISD::CondCode Condition = getFCmpCondCode(predicate);
02906   if (TM.Options.NoNaNsFPMath)
02907     Condition = getFCmpCodeWithoutNaN(Condition);
02908   EVT DestVT = DAG.getTargetLoweringInfo().getValueType(I.getType());
02909   setValue(&I, DAG.getSetCC(getCurSDLoc(), DestVT, Op1, Op2, Condition));
02910 }
02911 
02912 void SelectionDAGBuilder::visitSelect(const User &I) {
02913   SmallVector<EVT, 4> ValueVTs;
02914   ComputeValueVTs(DAG.getTargetLoweringInfo(), I.getType(), ValueVTs);
02915   unsigned NumValues = ValueVTs.size();
02916   if (NumValues == 0) return;
02917 
02918   SmallVector<SDValue, 4> Values(NumValues);
02919   SDValue Cond     = getValue(I.getOperand(0));
02920   SDValue TrueVal  = getValue(I.getOperand(1));
02921   SDValue FalseVal = getValue(I.getOperand(2));
02922   ISD::NodeType OpCode = Cond.getValueType().isVector() ?
02923     ISD::VSELECT : ISD::SELECT;
02924 
02925   for (unsigned i = 0; i != NumValues; ++i)
02926     Values[i] = DAG.getNode(OpCode, getCurSDLoc(),
02927                             TrueVal.getNode()->getValueType(TrueVal.getResNo()+i),
02928                             Cond,
02929                             SDValue(TrueVal.getNode(),
02930                                     TrueVal.getResNo() + i),
02931                             SDValue(FalseVal.getNode(),
02932                                     FalseVal.getResNo() + i));
02933 
02934   setValue(&I, DAG.getNode(ISD::MERGE_VALUES, getCurSDLoc(),
02935                            DAG.getVTList(ValueVTs), Values));
02936 }
02937 
02938 void SelectionDAGBuilder::visitTrunc(const User &I) {
02939   // TruncInst cannot be a no-op cast because sizeof(src) > sizeof(dest).
02940   SDValue N = getValue(I.getOperand(0));
02941   EVT DestVT = DAG.getTargetLoweringInfo().getValueType(I.getType());
02942   setValue(&I, DAG.getNode(ISD::TRUNCATE, getCurSDLoc(), DestVT, N));
02943 }
02944 
02945 void SelectionDAGBuilder::visitZExt(const User &I) {
02946   // ZExt cannot be a no-op cast because sizeof(src) < sizeof(dest).
02947   // ZExt also can't be a cast to bool for same reason. So, nothing much to do
02948   SDValue N = getValue(I.getOperand(0));
02949   EVT DestVT = DAG.getTargetLoweringInfo().getValueType(I.getType());
02950   setValue(&I, DAG.getNode(ISD::ZERO_EXTEND, getCurSDLoc(), DestVT, N));
02951 }
02952 
02953 void SelectionDAGBuilder::visitSExt(const User &I) {
02954   // SExt cannot be a no-op cast because sizeof(src) < sizeof(dest).
02955   // SExt also can't be a cast to bool for same reason. So, nothing much to do
02956   SDValue N = getValue(I.getOperand(0));
02957   EVT DestVT = DAG.getTargetLoweringInfo().getValueType(I.getType());
02958   setValue(&I, DAG.getNode(ISD::SIGN_EXTEND, getCurSDLoc(), DestVT, N));
02959 }
02960 
02961 void SelectionDAGBuilder::visitFPTrunc(const User &I) {
02962   // FPTrunc is never a no-op cast, no need to check
02963   SDValue N = getValue(I.getOperand(0));
02964   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
02965   EVT DestVT = TLI.getValueType(I.getType());
02966   setValue(&I, DAG.getNode(ISD::FP_ROUND, getCurSDLoc(), DestVT, N,
02967                            DAG.getTargetConstant(0, TLI.getPointerTy())));
02968 }
02969 
02970 void SelectionDAGBuilder::visitFPExt(const User &I) {
02971   // FPExt is never a no-op cast, no need to check
02972   SDValue N = getValue(I.getOperand(0));
02973   EVT DestVT = DAG.getTargetLoweringInfo().getValueType(I.getType());
02974   setValue(&I, DAG.getNode(ISD::FP_EXTEND, getCurSDLoc(), DestVT, N));
02975 }
02976 
02977 void SelectionDAGBuilder::visitFPToUI(const User &I) {
02978   // FPToUI is never a no-op cast, no need to check
02979   SDValue N = getValue(I.getOperand(0));
02980   EVT DestVT = DAG.getTargetLoweringInfo().getValueType(I.getType());
02981   setValue(&I, DAG.getNode(ISD::FP_TO_UINT, getCurSDLoc(), DestVT, N));
02982 }
02983 
02984 void SelectionDAGBuilder::visitFPToSI(const User &I) {
02985   // FPToSI is never a no-op cast, no need to check
02986   SDValue N = getValue(I.getOperand(0));
02987   EVT DestVT = DAG.getTargetLoweringInfo().getValueType(I.getType());
02988   setValue(&I, DAG.getNode(ISD::FP_TO_SINT, getCurSDLoc(), DestVT, N));
02989 }
02990 
02991 void SelectionDAGBuilder::visitUIToFP(const User &I) {
02992   // UIToFP is never a no-op cast, no need to check
02993   SDValue N = getValue(I.getOperand(0));
02994   EVT DestVT = DAG.getTargetLoweringInfo().getValueType(I.getType());
02995   setValue(&I, DAG.getNode(ISD::UINT_TO_FP, getCurSDLoc(), DestVT, N));
02996 }
02997 
02998 void SelectionDAGBuilder::visitSIToFP(const User &I) {
02999   // SIToFP is never a no-op cast, no need to check
03000   SDValue N = getValue(I.getOperand(0));
03001   EVT DestVT = DAG.getTargetLoweringInfo().getValueType(I.getType());
03002   setValue(&I, DAG.getNode(ISD::SINT_TO_FP, getCurSDLoc(), DestVT, N));
03003 }
03004 
03005 void SelectionDAGBuilder::visitPtrToInt(const User &I) {
03006   // What to do depends on the size of the integer and the size of the pointer.
03007   // We can either truncate, zero extend, or no-op, accordingly.
03008   SDValue N = getValue(I.getOperand(0));
03009   EVT DestVT = DAG.getTargetLoweringInfo().getValueType(I.getType());
03010   setValue(&I, DAG.getZExtOrTrunc(N, getCurSDLoc(), DestVT));
03011 }
03012 
03013 void SelectionDAGBuilder::visitIntToPtr(const User &I) {
03014   // What to do depends on the size of the integer and the size of the pointer.
03015   // We can either truncate, zero extend, or no-op, accordingly.
03016   SDValue N = getValue(I.getOperand(0));
03017   EVT DestVT = DAG.getTargetLoweringInfo().getValueType(I.getType());
03018   setValue(&I, DAG.getZExtOrTrunc(N, getCurSDLoc(), DestVT));
03019 }
03020 
03021 void SelectionDAGBuilder::visitBitCast(const User &I) {
03022   SDValue N = getValue(I.getOperand(0));
03023   EVT DestVT = DAG.getTargetLoweringInfo().getValueType(I.getType());
03024 
03025   // BitCast assures us that source and destination are the same size so this is
03026   // either a BITCAST or a no-op.
03027   if (DestVT != N.getValueType())
03028     setValue(&I, DAG.getNode(ISD::BITCAST, getCurSDLoc(),
03029                              DestVT, N)); // convert types.
03030   // Check if the original LLVM IR Operand was a ConstantInt, because getValue()
03031   // might fold any kind of constant expression to an integer constant and that
03032   // is not what we are looking for. Only regcognize a bitcast of a genuine
03033   // constant integer as an opaque constant.
03034   else if(ConstantInt *C = dyn_cast<ConstantInt>(I.getOperand(0)))
03035     setValue(&I, DAG.getConstant(C->getValue(), DestVT, /*isTarget=*/false,
03036                                  /*isOpaque*/true));
03037   else
03038     setValue(&I, N);            // noop cast.
03039 }
03040 
03041 void SelectionDAGBuilder::visitAddrSpaceCast(const User &I) {
03042   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
03043   const Value *SV = I.getOperand(0);
03044   SDValue N = getValue(SV);
03045   EVT DestVT = TLI.getValueType(I.getType());
03046 
03047   unsigned SrcAS = SV->getType()->getPointerAddressSpace();
03048   unsigned DestAS = I.getType()->getPointerAddressSpace();
03049 
03050   if (!TLI.isNoopAddrSpaceCast(SrcAS, DestAS))
03051     N = DAG.getAddrSpaceCast(getCurSDLoc(), DestVT, N, SrcAS, DestAS);
03052 
03053   setValue(&I, N);
03054 }
03055 
03056 void SelectionDAGBuilder::visitInsertElement(const User &I) {
03057   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
03058   SDValue InVec = getValue(I.getOperand(0));
03059   SDValue InVal = getValue(I.getOperand(1));
03060   SDValue InIdx = DAG.getSExtOrTrunc(getValue(I.getOperand(2)),
03061                                      getCurSDLoc(), TLI.getVectorIdxTy());
03062   setValue(&I, DAG.getNode(ISD::INSERT_VECTOR_ELT, getCurSDLoc(),
03063                            TLI.getValueType(I.getType()), InVec, InVal, InIdx));
03064 }
03065 
03066 void SelectionDAGBuilder::visitExtractElement(const User &I) {
03067   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
03068   SDValue InVec = getValue(I.getOperand(0));
03069   SDValue InIdx = DAG.getSExtOrTrunc(getValue(I.getOperand(1)),
03070                                      getCurSDLoc(), TLI.getVectorIdxTy());
03071   setValue(&I, DAG.getNode(ISD::EXTRACT_VECTOR_ELT, getCurSDLoc(),
03072                            TLI.getValueType(I.getType()), InVec, InIdx));
03073 }
03074 
03075 // Utility for visitShuffleVector - Return true if every element in Mask,
03076 // beginning from position Pos and ending in Pos+Size, falls within the
03077 // specified sequential range [L, L+Pos). or is undef.
03078 static bool isSequentialInRange(const SmallVectorImpl<int> &Mask,
03079                                 unsigned Pos, unsigned Size, int Low) {
03080   for (unsigned i = Pos, e = Pos+Size; i != e; ++i, ++Low)
03081     if (Mask[i] >= 0 && Mask[i] != Low)
03082       return false;
03083   return true;
03084 }
03085 
03086 void SelectionDAGBuilder::visitShuffleVector(const User &I) {
03087   SDValue Src1 = getValue(I.getOperand(0));
03088   SDValue Src2 = getValue(I.getOperand(1));
03089 
03090   SmallVector<int, 8> Mask;
03091   ShuffleVectorInst::getShuffleMask(cast<Constant>(I.getOperand(2)), Mask);
03092   unsigned MaskNumElts = Mask.size();
03093 
03094   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
03095   EVT VT = TLI.getValueType(I.getType());
03096   EVT SrcVT = Src1.getValueType();
03097   unsigned SrcNumElts = SrcVT.getVectorNumElements();
03098 
03099   if (SrcNumElts == MaskNumElts) {
03100     setValue(&I, DAG.getVectorShuffle(VT, getCurSDLoc(), Src1, Src2,
03101                                       &Mask[0]));
03102     return;
03103   }
03104 
03105   // Normalize the shuffle vector since mask and vector length don't match.
03106   if (SrcNumElts < MaskNumElts && MaskNumElts % SrcNumElts == 0) {
03107     // Mask is longer than the source vectors and is a multiple of the source
03108     // vectors.  We can use concatenate vector to make the mask and vectors
03109     // lengths match.
03110     if (SrcNumElts*2 == MaskNumElts) {
03111       // First check for Src1 in low and Src2 in high
03112       if (isSequentialInRange(Mask, 0, SrcNumElts, 0) &&
03113           isSequentialInRange(Mask, SrcNumElts, SrcNumElts, SrcNumElts)) {
03114         // The shuffle is concatenating two vectors together.
03115         setValue(&I, DAG.getNode(ISD::CONCAT_VECTORS, getCurSDLoc(),
03116                                  VT, Src1, Src2));
03117         return;
03118       }
03119       // Then check for Src2 in low and Src1 in high
03120       if (isSequentialInRange(Mask, 0, SrcNumElts, SrcNumElts) &&
03121           isSequentialInRange(Mask, SrcNumElts, SrcNumElts, 0)) {
03122         // The shuffle is concatenating two vectors together.
03123         setValue(&I, DAG.getNode(ISD::CONCAT_VECTORS, getCurSDLoc(),
03124                                  VT, Src2, Src1));
03125         return;
03126       }
03127     }
03128 
03129     // Pad both vectors with undefs to make them the same length as the mask.
03130     unsigned NumConcat = MaskNumElts / SrcNumElts;
03131     bool Src1U = Src1.getOpcode() == ISD::UNDEF;
03132     bool Src2U = Src2.getOpcode() == ISD::UNDEF;
03133     SDValue UndefVal = DAG.getUNDEF(SrcVT);
03134 
03135     SmallVector<SDValue, 8> MOps1(NumConcat, UndefVal);
03136     SmallVector<SDValue, 8> MOps2(NumConcat, UndefVal);
03137     MOps1[0] = Src1;
03138     MOps2[0] = Src2;
03139 
03140     Src1 = Src1U ? DAG.getUNDEF(VT) : DAG.getNode(ISD::CONCAT_VECTORS,
03141                                                   getCurSDLoc(), VT, MOps1);
03142     Src2 = Src2U ? DAG.getUNDEF(VT) : DAG.getNode(ISD::CONCAT_VECTORS,
03143                                                   getCurSDLoc(), VT, MOps2);
03144 
03145     // Readjust mask for new input vector length.
03146     SmallVector<int, 8> MappedOps;
03147     for (unsigned i = 0; i != MaskNumElts; ++i) {
03148       int Idx = Mask[i];
03149       if (Idx >= (int)SrcNumElts)
03150         Idx -= SrcNumElts - MaskNumElts;
03151       MappedOps.push_back(Idx);
03152     }
03153 
03154     setValue(&I, DAG.getVectorShuffle(VT, getCurSDLoc(), Src1, Src2,
03155                                       &MappedOps[0]));
03156     return;
03157   }
03158 
03159   if (SrcNumElts > MaskNumElts) {
03160     // Analyze the access pattern of the vector to see if we can extract
03161     // two subvectors and do the shuffle. The analysis is done by calculating
03162     // the range of elements the mask access on both vectors.
03163     int MinRange[2] = { static_cast<int>(SrcNumElts),
03164                         static_cast<int>(SrcNumElts)};
03165     int MaxRange[2] = {-1, -1};
03166 
03167     for (unsigned i = 0; i != MaskNumElts; ++i) {
03168       int Idx = Mask[i];
03169       unsigned Input = 0;
03170       if (Idx < 0)
03171         continue;
03172 
03173       if (Idx >= (int)SrcNumElts) {
03174         Input = 1;
03175         Idx -= SrcNumElts;
03176       }
03177       if (Idx > MaxRange[Input])
03178         MaxRange[Input] = Idx;
03179       if (Idx < MinRange[Input])
03180         MinRange[Input] = Idx;
03181     }
03182 
03183     // Check if the access is smaller than the vector size and can we find
03184     // a reasonable extract index.
03185     int RangeUse[2] = { -1, -1 };  // 0 = Unused, 1 = Extract, -1 = Can not
03186                                    // Extract.
03187     int StartIdx[2];  // StartIdx to extract from
03188     for (unsigned Input = 0; Input < 2; ++Input) {
03189       if (MinRange[Input] >= (int)SrcNumElts && MaxRange[Input] < 0) {
03190         RangeUse[Input] = 0; // Unused
03191         StartIdx[Input] = 0;
03192         continue;
03193       }
03194 
03195       // Find a good start index that is a multiple of the mask length. Then
03196       // see if the rest of the elements are in range.
03197       StartIdx[Input] = (MinRange[Input]/MaskNumElts)*MaskNumElts;
03198       if (MaxRange[Input] - StartIdx[Input] < (int)MaskNumElts &&
03199           StartIdx[Input] + MaskNumElts <= SrcNumElts)
03200         RangeUse[Input] = 1; // Extract from a multiple of the mask length.
03201     }
03202 
03203     if (RangeUse[0] == 0 && RangeUse[1] == 0) {
03204       setValue(&I, DAG.getUNDEF(VT)); // Vectors are not used.
03205       return;
03206     }
03207     if (RangeUse[0] >= 0 && RangeUse[1] >= 0) {
03208       // Extract appropriate subvector and generate a vector shuffle
03209       for (unsigned Input = 0; Input < 2; ++Input) {
03210         SDValue &Src = Input == 0 ? Src1 : Src2;
03211         if (RangeUse[Input] == 0)
03212           Src = DAG.getUNDEF(VT);
03213         else
03214           Src = DAG.getNode(
03215               ISD::EXTRACT_SUBVECTOR, getCurSDLoc(), VT, Src,
03216               DAG.getConstant(StartIdx[Input], TLI.getVectorIdxTy()));
03217       }
03218 
03219       // Calculate new mask.
03220       SmallVector<int, 8> MappedOps;
03221       for (unsigned i = 0; i != MaskNumElts; ++i) {
03222         int Idx = Mask[i];
03223         if (Idx >= 0) {
03224           if (Idx < (int)SrcNumElts)
03225             Idx -= StartIdx[0];
03226           else
03227             Idx -= SrcNumElts + StartIdx[1] - MaskNumElts;
03228         }
03229         MappedOps.push_back(Idx);
03230       }
03231 
03232       setValue(&I, DAG.getVectorShuffle(VT, getCurSDLoc(), Src1, Src2,
03233                                         &MappedOps[0]));
03234       return;
03235     }
03236   }
03237 
03238   // We can't use either concat vectors or extract subvectors so fall back to
03239   // replacing the shuffle with extract and build vector.
03240   // to insert and build vector.
03241   EVT EltVT = VT.getVectorElementType();
03242   EVT IdxVT = TLI.getVectorIdxTy();
03243   SmallVector<SDValue,8> Ops;
03244   for (unsigned i = 0; i != MaskNumElts; ++i) {
03245     int Idx = Mask[i];
03246     SDValue Res;
03247 
03248     if (Idx < 0) {
03249       Res = DAG.getUNDEF(EltVT);
03250     } else {
03251       SDValue &Src = Idx < (int)SrcNumElts ? Src1 : Src2;
03252       if (Idx >= (int)SrcNumElts) Idx -= SrcNumElts;
03253 
03254       Res = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, getCurSDLoc(),
03255                         EltVT, Src, DAG.getConstant(Idx, IdxVT));
03256     }
03257 
03258     Ops.push_back(Res);
03259   }
03260 
03261   setValue(&I, DAG.getNode(ISD::BUILD_VECTOR, getCurSDLoc(), VT, Ops));
03262 }
03263 
03264 void SelectionDAGBuilder::visitInsertValue(const InsertValueInst &I) {
03265   const Value *Op0 = I.getOperand(0);
03266   const Value *Op1 = I.getOperand(1);
03267   Type *AggTy = I.getType();
03268   Type *ValTy = Op1->getType();
03269   bool IntoUndef = isa<UndefValue>(Op0);
03270   bool FromUndef = isa<UndefValue>(Op1);
03271 
03272   unsigned LinearIndex = ComputeLinearIndex(AggTy, I.getIndices());
03273 
03274   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
03275   SmallVector<EVT, 4> AggValueVTs;
03276   ComputeValueVTs(TLI, AggTy, AggValueVTs);
03277   SmallVector<EVT, 4> ValValueVTs;
03278   ComputeValueVTs(TLI, ValTy, ValValueVTs);
03279 
03280   unsigned NumAggValues = AggValueVTs.size();
03281   unsigned NumValValues = ValValueVTs.size();
03282   SmallVector<SDValue, 4> Values(NumAggValues);
03283 
03284   // Ignore an insertvalue that produces an empty object
03285   if (!NumAggValues) {
03286     setValue(&I, DAG.getUNDEF(MVT(MVT::Other)));
03287     return;
03288   }
03289 
03290   SDValue Agg = getValue(Op0);
03291   unsigned i = 0;
03292   // Copy the beginning value(s) from the original aggregate.
03293   for (; i != LinearIndex; ++i)
03294     Values[i] = IntoUndef ? DAG.getUNDEF(AggValueVTs[i]) :
03295                 SDValue(Agg.getNode(), Agg.getResNo() + i);
03296   // Copy values from the inserted value(s).
03297   if (NumValValues) {
03298     SDValue Val = getValue(Op1);
03299     for (; i != LinearIndex + NumValValues; ++i)
03300       Values[i] = FromUndef ? DAG.getUNDEF(AggValueVTs[i]) :
03301                   SDValue(Val.getNode(), Val.getResNo() + i - LinearIndex);
03302   }
03303   // Copy remaining value(s) from the original aggregate.
03304   for (; i != NumAggValues; ++i)
03305     Values[i] = IntoUndef ? DAG.getUNDEF(AggValueVTs[i]) :
03306                 SDValue(Agg.getNode(), Agg.getResNo() + i);
03307 
03308   setValue(&I, DAG.getNode(ISD::MERGE_VALUES, getCurSDLoc(),
03309                            DAG.getVTList(AggValueVTs), Values));
03310 }
03311 
03312 void SelectionDAGBuilder::visitExtractValue(const ExtractValueInst &I) {
03313   const Value *Op0 = I.getOperand(0);
03314   Type *AggTy = Op0->getType();
03315   Type *ValTy = I.getType();
03316   bool OutOfUndef = isa<UndefValue>(Op0);
03317 
03318   unsigned LinearIndex = ComputeLinearIndex(AggTy, I.getIndices());
03319 
03320   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
03321   SmallVector<EVT, 4> ValValueVTs;
03322   ComputeValueVTs(TLI, ValTy, ValValueVTs);
03323 
03324   unsigned NumValValues = ValValueVTs.size();
03325 
03326   // Ignore a extractvalue that produces an empty object
03327   if (!NumValValues) {
03328     setValue(&I, DAG.getUNDEF(MVT(MVT::Other)));
03329     return;
03330   }
03331 
03332   SmallVector<SDValue, 4> Values(NumValValues);
03333 
03334   SDValue Agg = getValue(Op0);
03335   // Copy out the selected value(s).
03336   for (unsigned i = LinearIndex; i != LinearIndex + NumValValues; ++i)
03337     Values[i - LinearIndex] =
03338       OutOfUndef ?
03339         DAG.getUNDEF(Agg.getNode()->getValueType(Agg.getResNo() + i)) :
03340         SDValue(Agg.getNode(), Agg.getResNo() + i);
03341 
03342   setValue(&I, DAG.getNode(ISD::MERGE_VALUES, getCurSDLoc(),
03343                            DAG.getVTList(ValValueVTs), Values));
03344 }
03345 
03346 void SelectionDAGBuilder::visitGetElementPtr(const User &I) {
03347   Value *Op0 = I.getOperand(0);
03348   // Note that the pointer operand may be a vector of pointers. Take the scalar
03349   // element which holds a pointer.
03350   Type *Ty = Op0->getType()->getScalarType();
03351   unsigned AS = Ty->getPointerAddressSpace();
03352   SDValue N = getValue(Op0);
03353 
03354   for (GetElementPtrInst::const_op_iterator OI = I.op_begin()+1, E = I.op_end();
03355        OI != E; ++OI) {
03356     const Value *Idx = *OI;
03357     if (StructType *StTy = dyn_cast<StructType>(Ty)) {
03358       unsigned Field = cast<Constant>(Idx)->getUniqueInteger().getZExtValue();
03359       if (Field) {
03360         // N = N + Offset
03361         uint64_t Offset = DL->getStructLayout(StTy)->getElementOffset(Field);
03362         N = DAG.getNode(ISD::ADD, getCurSDLoc(), N.getValueType(), N,
03363                         DAG.getConstant(Offset, N.getValueType()));
03364       }
03365 
03366       Ty = StTy->getElementType(Field);
03367     } else {
03368       Ty = cast<SequentialType>(Ty)->getElementType();
03369 
03370       // If this is a constant subscript, handle it quickly.
03371       const TargetLowering &TLI = DAG.getTargetLoweringInfo();
03372       if (const ConstantInt *CI = dyn_cast<ConstantInt>(Idx)) {
03373         if (CI->isZero()) continue;
03374         uint64_t Offs =
03375             DL->getTypeAllocSize(Ty)*cast<ConstantInt>(CI)->getSExtValue();
03376         SDValue OffsVal;
03377         EVT PTy = TLI.getPointerTy(AS);
03378         unsigned PtrBits = PTy.getSizeInBits();
03379         if (PtrBits < 64)
03380           OffsVal = DAG.getNode(ISD::TRUNCATE, getCurSDLoc(), PTy,
03381                                 DAG.getConstant(Offs, MVT::i64));
03382         else
03383           OffsVal = DAG.getConstant(Offs, PTy);
03384 
03385         N = DAG.getNode(ISD::ADD, getCurSDLoc(), N.getValueType(), N,
03386                         OffsVal);
03387         continue;
03388       }
03389 
03390       // N = N + Idx * ElementSize;
03391       APInt ElementSize =
03392           APInt(TLI.getPointerSizeInBits(AS), DL->getTypeAllocSize(Ty));
03393       SDValue IdxN = getValue(Idx);
03394 
03395       // If the index is smaller or larger than intptr_t, truncate or extend
03396       // it.
03397       IdxN = DAG.getSExtOrTrunc(IdxN, getCurSDLoc(), N.getValueType());
03398 
03399       // If this is a multiply by a power of two, turn it into a shl
03400       // immediately.  This is a very common case.
03401       if (ElementSize != 1) {
03402         if (ElementSize.isPowerOf2()) {
03403           unsigned Amt = ElementSize.logBase2();
03404           IdxN = DAG.getNode(ISD::SHL, getCurSDLoc(),
03405                              N.getValueType(), IdxN,
03406                              DAG.getConstant(Amt, IdxN.getValueType()));
03407         } else {
03408           SDValue Scale = DAG.getConstant(ElementSize, IdxN.getValueType());
03409           IdxN = DAG.getNode(ISD::MUL, getCurSDLoc(),
03410                              N.getValueType(), IdxN, Scale);
03411         }
03412       }
03413 
03414       N = DAG.getNode(ISD::ADD, getCurSDLoc(),
03415                       N.getValueType(), N, IdxN);
03416     }
03417   }
03418 
03419   setValue(&I, N);
03420 }
03421 
03422 void SelectionDAGBuilder::visitAlloca(const AllocaInst &I) {
03423   // If this is a fixed sized alloca in the entry block of the function,
03424   // allocate it statically on the stack.
03425   if (FuncInfo.StaticAllocaMap.count(&I))
03426     return;   // getValue will auto-populate this.
03427 
03428   Type *Ty = I.getAllocatedType();
03429   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
03430   uint64_t TySize = TLI.getDataLayout()->getTypeAllocSize(Ty);
03431   unsigned Align =
03432       std::max((unsigned)TLI.getDataLayout()->getPrefTypeAlignment(Ty),
03433                I.getAlignment());
03434 
03435   SDValue AllocSize = getValue(I.getArraySize());
03436 
03437   EVT IntPtr = TLI.getPointerTy();
03438   if (AllocSize.getValueType() != IntPtr)
03439     AllocSize = DAG.getZExtOrTrunc(AllocSize, getCurSDLoc(), IntPtr);
03440 
03441   AllocSize = DAG.getNode(ISD::MUL, getCurSDLoc(), IntPtr,
03442                           AllocSize,
03443                           DAG.getConstant(TySize, IntPtr));
03444 
03445   // Handle alignment.  If the requested alignment is less than or equal to
03446   // the stack alignment, ignore it.  If the size is greater than or equal to
03447   // the stack alignment, we note this in the DYNAMIC_STACKALLOC node.
03448   unsigned StackAlign =
03449       DAG.getSubtarget().getFrameLowering()->getStackAlignment();
03450   if (Align <= StackAlign)
03451     Align = 0;
03452 
03453   // Round the size of the allocation up to the stack alignment size
03454   // by add SA-1 to the size.
03455   AllocSize = DAG.getNode(ISD::ADD, getCurSDLoc(),
03456                           AllocSize.getValueType(), AllocSize,
03457                           DAG.getIntPtrConstant(StackAlign-1));
03458 
03459   // Mask out the low bits for alignment purposes.
03460   AllocSize = DAG.getNode(ISD::AND, getCurSDLoc(),
03461                           AllocSize.getValueType(), AllocSize,
03462                           DAG.getIntPtrConstant(~(uint64_t)(StackAlign-1)));
03463 
03464   SDValue Ops[] = { getRoot(), AllocSize, DAG.getIntPtrConstant(Align) };
03465   SDVTList VTs = DAG.getVTList(AllocSize.getValueType(), MVT::Other);
03466   SDValue DSA = DAG.getNode(ISD::DYNAMIC_STACKALLOC, getCurSDLoc(), VTs, Ops);
03467   setValue(&I, DSA);
03468   DAG.setRoot(DSA.getValue(1));
03469 
03470   assert(FuncInfo.MF->getFrameInfo()->hasVarSizedObjects());
03471 }
03472 
03473 void SelectionDAGBuilder::visitLoad(const LoadInst &I) {
03474   if (I.isAtomic())
03475     return visitAtomicLoad(I);
03476 
03477   const Value *SV = I.getOperand(0);
03478   SDValue Ptr = getValue(SV);
03479 
03480   Type *Ty = I.getType();
03481 
03482   bool isVolatile = I.isVolatile();
03483   bool isNonTemporal = I.getMetadata(LLVMContext::MD_nontemporal) != nullptr;
03484   bool isInvariant = I.getMetadata(LLVMContext::MD_invariant_load) != nullptr;
03485   unsigned Alignment = I.getAlignment();
03486 
03487   AAMDNodes AAInfo;
03488   I.getAAMetadata(AAInfo);
03489   const MDNode *Ranges = I.getMetadata(LLVMContext::MD_range);
03490 
03491   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
03492   SmallVector<EVT, 4> ValueVTs;
03493   SmallVector<uint64_t, 4> Offsets;
03494   ComputeValueVTs(TLI, Ty, ValueVTs, &Offsets);
03495   unsigned NumValues = ValueVTs.size();
03496   if (NumValues == 0)
03497     return;
03498 
03499   SDValue Root;
03500   bool ConstantMemory = false;
03501   if (isVolatile || NumValues > MaxParallelChains)
03502     // Serialize volatile loads with other side effects.
03503     Root = getRoot();
03504   else if (AA->pointsToConstantMemory(
03505              AliasAnalysis::Location(SV, AA->getTypeStoreSize(Ty), AAInfo))) {
03506     // Do not serialize (non-volatile) loads of constant memory with anything.
03507     Root = DAG.getEntryNode();
03508     ConstantMemory = true;
03509   } else {
03510     // Do not serialize non-volatile loads against each other.
03511     Root = DAG.getRoot();
03512   }
03513 
03514   if (isVolatile)
03515     Root = TLI.prepareVolatileOrAtomicLoad(Root, getCurSDLoc(), DAG);
03516 
03517   SmallVector<SDValue, 4> Values(NumValues);
03518   SmallVector<SDValue, 4> Chains(std::min(unsigned(MaxParallelChains),
03519                                           NumValues));
03520   EVT PtrVT = Ptr.getValueType();
03521   unsigned ChainI = 0;
03522   for (unsigned i = 0; i != NumValues; ++i, ++ChainI) {
03523     // Serializing loads here may result in excessive register pressure, and
03524     // TokenFactor places arbitrary choke points on the scheduler. SD scheduling
03525     // could recover a bit by hoisting nodes upward in the chain by recognizing
03526     // they are side-effect free or do not alias. The optimizer should really
03527     // avoid this case by converting large object/array copies to llvm.memcpy
03528     // (MaxParallelChains should always remain as failsafe).
03529     if (ChainI == MaxParallelChains) {
03530       assert(PendingLoads.empty() && "PendingLoads must be serialized first");
03531       SDValue Chain = DAG.getNode(ISD::TokenFactor, getCurSDLoc(), MVT::Other,
03532                                   makeArrayRef(Chains.data(), ChainI));
03533       Root = Chain;
03534       ChainI = 0;
03535     }
03536     SDValue A = DAG.getNode(ISD::ADD, getCurSDLoc(),
03537                             PtrVT, Ptr,
03538                             DAG.getConstant(Offsets[i], PtrVT));
03539     SDValue L = DAG.getLoad(ValueVTs[i], getCurSDLoc(), Root,
03540                             A, MachinePointerInfo(SV, Offsets[i]), isVolatile,
03541                             isNonTemporal, isInvariant, Alignment, AAInfo,
03542                             Ranges);
03543 
03544     Values[i] = L;
03545     Chains[ChainI] = L.getValue(1);
03546   }
03547 
03548   if (!ConstantMemory) {
03549     SDValue Chain = DAG.getNode(ISD::TokenFactor, getCurSDLoc(), MVT::Other,
03550                                 makeArrayRef(Chains.data(), ChainI));
03551     if (isVolatile)
03552       DAG.setRoot(Chain);
03553     else
03554       PendingLoads.push_back(Chain);
03555   }
03556 
03557   setValue(&I, DAG.getNode(ISD::MERGE_VALUES, getCurSDLoc(),
03558                            DAG.getVTList(ValueVTs), Values));
03559 }
03560 
03561 void SelectionDAGBuilder::visitStore(const StoreInst &I) {
03562   if (I.isAtomic())
03563     return visitAtomicStore(I);
03564 
03565   const Value *SrcV = I.getOperand(0);
03566   const Value *PtrV = I.getOperand(1);
03567 
03568   SmallVector<EVT, 4> ValueVTs;
03569   SmallVector<uint64_t, 4> Offsets;
03570   ComputeValueVTs(DAG.getTargetLoweringInfo(), SrcV->getType(),
03571                   ValueVTs, &Offsets);
03572   unsigned NumValues = ValueVTs.size();
03573   if (NumValues == 0)
03574     return;
03575 
03576   // Get the lowered operands. Note that we do this after
03577   // checking if NumResults is zero, because with zero results
03578   // the operands won't have values in the map.
03579   SDValue Src = getValue(SrcV);
03580   SDValue Ptr = getValue(PtrV);
03581 
03582   SDValue Root = getRoot();
03583   SmallVector<SDValue, 4> Chains(std::min(unsigned(MaxParallelChains),
03584                                           NumValues));
03585   EVT PtrVT = Ptr.getValueType();
03586   bool isVolatile = I.isVolatile();
03587   bool isNonTemporal = I.getMetadata(LLVMContext::MD_nontemporal) != nullptr;
03588   unsigned Alignment = I.getAlignment();
03589 
03590   AAMDNodes AAInfo;
03591   I.getAAMetadata(AAInfo);
03592 
03593   unsigned ChainI = 0;
03594   for (unsigned i = 0; i != NumValues; ++i, ++ChainI) {
03595     // See visitLoad comments.
03596     if (ChainI == MaxParallelChains) {
03597       SDValue Chain = DAG.getNode(ISD::TokenFactor, getCurSDLoc(), MVT::Other,
03598                                   makeArrayRef(Chains.data(), ChainI));
03599       Root = Chain;
03600       ChainI = 0;
03601     }
03602     SDValue Add = DAG.getNode(ISD::ADD, getCurSDLoc(), PtrVT, Ptr,
03603                               DAG.getConstant(Offsets[i], PtrVT));
03604     SDValue St = DAG.getStore(Root, getCurSDLoc(),
03605                               SDValue(Src.getNode(), Src.getResNo() + i),
03606                               Add, MachinePointerInfo(PtrV, Offsets[i]),
03607                               isVolatile, isNonTemporal, Alignment, AAInfo);
03608     Chains[ChainI] = St;
03609   }
03610 
03611   SDValue StoreNode = DAG.getNode(ISD::TokenFactor, getCurSDLoc(), MVT::Other,
03612                                   makeArrayRef(Chains.data(), ChainI));
03613   DAG.setRoot(StoreNode);
03614 }
03615 
03616 void SelectionDAGBuilder::visitAtomicCmpXchg(const AtomicCmpXchgInst &I) {
03617   SDLoc dl = getCurSDLoc();
03618   AtomicOrdering SuccessOrder = I.getSuccessOrdering();
03619   AtomicOrdering FailureOrder = I.getFailureOrdering();
03620   SynchronizationScope Scope = I.getSynchScope();
03621 
03622   SDValue InChain = getRoot();
03623 
03624   MVT MemVT = getValue(I.getCompareOperand()).getSimpleValueType();
03625   SDVTList VTs = DAG.getVTList(MemVT, MVT::i1, MVT::Other);
03626   SDValue L = DAG.getAtomicCmpSwap(
03627       ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS, dl, MemVT, VTs, InChain,
03628       getValue(I.getPointerOperand()), getValue(I.getCompareOperand()),
03629       getValue(I.getNewValOperand()), MachinePointerInfo(I.getPointerOperand()),
03630       /*Alignment=*/ 0, SuccessOrder, FailureOrder, Scope);
03631 
03632   SDValue OutChain = L.getValue(2);
03633 
03634   setValue(&I, L);
03635   DAG.setRoot(OutChain);
03636 }
03637 
03638 void SelectionDAGBuilder::visitAtomicRMW(const AtomicRMWInst &I) {
03639   SDLoc dl = getCurSDLoc();
03640   ISD::NodeType NT;
03641   switch (I.getOperation()) {
03642   default: llvm_unreachable("Unknown atomicrmw operation");
03643   case AtomicRMWInst::Xchg: NT = ISD::ATOMIC_SWAP; break;
03644   case AtomicRMWInst::Add:  NT = ISD::ATOMIC_LOAD_ADD; break;
03645   case AtomicRMWInst::Sub:  NT = ISD::ATOMIC_LOAD_SUB; break;
03646   case AtomicRMWInst::And:  NT = ISD::ATOMIC_LOAD_AND; break;
03647   case AtomicRMWInst::Nand: NT = ISD::ATOMIC_LOAD_NAND; break;
03648   case AtomicRMWInst::Or:   NT = ISD::ATOMIC_LOAD_OR; break;
03649   case AtomicRMWInst::Xor:  NT = ISD::ATOMIC_LOAD_XOR; break;
03650   case AtomicRMWInst::Max:  NT = ISD::ATOMIC_LOAD_MAX; break;
03651   case AtomicRMWInst::Min:  NT = ISD::ATOMIC_LOAD_MIN; break;
03652   case AtomicRMWInst::UMax: NT = ISD::ATOMIC_LOAD_UMAX; break;
03653   case AtomicRMWInst::UMin: NT = ISD::ATOMIC_LOAD_UMIN; break;
03654   }
03655   AtomicOrdering Order = I.getOrdering();
03656   SynchronizationScope Scope = I.getSynchScope();
03657 
03658   SDValue InChain = getRoot();
03659 
03660   SDValue L =
03661     DAG.getAtomic(NT, dl,
03662                   getValue(I.getValOperand()).getSimpleValueType(),
03663                   InChain,
03664                   getValue(I.getPointerOperand()),
03665                   getValue(I.getValOperand()),
03666                   I.getPointerOperand(),
03667                   /* Alignment=*/ 0, Order, Scope);
03668 
03669   SDValue OutChain = L.getValue(1);
03670 
03671   setValue(&I, L);
03672   DAG.setRoot(OutChain);
03673 }
03674 
03675 void SelectionDAGBuilder::visitFence(const FenceInst &I) {
03676   SDLoc dl = getCurSDLoc();
03677   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
03678   SDValue Ops[3];
03679   Ops[0] = getRoot();
03680   Ops[1] = DAG.getConstant(I.getOrdering(), TLI.getPointerTy());
03681   Ops[2] = DAG.getConstant(I.getSynchScope(), TLI.getPointerTy());
03682   DAG.setRoot(DAG.getNode(ISD::ATOMIC_FENCE, dl, MVT::Other, Ops));
03683 }
03684 
03685 void SelectionDAGBuilder::visitAtomicLoad(const LoadInst &I) {
03686   SDLoc dl = getCurSDLoc();
03687   AtomicOrdering Order = I.getOrdering();
03688   SynchronizationScope Scope = I.getSynchScope();
03689 
03690   SDValue InChain = getRoot();
03691 
03692   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
03693   EVT VT = TLI.getValueType(I.getType());
03694 
03695   if (I.getAlignment() < VT.getSizeInBits() / 8)
03696     report_fatal_error("Cannot generate unaligned atomic load");
03697 
03698   MachineMemOperand *MMO =
03699       DAG.getMachineFunction().
03700       getMachineMemOperand(MachinePointerInfo(I.getPointerOperand()),
03701                            MachineMemOperand::MOVolatile |
03702                            MachineMemOperand::MOLoad,
03703                            VT.getStoreSize(),
03704                            I.getAlignment() ? I.getAlignment() :
03705                                               DAG.getEVTAlignment(VT));
03706 
03707   InChain = TLI.prepareVolatileOrAtomicLoad(InChain, dl, DAG);
03708   SDValue L =
03709       DAG.getAtomic(ISD::ATOMIC_LOAD, dl, VT, VT, InChain,
03710                     getValue(I.getPointerOperand()), MMO,
03711                     Order, Scope);
03712 
03713   SDValue OutChain = L.getValue(1);
03714 
03715   setValue(&I, L);
03716   DAG.setRoot(OutChain);
03717 }
03718 
03719 void SelectionDAGBuilder::visitAtomicStore(const StoreInst &I) {
03720   SDLoc dl = getCurSDLoc();
03721 
03722   AtomicOrdering Order = I.getOrdering();
03723   SynchronizationScope Scope = I.getSynchScope();
03724 
03725   SDValue InChain = getRoot();
03726 
03727   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
03728   EVT VT = TLI.getValueType(I.getValueOperand()->getType());
03729 
03730   if (I.getAlignment() < VT.getSizeInBits() / 8)
03731     report_fatal_error("Cannot generate unaligned atomic store");
03732 
03733   SDValue OutChain =
03734     DAG.getAtomic(ISD::ATOMIC_STORE, dl, VT,
03735                   InChain,
03736                   getValue(I.getPointerOperand()),
03737                   getValue(I.getValueOperand()),
03738                   I.getPointerOperand(), I.getAlignment(),
03739                   Order, Scope);
03740 
03741   DAG.setRoot(OutChain);
03742 }
03743 
03744 /// visitTargetIntrinsic - Lower a call of a target intrinsic to an INTRINSIC
03745 /// node.
03746 void SelectionDAGBuilder::visitTargetIntrinsic(const CallInst &I,
03747                                                unsigned Intrinsic) {
03748   bool HasChain = !I.doesNotAccessMemory();
03749   bool OnlyLoad = HasChain && I.onlyReadsMemory();
03750 
03751   // Build the operand list.
03752   SmallVector<SDValue, 8> Ops;
03753   if (HasChain) {  // If this intrinsic has side-effects, chainify it.
03754     if (OnlyLoad) {
03755       // We don't need to serialize loads against other loads.
03756       Ops.push_back(DAG.getRoot());
03757     } else {
03758       Ops.push_back(getRoot());
03759     }
03760   }
03761 
03762   // Info is set by getTgtMemInstrinsic
03763   TargetLowering::IntrinsicInfo Info;
03764   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
03765   bool IsTgtIntrinsic = TLI.getTgtMemIntrinsic(Info, I, Intrinsic);
03766 
03767   // Add the intrinsic ID as an integer operand if it's not a target intrinsic.
03768   if (!IsTgtIntrinsic || Info.opc == ISD::INTRINSIC_VOID ||
03769       Info.opc == ISD::INTRINSIC_W_CHAIN)
03770     Ops.push_back(DAG.getTargetConstant(Intrinsic, TLI.getPointerTy()));
03771 
03772   // Add all operands of the call to the operand list.
03773   for (unsigned i = 0, e = I.getNumArgOperands(); i != e; ++i) {
03774     SDValue Op = getValue(I.getArgOperand(i));
03775     Ops.push_back(Op);
03776   }
03777 
03778   SmallVector<EVT, 4> ValueVTs;
03779   ComputeValueVTs(TLI, I.getType(), ValueVTs);
03780 
03781   if (HasChain)
03782     ValueVTs.push_back(MVT::Other);
03783 
03784   SDVTList VTs = DAG.getVTList(ValueVTs);
03785 
03786   // Create the node.
03787   SDValue Result;
03788   if (IsTgtIntrinsic) {
03789     // This is target intrinsic that touches memory
03790     Result = DAG.getMemIntrinsicNode(Info.opc, getCurSDLoc(),
03791                                      VTs, Ops, Info.memVT,
03792                                    MachinePointerInfo(Info.ptrVal, Info.offset),
03793                                      Info.align, Info.vol,
03794                                      Info.readMem, Info.writeMem, Info.size);
03795   } else if (!HasChain) {
03796     Result = DAG.getNode(ISD::INTRINSIC_WO_CHAIN, getCurSDLoc(), VTs, Ops);
03797   } else if (!I.getType()->isVoidTy()) {
03798     Result = DAG.getNode(ISD::INTRINSIC_W_CHAIN, getCurSDLoc(), VTs, Ops);
03799   } else {
03800     Result = DAG.getNode(ISD::INTRINSIC_VOID, getCurSDLoc(), VTs, Ops);
03801   }
03802 
03803   if (HasChain) {
03804     SDValue Chain = Result.getValue(Result.getNode()->getNumValues()-1);
03805     if (OnlyLoad)
03806       PendingLoads.push_back(Chain);
03807     else
03808       DAG.setRoot(Chain);
03809   }
03810 
03811   if (!I.getType()->isVoidTy()) {
03812     if (VectorType *PTy = dyn_cast<VectorType>(I.getType())) {
03813       EVT VT = TLI.getValueType(PTy);
03814       Result = DAG.getNode(ISD::BITCAST, getCurSDLoc(), VT, Result);
03815     }
03816 
03817     setValue(&I, Result);
03818   }
03819 }
03820 
03821 /// GetSignificand - Get the significand and build it into a floating-point
03822 /// number with exponent of 1:
03823 ///
03824 ///   Op = (Op & 0x007fffff) | 0x3f800000;
03825 ///
03826 /// where Op is the hexadecimal representation of floating point value.
03827 static SDValue
03828 GetSignificand(SelectionDAG &DAG, SDValue Op, SDLoc dl) {
03829   SDValue t1 = DAG.getNode(ISD::AND, dl, MVT::i32, Op,
03830                            DAG.getConstant(0x007fffff, MVT::i32));
03831   SDValue t2 = DAG.getNode(ISD::OR, dl, MVT::i32, t1,
03832                            DAG.getConstant(0x3f800000, MVT::i32));
03833   return DAG.getNode(ISD::BITCAST, dl, MVT::f32, t2);
03834 }
03835 
03836 /// GetExponent - Get the exponent:
03837 ///
03838 ///   (float)(int)(((Op & 0x7f800000) >> 23) - 127);
03839 ///
03840 /// where Op is the hexadecimal representation of floating point value.
03841 static SDValue
03842 GetExponent(SelectionDAG &DAG, SDValue Op, const TargetLowering &TLI,
03843             SDLoc dl) {
03844   SDValue t0 = DAG.getNode(ISD::AND, dl, MVT::i32, Op,
03845                            DAG.getConstant(0x7f800000, MVT::i32));
03846   SDValue t1 = DAG.getNode(ISD::SRL, dl, MVT::i32, t0,
03847                            DAG.getConstant(23, TLI.getPointerTy()));
03848   SDValue t2 = DAG.getNode(ISD::SUB, dl, MVT::i32, t1,
03849                            DAG.getConstant(127, MVT::i32));
03850   return DAG.getNode(ISD::SINT_TO_FP, dl, MVT::f32, t2);
03851 }
03852 
03853 /// getF32Constant - Get 32-bit floating point constant.
03854 static SDValue
03855 getF32Constant(SelectionDAG &DAG, unsigned Flt) {
03856   return DAG.getConstantFP(APFloat(APFloat::IEEEsingle, APInt(32, Flt)),
03857                            MVT::f32);
03858 }
03859 
03860 /// expandExp - Lower an exp intrinsic. Handles the special sequences for
03861 /// limited-precision mode.
03862 static SDValue expandExp(SDLoc dl, SDValue Op, SelectionDAG &DAG,
03863                          const TargetLowering &TLI) {
03864   if (Op.getValueType() == MVT::f32 &&
03865       LimitFloatPrecision > 0 && LimitFloatPrecision <= 18) {
03866 
03867     // Put the exponent in the right bit position for later addition to the
03868     // final result:
03869     //
03870     //   #define LOG2OFe 1.4426950f
03871     //   IntegerPartOfX = ((int32_t)(X * LOG2OFe));
03872     SDValue t0 = DAG.getNode(ISD::FMUL, dl, MVT::f32, Op,
03873                              getF32Constant(DAG, 0x3fb8aa3b));
03874     SDValue IntegerPartOfX = DAG.getNode(ISD::FP_TO_SINT, dl, MVT::i32, t0);
03875 
03876     //   FractionalPartOfX = (X * LOG2OFe) - (float)IntegerPartOfX;
03877     SDValue t1 = DAG.getNode(ISD::SINT_TO_FP, dl, MVT::f32, IntegerPartOfX);
03878     SDValue X = DAG.getNode(ISD::FSUB, dl, MVT::f32, t0, t1);
03879 
03880     //   IntegerPartOfX <<= 23;
03881     IntegerPartOfX = DAG.getNode(ISD::SHL, dl, MVT::i32, IntegerPartOfX,
03882                                  DAG.getConstant(23, TLI.getPointerTy()));
03883 
03884     SDValue TwoToFracPartOfX;
03885     if (LimitFloatPrecision <= 6) {
03886       // For floating-point precision of 6:
03887       //
03888       //   TwoToFractionalPartOfX =
03889       //     0.997535578f +
03890       //       (0.735607626f + 0.252464424f * x) * x;
03891       //
03892       // error 0.0144103317, which is 6 bits
03893       SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X,
03894                                getF32Constant(DAG, 0x3e814304));
03895       SDValue t3 = DAG.getNode(ISD::FADD, dl, MVT::f32, t2,
03896                                getF32Constant(DAG, 0x3f3c50c8));
03897       SDValue t4 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t3, X);
03898       TwoToFracPartOfX = DAG.getNode(ISD::FADD, dl, MVT::f32, t4,
03899                                      getF32Constant(DAG, 0x3f7f5e7e));
03900     } else if (LimitFloatPrecision <= 12) {
03901       // For floating-point precision of 12:
03902       //
03903       //   TwoToFractionalPartOfX =
03904       //     0.999892986f +
03905       //       (0.696457318f +
03906       //         (0.224338339f + 0.792043434e-1f * x) * x) * x;
03907       //
03908       // 0.000107046256 error, which is 13 to 14 bits
03909       SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X,
03910                                getF32Constant(DAG, 0x3da235e3));
03911       SDValue t3 = DAG.getNode(ISD::FADD, dl, MVT::f32, t2,
03912                                getF32Constant(DAG, 0x3e65b8f3));
03913       SDValue t4 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t3, X);
03914       SDValue t5 = DAG.getNode(ISD::FADD, dl, MVT::f32, t4,
03915                                getF32Constant(DAG, 0x3f324b07));
03916       SDValue t6 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t5, X);
03917       TwoToFracPartOfX = DAG.getNode(ISD::FADD, dl, MVT::f32, t6,
03918                                      getF32Constant(DAG, 0x3f7ff8fd));
03919     } else { // LimitFloatPrecision <= 18
03920       // For floating-point precision of 18:
03921       //
03922       //   TwoToFractionalPartOfX =
03923       //     0.999999982f +
03924       //       (0.693148872f +
03925       //         (0.240227044f +
03926       //           (0.554906021e-1f +
03927       //             (0.961591928e-2f +
03928       //               (0.136028312e-2f + 0.157059148e-3f *x)*x)*x)*x)*x)*x;
03929       //
03930       // error 2.47208000*10^(-7), which is better than 18 bits
03931       SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X,
03932                                getF32Constant(DAG, 0x3924b03e));
03933       SDValue t3 = DAG.getNode(ISD::FADD, dl, MVT::f32, t2,
03934                                getF32Constant(DAG, 0x3ab24b87));
03935       SDValue t4 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t3, X);
03936       SDValue t5 = DAG.getNode(ISD::FADD, dl, MVT::f32, t4,
03937                                getF32Constant(DAG, 0x3c1d8c17));
03938       SDValue t6 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t5, X);
03939       SDValue t7 = DAG.getNode(ISD::FADD, dl, MVT::f32, t6,
03940                                getF32Constant(DAG, 0x3d634a1d));
03941       SDValue t8 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t7, X);
03942       SDValue t9 = DAG.getNode(ISD::FADD, dl, MVT::f32, t8,
03943                                getF32Constant(DAG, 0x3e75fe14));
03944       SDValue t10 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t9, X);
03945       SDValue t11 = DAG.getNode(ISD::FADD, dl, MVT::f32, t10,
03946                                 getF32Constant(DAG, 0x3f317234));
03947       SDValue t12 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t11, X);
03948       TwoToFracPartOfX = DAG.getNode(ISD::FADD, dl, MVT::f32, t12,
03949                                      getF32Constant(DAG, 0x3f800000));
03950     }
03951 
03952     // Add the exponent into the result in integer domain.
03953     SDValue t13 = DAG.getNode(ISD::BITCAST, dl, MVT::i32, TwoToFracPartOfX);
03954     return DAG.getNode(ISD::BITCAST, dl, MVT::f32,
03955                        DAG.getNode(ISD::ADD, dl, MVT::i32,
03956                                    t13, IntegerPartOfX));
03957   }
03958 
03959   // No special expansion.
03960   return DAG.getNode(ISD::FEXP, dl, Op.getValueType(), Op);
03961 }
03962 
03963 /// expandLog - Lower a log intrinsic. Handles the special sequences for
03964 /// limited-precision mode.
03965 static SDValue expandLog(SDLoc dl, SDValue Op, SelectionDAG &DAG,
03966                          const TargetLowering &TLI) {
03967   if (Op.getValueType() == MVT::f32 &&
03968       LimitFloatPrecision > 0 && LimitFloatPrecision <= 18) {
03969     SDValue Op1 = DAG.getNode(ISD::BITCAST, dl, MVT::i32, Op);
03970 
03971     // Scale the exponent by log(2) [0.69314718f].
03972     SDValue Exp = GetExponent(DAG, Op1, TLI, dl);
03973     SDValue LogOfExponent = DAG.getNode(ISD::FMUL, dl, MVT::f32, Exp,
03974                                         getF32Constant(DAG, 0x3f317218));
03975 
03976     // Get the significand and build it into a floating-point number with
03977     // exponent of 1.
03978     SDValue X = GetSignificand(DAG, Op1, dl);
03979 
03980     SDValue LogOfMantissa;
03981     if (LimitFloatPrecision <= 6) {
03982       // For floating-point precision of 6:
03983       //
03984       //   LogofMantissa =
03985       //     -1.1609546f +
03986       //       (1.4034025f - 0.23903021f * x) * x;
03987       //
03988       // error 0.0034276066, which is better than 8 bits
03989       SDValue t0 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X,
03990                                getF32Constant(DAG, 0xbe74c456));
03991       SDValue t1 = DAG.getNode(ISD::FADD, dl, MVT::f32, t0,
03992                                getF32Constant(DAG, 0x3fb3a2b1));
03993       SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t1, X);
03994       LogOfMantissa = DAG.getNode(ISD::FSUB, dl, MVT::f32, t2,
03995                                   getF32Constant(DAG, 0x3f949a29));
03996     } else if (LimitFloatPrecision <= 12) {
03997       // For floating-point precision of 12:
03998       //
03999       //   LogOfMantissa =
04000       //     -1.7417939f +
04001       //       (2.8212026f +
04002       //         (-1.4699568f +
04003       //           (0.44717955f - 0.56570851e-1f * x) * x) * x) * x;
04004       //
04005       // error 0.000061011436, which is 14 bits
04006       SDValue t0 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X,
04007                                getF32Constant(DAG, 0xbd67b6d6));
04008       SDValue t1 = DAG.getNode(ISD::FADD, dl, MVT::f32, t0,
04009                                getF32Constant(DAG, 0x3ee4f4b8));
04010       SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t1, X);
04011       SDValue t3 = DAG.getNode(ISD::FSUB, dl, MVT::f32, t2,
04012                                getF32Constant(DAG, 0x3fbc278b));
04013       SDValue t4 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t3, X);
04014       SDValue t5 = DAG.getNode(ISD::FADD, dl, MVT::f32, t4,
04015                                getF32Constant(DAG, 0x40348e95));
04016       SDValue t6 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t5, X);
04017       LogOfMantissa = DAG.getNode(ISD::FSUB, dl, MVT::f32, t6,
04018                                   getF32Constant(DAG, 0x3fdef31a));
04019     } else { // LimitFloatPrecision <= 18
04020       // For floating-point precision of 18:
04021       //
04022       //   LogOfMantissa =
04023       //     -2.1072184f +
04024       //       (4.2372794f +
04025       //         (-3.7029485f +
04026       //           (2.2781945f +
04027       //             (-0.87823314f +
04028       //               (0.19073739f - 0.17809712e-1f * x) * x) * x) * x) * x)*x;
04029       //
04030       // error 0.0000023660568, which is better than 18 bits
04031       SDValue t0 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X,
04032                                getF32Constant(DAG, 0xbc91e5ac));
04033       SDValue t1 = DAG.getNode(ISD::FADD, dl, MVT::f32, t0,
04034                                getF32Constant(DAG, 0x3e4350aa));
04035       SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t1, X);
04036       SDValue t3 = DAG.getNode(ISD::FSUB, dl, MVT::f32, t2,
04037                                getF32Constant(DAG, 0x3f60d3e3));
04038       SDValue t4 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t3, X);
04039       SDValue t5 = DAG.getNode(ISD::FADD, dl, MVT::f32, t4,
04040                                getF32Constant(DAG, 0x4011cdf0));
04041       SDValue t6 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t5, X);
04042       SDValue t7 = DAG.getNode(ISD::FSUB, dl, MVT::f32, t6,
04043                                getF32Constant(DAG, 0x406cfd1c));
04044       SDValue t8 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t7, X);
04045       SDValue t9 = DAG.getNode(ISD::FADD, dl, MVT::f32, t8,
04046                                getF32Constant(DAG, 0x408797cb));
04047       SDValue t10 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t9, X);
04048       LogOfMantissa = DAG.getNode(ISD::FSUB, dl, MVT::f32, t10,
04049                                   getF32Constant(DAG, 0x4006dcab));
04050     }
04051 
04052     return DAG.getNode(ISD::FADD, dl, MVT::f32, LogOfExponent, LogOfMantissa);
04053   }
04054 
04055   // No special expansion.
04056   return DAG.getNode(ISD::FLOG, dl, Op.getValueType(), Op);
04057 }
04058 
04059 /// expandLog2 - Lower a log2 intrinsic. Handles the special sequences for
04060 /// limited-precision mode.
04061 static SDValue expandLog2(SDLoc dl, SDValue Op, SelectionDAG &DAG,
04062                           const TargetLowering &TLI) {
04063   if (Op.getValueType() == MVT::f32 &&
04064       LimitFloatPrecision > 0 && LimitFloatPrecision <= 18) {
04065     SDValue Op1 = DAG.getNode(ISD::BITCAST, dl, MVT::i32, Op);
04066 
04067     // Get the exponent.
04068     SDValue LogOfExponent = GetExponent(DAG, Op1, TLI, dl);
04069 
04070     // Get the significand and build it into a floating-point number with
04071     // exponent of 1.
04072     SDValue X = GetSignificand(DAG, Op1, dl);
04073 
04074     // Different possible minimax approximations of significand in
04075     // floating-point for various degrees of accuracy over [1,2].
04076     SDValue Log2ofMantissa;
04077     if (LimitFloatPrecision <= 6) {
04078       // For floating-point precision of 6:
04079       //
04080       //   Log2ofMantissa = -1.6749035f + (2.0246817f - .34484768f * x) * x;
04081       //
04082       // error 0.0049451742, which is more than 7 bits
04083       SDValue t0 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X,
04084                                getF32Constant(DAG, 0xbeb08fe0));
04085       SDValue t1 = DAG.getNode(ISD::FADD, dl, MVT::f32, t0,
04086                                getF32Constant(DAG, 0x40019463));
04087       SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t1, X);
04088       Log2ofMantissa = DAG.getNode(ISD::FSUB, dl, MVT::f32, t2,
04089                                    getF32Constant(DAG, 0x3fd6633d));
04090     } else if (LimitFloatPrecision <= 12) {
04091       // For floating-point precision of 12:
04092       //
04093       //   Log2ofMantissa =
04094       //     -2.51285454f +
04095       //       (4.07009056f +
04096       //         (-2.12067489f +
04097       //           (.645142248f - 0.816157886e-1f * x) * x) * x) * x;
04098       //
04099       // error 0.0000876136000, which is better than 13 bits
04100       SDValue t0 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X,
04101                                getF32Constant(DAG, 0xbda7262e));
04102       SDValue t1 = DAG.getNode(ISD::FADD, dl, MVT::f32, t0,
04103                                getF32Constant(DAG, 0x3f25280b));
04104       SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t1, X);
04105       SDValue t3 = DAG.getNode(ISD::FSUB, dl, MVT::f32, t2,
04106                                getF32Constant(DAG, 0x4007b923));
04107       SDValue t4 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t3, X);
04108       SDValue t5 = DAG.getNode(ISD::FADD, dl, MVT::f32, t4,
04109                                getF32Constant(DAG, 0x40823e2f));
04110       SDValue t6 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t5, X);
04111       Log2ofMantissa = DAG.getNode(ISD::FSUB, dl, MVT::f32, t6,
04112                                    getF32Constant(DAG, 0x4020d29c));
04113     } else { // LimitFloatPrecision <= 18
04114       // For floating-point precision of 18:
04115       //
04116       //   Log2ofMantissa =
04117       //     -3.0400495f +
04118       //       (6.1129976f +
04119       //         (-5.3420409f +
04120       //           (3.2865683f +
04121       //             (-1.2669343f +
04122       //               (0.27515199f -
04123       //                 0.25691327e-1f * x) * x) * x) * x) * x) * x;
04124       //
04125       // error 0.0000018516, which is better than 18 bits
04126       SDValue t0 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X,
04127                                getF32Constant(DAG, 0xbcd2769e));
04128       SDValue t1 = DAG.getNode(ISD::FADD, dl, MVT::f32, t0,
04129                                getF32Constant(DAG, 0x3e8ce0b9));
04130       SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t1, X);
04131       SDValue t3 = DAG.getNode(ISD::FSUB, dl, MVT::f32, t2,
04132                                getF32Constant(DAG, 0x3fa22ae7));
04133       SDValue t4 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t3, X);
04134       SDValue t5 = DAG.getNode(ISD::FADD, dl, MVT::f32, t4,
04135                                getF32Constant(DAG, 0x40525723));
04136       SDValue t6 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t5, X);
04137       SDValue t7 = DAG.getNode(ISD::FSUB, dl, MVT::f32, t6,
04138                                getF32Constant(DAG, 0x40aaf200));
04139       SDValue t8 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t7, X);
04140       SDValue t9 = DAG.getNode(ISD::FADD, dl, MVT::f32, t8,
04141                                getF32Constant(DAG, 0x40c39dad));
04142       SDValue t10 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t9, X);
04143       Log2ofMantissa = DAG.getNode(ISD::FSUB, dl, MVT::f32, t10,
04144                                    getF32Constant(DAG, 0x4042902c));
04145     }
04146 
04147     return DAG.getNode(ISD::FADD, dl, MVT::f32, LogOfExponent, Log2ofMantissa);
04148   }
04149 
04150   // No special expansion.
04151   return DAG.getNode(ISD::FLOG2, dl, Op.getValueType(), Op);
04152 }
04153 
04154 /// expandLog10 - Lower a log10 intrinsic. Handles the special sequences for
04155 /// limited-precision mode.
04156 static SDValue expandLog10(SDLoc dl, SDValue Op, SelectionDAG &DAG,
04157                            const TargetLowering &TLI) {
04158   if (Op.getValueType() == MVT::f32 &&
04159       LimitFloatPrecision > 0 && LimitFloatPrecision <= 18) {
04160     SDValue Op1 = DAG.getNode(ISD::BITCAST, dl, MVT::i32, Op);
04161 
04162     // Scale the exponent by log10(2) [0.30102999f].
04163     SDValue Exp = GetExponent(DAG, Op1, TLI, dl);
04164     SDValue LogOfExponent = DAG.getNode(ISD::FMUL, dl, MVT::f32, Exp,
04165                                         getF32Constant(DAG, 0x3e9a209a));
04166 
04167     // Get the significand and build it into a floating-point number with
04168     // exponent of 1.
04169     SDValue X = GetSignificand(DAG, Op1, dl);
04170 
04171     SDValue Log10ofMantissa;
04172     if (LimitFloatPrecision <= 6) {
04173       // For floating-point precision of 6:
04174       //
04175       //   Log10ofMantissa =
04176       //     -0.50419619f +
04177       //       (0.60948995f - 0.10380950f * x) * x;
04178       //
04179       // error 0.0014886165, which is 6 bits
04180       SDValue t0 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X,
04181                                getF32Constant(DAG, 0xbdd49a13));
04182       SDValue t1 = DAG.getNode(ISD::FADD, dl, MVT::f32, t0,
04183                                getF32Constant(DAG, 0x3f1c0789));
04184       SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t1, X);
04185       Log10ofMantissa = DAG.getNode(ISD::FSUB, dl, MVT::f32, t2,
04186                                     getF32Constant(DAG, 0x3f011300));
04187     } else if (LimitFloatPrecision <= 12) {
04188       // For floating-point precision of 12:
04189       //
04190       //   Log10ofMantissa =
04191       //     -0.64831180f +
04192       //       (0.91751397f +
04193       //         (-0.31664806f + 0.47637168e-1f * x) * x) * x;
04194       //
04195       // error 0.00019228036, which is better than 12 bits
04196       SDValue t0 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X,
04197                                getF32Constant(DAG, 0x3d431f31));
04198       SDValue t1 = DAG.getNode(ISD::FSUB, dl, MVT::f32, t0,
04199                                getF32Constant(DAG, 0x3ea21fb2));
04200       SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t1, X);
04201       SDValue t3 = DAG.getNode(ISD::FADD, dl, MVT::f32, t2,
04202                                getF32Constant(DAG, 0x3f6ae232));
04203       SDValue t4 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t3, X);
04204       Log10ofMantissa = DAG.getNode(ISD::FSUB, dl, MVT::f32, t4,
04205                                     getF32Constant(DAG, 0x3f25f7c3));
04206     } else { // LimitFloatPrecision <= 18
04207       // For floating-point precision of 18:
04208       //
04209       //   Log10ofMantissa =
04210       //     -0.84299375f +
04211       //       (1.5327582f +
04212       //         (-1.0688956f +
04213       //           (0.49102474f +
04214       //             (-0.12539807f + 0.13508273e-1f * x) * x) * x) * x) * x;
04215       //
04216       // error 0.0000037995730, which is better than 18 bits
04217       SDValue t0 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X,
04218                                getF32Constant(DAG, 0x3c5d51ce));
04219       SDValue t1 = DAG.getNode(ISD::FSUB, dl, MVT::f32, t0,
04220                                getF32Constant(DAG, 0x3e00685a));
04221       SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t1, X);
04222       SDValue t3 = DAG.getNode(ISD::FADD, dl, MVT::f32, t2,
04223                                getF32Constant(DAG, 0x3efb6798));
04224       SDValue t4 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t3, X);
04225       SDValue t5 = DAG.getNode(ISD::FSUB, dl, MVT::f32, t4,
04226                                getF32Constant(DAG, 0x3f88d192));
04227       SDValue t6 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t5, X);
04228       SDValue t7 = DAG.getNode(ISD::FADD, dl, MVT::f32, t6,
04229                                getF32Constant(DAG, 0x3fc4316c));
04230       SDValue t8 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t7, X);
04231       Log10ofMantissa = DAG.getNode(ISD::FSUB, dl, MVT::f32, t8,
04232                                     getF32Constant(DAG, 0x3f57ce70));
04233     }
04234 
04235     return DAG.getNode(ISD::FADD, dl, MVT::f32, LogOfExponent, Log10ofMantissa);
04236   }
04237 
04238   // No special expansion.
04239   return DAG.getNode(ISD::FLOG10, dl, Op.getValueType(), Op);
04240 }
04241 
04242 /// expandExp2 - Lower an exp2 intrinsic. Handles the special sequences for
04243 /// limited-precision mode.
04244 static SDValue expandExp2(SDLoc dl, SDValue Op, SelectionDAG &DAG,
04245                           const TargetLowering &TLI) {
04246   if (Op.getValueType() == MVT::f32 &&
04247       LimitFloatPrecision > 0 && LimitFloatPrecision <= 18) {
04248     SDValue IntegerPartOfX = DAG.getNode(ISD::FP_TO_SINT, dl, MVT::i32, Op);
04249 
04250     //   FractionalPartOfX = x - (float)IntegerPartOfX;
04251     SDValue t1 = DAG.getNode(ISD::SINT_TO_FP, dl, MVT::f32, IntegerPartOfX);
04252     SDValue X = DAG.getNode(ISD::FSUB, dl, MVT::f32, Op, t1);
04253 
04254     //   IntegerPartOfX <<= 23;
04255     IntegerPartOfX = DAG.getNode(ISD::SHL, dl, MVT::i32, IntegerPartOfX,
04256                                  DAG.getConstant(23, TLI.getPointerTy()));
04257 
04258     SDValue TwoToFractionalPartOfX;
04259     if (LimitFloatPrecision <= 6) {
04260       // For floating-point precision of 6:
04261       //
04262       //   TwoToFractionalPartOfX =
04263       //     0.997535578f +
04264       //       (0.735607626f + 0.252464424f * x) * x;
04265       //
04266       // error 0.0144103317, which is 6 bits
04267       SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X,
04268                                getF32Constant(DAG, 0x3e814304));
04269       SDValue t3 = DAG.getNode(ISD::FADD, dl, MVT::f32, t2,
04270                                getF32Constant(DAG, 0x3f3c50c8));
04271       SDValue t4 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t3, X);
04272       TwoToFractionalPartOfX = DAG.getNode(ISD::FADD, dl, MVT::f32, t4,
04273                                            getF32Constant(DAG, 0x3f7f5e7e));
04274     } else if (LimitFloatPrecision <= 12) {
04275       // For floating-point precision of 12:
04276       //
04277       //   TwoToFractionalPartOfX =
04278       //     0.999892986f +
04279       //       (0.696457318f +
04280       //         (0.224338339f + 0.792043434e-1f * x) * x) * x;
04281       //
04282       // error 0.000107046256, which is 13 to 14 bits
04283       SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X,
04284                                getF32Constant(DAG, 0x3da235e3));
04285       SDValue t3 = DAG.getNode(ISD::FADD, dl, MVT::f32, t2,
04286                                getF32Constant(DAG, 0x3e65b8f3));
04287       SDValue t4 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t3, X);
04288       SDValue t5 = DAG.getNode(ISD::FADD, dl, MVT::f32, t4,
04289                                getF32Constant(DAG, 0x3f324b07));
04290       SDValue t6 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t5, X);
04291       TwoToFractionalPartOfX = DAG.getNode(ISD::FADD, dl, MVT::f32, t6,
04292                                            getF32Constant(DAG, 0x3f7ff8fd));
04293     } else { // LimitFloatPrecision <= 18
04294       // For floating-point precision of 18:
04295       //
04296       //   TwoToFractionalPartOfX =
04297       //     0.999999982f +
04298       //       (0.693148872f +
04299       //         (0.240227044f +
04300       //           (0.554906021e-1f +
04301       //             (0.961591928e-2f +
04302       //               (0.136028312e-2f + 0.157059148e-3f *x)*x)*x)*x)*x)*x;
04303       // error 2.47208000*10^(-7), which is better than 18 bits
04304       SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X,
04305                                getF32Constant(DAG, 0x3924b03e));
04306       SDValue t3 = DAG.getNode(ISD::FADD, dl, MVT::f32, t2,
04307                                getF32Constant(DAG, 0x3ab24b87));
04308       SDValue t4 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t3, X);
04309       SDValue t5 = DAG.getNode(ISD::FADD, dl, MVT::f32, t4,
04310                                getF32Constant(DAG, 0x3c1d8c17));
04311       SDValue t6 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t5, X);
04312       SDValue t7 = DAG.getNode(ISD::FADD, dl, MVT::f32, t6,
04313                                getF32Constant(DAG, 0x3d634a1d));
04314       SDValue t8 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t7, X);
04315       SDValue t9 = DAG.getNode(ISD::FADD, dl, MVT::f32, t8,
04316                                getF32Constant(DAG, 0x3e75fe14));
04317       SDValue t10 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t9, X);
04318       SDValue t11 = DAG.getNode(ISD::FADD, dl, MVT::f32, t10,
04319                                 getF32Constant(DAG, 0x3f317234));
04320       SDValue t12 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t11, X);
04321       TwoToFractionalPartOfX = DAG.getNode(ISD::FADD, dl, MVT::f32, t12,
04322                                            getF32Constant(DAG, 0x3f800000));
04323     }
04324 
04325     // Add the exponent into the result in integer domain.
04326     SDValue t13 = DAG.getNode(ISD::BITCAST, dl, MVT::i32,
04327                               TwoToFractionalPartOfX);
04328     return DAG.getNode(ISD::BITCAST, dl, MVT::f32,
04329                        DAG.getNode(ISD::ADD, dl, MVT::i32,
04330                                    t13, IntegerPartOfX));
04331   }
04332 
04333   // No special expansion.
04334   return DAG.getNode(ISD::FEXP2, dl, Op.getValueType(), Op);
04335 }
04336 
04337 /// visitPow - Lower a pow intrinsic. Handles the special sequences for
04338 /// limited-precision mode with x == 10.0f.
04339 static SDValue expandPow(SDLoc dl, SDValue LHS, SDValue RHS,
04340                          SelectionDAG &DAG, const TargetLowering &TLI) {
04341   bool IsExp10 = false;
04342   if (LHS.getValueType() == MVT::f32 && RHS.getValueType() == MVT::f32 &&
04343       LimitFloatPrecision > 0 && LimitFloatPrecision <= 18) {
04344     if (ConstantFPSDNode *LHSC = dyn_cast<ConstantFPSDNode>(LHS)) {
04345       APFloat Ten(10.0f);
04346       IsExp10 = LHSC->isExactlyValue(Ten);
04347     }
04348   }
04349 
04350   if (IsExp10) {
04351     // Put the exponent in the right bit position for later addition to the
04352     // final result:
04353     //
04354     //   #define LOG2OF10 3.3219281f
04355     //   IntegerPartOfX = (int32_t)(x * LOG2OF10);
04356     SDValue t0 = DAG.getNode(ISD::FMUL, dl, MVT::f32, RHS,
04357                              getF32Constant(DAG, 0x40549a78));
04358     SDValue IntegerPartOfX = DAG.getNode(ISD::FP_TO_SINT, dl, MVT::i32, t0);
04359 
04360     //   FractionalPartOfX = x - (float)IntegerPartOfX;
04361     SDValue t1 = DAG.getNode(ISD::SINT_TO_FP, dl, MVT::f32, IntegerPartOfX);
04362     SDValue X = DAG.getNode(ISD::FSUB, dl, MVT::f32, t0, t1);
04363 
04364     //   IntegerPartOfX <<= 23;
04365     IntegerPartOfX = DAG.getNode(ISD::SHL, dl, MVT::i32, IntegerPartOfX,
04366                                  DAG.getConstant(23, TLI.getPointerTy()));
04367 
04368     SDValue TwoToFractionalPartOfX;
04369     if (LimitFloatPrecision <= 6) {
04370       // For floating-point precision of 6:
04371       //
04372       //   twoToFractionalPartOfX =
04373       //     0.997535578f +
04374       //       (0.735607626f + 0.252464424f * x) * x;
04375       //
04376       // error 0.0144103317, which is 6 bits
04377       SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X,
04378                                getF32Constant(DAG, 0x3e814304));
04379       SDValue t3 = DAG.getNode(ISD::FADD, dl, MVT::f32, t2,
04380                                getF32Constant(DAG, 0x3f3c50c8));
04381       SDValue t4 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t3, X);
04382       TwoToFractionalPartOfX = DAG.getNode(ISD::FADD, dl, MVT::f32, t4,
04383                                            getF32Constant(DAG, 0x3f7f5e7e));
04384     } else if (LimitFloatPrecision <= 12) {
04385       // For floating-point precision of 12:
04386       //
04387       //   TwoToFractionalPartOfX =
04388       //     0.999892986f +
04389       //       (0.696457318f +
04390       //         (0.224338339f + 0.792043434e-1f * x) * x) * x;
04391       //
04392       // error 0.000107046256, which is 13 to 14 bits
04393       SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X,
04394                                getF32Constant(DAG, 0x3da235e3));
04395       SDValue t3 = DAG.getNode(ISD::FADD, dl, MVT::f32, t2,
04396                                getF32Constant(DAG, 0x3e65b8f3));
04397       SDValue t4 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t3, X);
04398       SDValue t5 = DAG.getNode(ISD::FADD, dl, MVT::f32, t4,
04399                                getF32Constant(DAG, 0x3f324b07));
04400       SDValue t6 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t5, X);
04401       TwoToFractionalPartOfX = DAG.getNode(ISD::FADD, dl, MVT::f32, t6,
04402                                            getF32Constant(DAG, 0x3f7ff8fd));
04403     } else { // LimitFloatPrecision <= 18
04404       // For floating-point precision of 18:
04405       //
04406       //   TwoToFractionalPartOfX =
04407       //     0.999999982f +
04408       //       (0.693148872f +
04409       //         (0.240227044f +
04410       //           (0.554906021e-1f +
04411       //             (0.961591928e-2f +
04412       //               (0.136028312e-2f + 0.157059148e-3f *x)*x)*x)*x)*x)*x;
04413       // error 2.47208000*10^(-7), which is better than 18 bits
04414       SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X,
04415                                getF32Constant(DAG, 0x3924b03e));
04416       SDValue t3 = DAG.getNode(ISD::FADD, dl, MVT::f32, t2,
04417                                getF32Constant(DAG, 0x3ab24b87));
04418       SDValue t4 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t3, X);
04419       SDValue t5 = DAG.getNode(ISD::FADD, dl, MVT::f32, t4,
04420                                getF32Constant(DAG, 0x3c1d8c17));
04421       SDValue t6 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t5, X);
04422       SDValue t7 = DAG.getNode(ISD::FADD, dl, MVT::f32, t6,
04423                                getF32Constant(DAG, 0x3d634a1d));
04424       SDValue t8 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t7, X);
04425       SDValue t9 = DAG.getNode(ISD::FADD, dl, MVT::f32, t8,
04426                                getF32Constant(DAG, 0x3e75fe14));
04427       SDValue t10 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t9, X);
04428       SDValue t11 = DAG.getNode(ISD::FADD, dl, MVT::f32, t10,
04429                                 getF32Constant(DAG, 0x3f317234));
04430       SDValue t12 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t11, X);
04431       TwoToFractionalPartOfX = DAG.getNode(ISD::FADD, dl, MVT::f32, t12,
04432                                            getF32Constant(DAG, 0x3f800000));
04433     }
04434 
04435     SDValue t13 = DAG.getNode(ISD::BITCAST, dl,MVT::i32,TwoToFractionalPartOfX);
04436     return DAG.getNode(ISD::BITCAST, dl, MVT::f32,
04437                        DAG.getNode(ISD::ADD, dl, MVT::i32,
04438                                    t13, IntegerPartOfX));
04439   }
04440 
04441   // No special expansion.
04442   return DAG.getNode(ISD::FPOW, dl, LHS.getValueType(), LHS, RHS);
04443 }
04444 
04445 
04446 /// ExpandPowI - Expand a llvm.powi intrinsic.
04447 static SDValue ExpandPowI(SDLoc DL, SDValue LHS, SDValue RHS,
04448                           SelectionDAG &DAG) {
04449   // If RHS is a constant, we can expand this out to a multiplication tree,
04450   // otherwise we end up lowering to a call to __powidf2 (for example).  When
04451   // optimizing for size, we only want to do this if the expansion would produce
04452   // a small number of multiplies, otherwise we do the full expansion.
04453   if (ConstantSDNode *RHSC = dyn_cast<ConstantSDNode>(RHS)) {
04454     // Get the exponent as a positive value.
04455     unsigned Val = RHSC->getSExtValue();
04456     if ((int)Val < 0) Val = -Val;
04457 
04458     // powi(x, 0) -> 1.0
04459     if (Val == 0)
04460       return DAG.getConstantFP(1.0, LHS.getValueType());
04461 
04462     const Function *F = DAG.getMachineFunction().getFunction();
04463     if (!F->getAttributes().hasAttribute(AttributeSet::FunctionIndex,
04464                                          Attribute::OptimizeForSize) ||
04465         // If optimizing for size, don't insert too many multiplies.  This
04466         // inserts up to 5 multiplies.
04467         CountPopulation_32(Val)+Log2_32(Val) < 7) {
04468       // We use the simple binary decomposition method to generate the multiply
04469       // sequence.  There are more optimal ways to do this (for example,
04470       // powi(x,15) generates one more multiply than it should), but this has
04471       // the benefit of being both really simple and much better than a libcall.
04472       SDValue Res;  // Logically starts equal to 1.0
04473       SDValue CurSquare = LHS;
04474       while (Val) {
04475         if (Val & 1) {
04476           if (Res.getNode())
04477             Res = DAG.getNode(ISD::FMUL, DL,Res.getValueType(), Res, CurSquare);
04478           else
04479             Res = CurSquare;  // 1.0*CurSquare.
04480         }
04481 
04482         CurSquare = DAG.getNode(ISD::FMUL, DL, CurSquare.getValueType(),
04483                                 CurSquare, CurSquare);
04484         Val >>= 1;
04485       }
04486 
04487       // If the original was negative, invert the result, producing 1/(x*x*x).
04488       if (RHSC->getSExtValue() < 0)
04489         Res = DAG.getNode(ISD::FDIV, DL, LHS.getValueType(),
04490                           DAG.getConstantFP(1.0, LHS.getValueType()), Res);
04491       return Res;
04492     }
04493   }
04494 
04495   // Otherwise, expand to a libcall.
04496   return DAG.getNode(ISD::FPOWI, DL, LHS.getValueType(), LHS, RHS);
04497 }
04498 
04499 // getTruncatedArgReg - Find underlying register used for an truncated
04500 // argument.
04501 static unsigned getTruncatedArgReg(const SDValue &N) {
04502   if (N.getOpcode() != ISD::TRUNCATE)
04503     return 0;
04504 
04505   const SDValue &Ext = N.getOperand(0);
04506   if (Ext.getOpcode() == ISD::AssertZext ||
04507       Ext.getOpcode() == ISD::AssertSext) {
04508     const SDValue &CFR = Ext.getOperand(0);
04509     if (CFR.getOpcode() == ISD::CopyFromReg)
04510       return cast<RegisterSDNode>(CFR.getOperand(1))->getReg();
04511     if (CFR.getOpcode() == ISD::TRUNCATE)
04512       return getTruncatedArgReg(CFR);
04513   }
04514   return 0;
04515 }
04516 
04517 /// EmitFuncArgumentDbgValue - If the DbgValueInst is a dbg_value of a function
04518 /// argument, create the corresponding DBG_VALUE machine instruction for it now.
04519 /// At the end of instruction selection, they will be inserted to the entry BB.
04520 bool SelectionDAGBuilder::EmitFuncArgumentDbgValue(const Value *V,
04521                                                    MDNode *Variable,
04522                                                    MDNode *Expr, int64_t Offset,
04523                                                    bool IsIndirect,
04524                                                    const SDValue &N) {
04525   const Argument *Arg = dyn_cast<Argument>(V);
04526   if (!Arg)
04527     return false;
04528 
04529   MachineFunction &MF = DAG.getMachineFunction();
04530   const TargetInstrInfo *TII = DAG.getSubtarget().getInstrInfo();
04531 
04532   // Ignore inlined function arguments here.
04533   DIVariable DV(Variable);
04534   if (DV.isInlinedFnArgument(MF.getFunction()))
04535     return false;
04536 
04537   Optional<MachineOperand> Op;
04538   // Some arguments' frame index is recorded during argument lowering.
04539   if (int FI = FuncInfo.getArgumentFrameIndex(Arg))
04540     Op = MachineOperand::CreateFI(FI);
04541 
04542   if (!Op && N.getNode()) {
04543     unsigned Reg;
04544     if (N.getOpcode() == ISD::CopyFromReg)
04545       Reg = cast<RegisterSDNode>(N.getOperand(1))->getReg();
04546     else
04547       Reg = getTruncatedArgReg(N);
04548     if (Reg && TargetRegisterInfo::isVirtualRegister(Reg)) {
04549       MachineRegisterInfo &RegInfo = MF.getRegInfo();
04550       unsigned PR = RegInfo.getLiveInPhysReg(Reg);
04551       if (PR)
04552         Reg = PR;
04553     }
04554     if (Reg)
04555       Op = MachineOperand::CreateReg(Reg, false);
04556   }
04557 
04558   if (!Op) {
04559     // Check if ValueMap has reg number.
04560     DenseMap<const Value *, unsigned>::iterator VMI = FuncInfo.ValueMap.find(V);
04561     if (VMI != FuncInfo.ValueMap.end())
04562       Op = MachineOperand::CreateReg(VMI->second, false);
04563   }
04564 
04565   if (!Op && N.getNode())
04566     // Check if frame index is available.
04567     if (LoadSDNode *LNode = dyn_cast<LoadSDNode>(N.getNode()))
04568       if (FrameIndexSDNode *FINode =
04569           dyn_cast<FrameIndexSDNode>(LNode->getBasePtr().getNode()))
04570         Op = MachineOperand::CreateFI(FINode->getIndex());
04571 
04572   if (!Op)
04573     return false;
04574 
04575   if (Op->isReg())
04576     FuncInfo.ArgDbgValues.push_back(
04577         BuildMI(MF, getCurDebugLoc(), TII->get(TargetOpcode::DBG_VALUE),
04578                 IsIndirect, Op->getReg(), Offset, Variable, Expr));
04579   else
04580     FuncInfo.ArgDbgValues.push_back(
04581         BuildMI(MF, getCurDebugLoc(), TII->get(TargetOpcode::DBG_VALUE))
04582             .addOperand(*Op)
04583             .addImm(Offset)
04584             .addMetadata(Variable)
04585             .addMetadata(Expr));
04586 
04587   return true;
04588 }
04589 
04590 // VisualStudio defines setjmp as _setjmp
04591 #if defined(_MSC_VER) && defined(setjmp) && \
04592                          !defined(setjmp_undefined_for_msvc)
04593 #  pragma push_macro("setjmp")
04594 #  undef setjmp
04595 #  define setjmp_undefined_for_msvc
04596 #endif
04597 
04598 /// visitIntrinsicCall - Lower the call to the specified intrinsic function.  If
04599 /// we want to emit this as a call to a named external function, return the name
04600 /// otherwise lower it and return null.
04601 const char *
04602 SelectionDAGBuilder::visitIntrinsicCall(const CallInst &I, unsigned Intrinsic) {
04603   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
04604   SDLoc sdl = getCurSDLoc();
04605   DebugLoc dl = getCurDebugLoc();
04606   SDValue Res;
04607 
04608   switch (Intrinsic) {
04609   default:
04610     // By default, turn this into a target intrinsic node.
04611     visitTargetIntrinsic(I, Intrinsic);
04612     return nullptr;
04613   case Intrinsic::vastart:  visitVAStart(I); return nullptr;
04614   case Intrinsic::vaend:    visitVAEnd(I); return nullptr;
04615   case Intrinsic::vacopy:   visitVACopy(I); return nullptr;
04616   case Intrinsic::returnaddress:
04617     setValue(&I, DAG.getNode(ISD::RETURNADDR, sdl, TLI.getPointerTy(),
04618                              getValue(I.getArgOperand(0))));
04619     return nullptr;
04620   case Intrinsic::frameaddress:
04621     setValue(&I, DAG.getNode(ISD::FRAMEADDR, sdl, TLI.getPointerTy(),
04622                              getValue(I.getArgOperand(0))));
04623     return nullptr;
04624   case Intrinsic::read_register: {
04625     Value *Reg = I.getArgOperand(0);
04626     SDValue RegName = DAG.getMDNode(cast<MDNode>(Reg));
04627     EVT VT = TLI.getValueType(I.getType());
04628     setValue(&I, DAG.getNode(ISD::READ_REGISTER, sdl, VT, RegName));
04629     return nullptr;
04630   }
04631   case Intrinsic::write_register: {
04632     Value *Reg = I.getArgOperand(0);
04633     Value *RegValue = I.getArgOperand(1);
04634     SDValue Chain = getValue(RegValue).getOperand(0);
04635     SDValue RegName = DAG.getMDNode(cast<MDNode>(Reg));
04636     DAG.setRoot(DAG.getNode(ISD::WRITE_REGISTER, sdl, MVT::Other, Chain,
04637                             RegName, getValue(RegValue)));
04638     return nullptr;
04639   }
04640   case Intrinsic::setjmp:
04641     return &"_setjmp"[!TLI.usesUnderscoreSetJmp()];
04642   case Intrinsic::longjmp:
04643     return &"_longjmp"[!TLI.usesUnderscoreLongJmp()];
04644   case Intrinsic::memcpy: {
04645     // Assert for address < 256 since we support only user defined address
04646     // spaces.
04647     assert(cast<PointerType>(I.getArgOperand(0)->getType())->getAddressSpace()
04648            < 256 &&
04649            cast<PointerType>(I.getArgOperand(1)->getType())->getAddressSpace()
04650            < 256 &&
04651            "Unknown address space");
04652     SDValue Op1 = getValue(I.getArgOperand(0));
04653     SDValue Op2 = getValue(I.getArgOperand(1));
04654     SDValue Op3 = getValue(I.getArgOperand(2));
04655     unsigned Align = cast<ConstantInt>(I.getArgOperand(3))->getZExtValue();
04656     if (!Align)
04657       Align = 1; // @llvm.memcpy defines 0 and 1 to both mean no alignment.
04658     bool isVol = cast<ConstantInt>(I.getArgOperand(4))->getZExtValue();
04659     DAG.setRoot(DAG.getMemcpy(getRoot(), sdl, Op1, Op2, Op3, Align, isVol, false,
04660                               MachinePointerInfo(I.getArgOperand(0)),
04661                               MachinePointerInfo(I.getArgOperand(1))));
04662     return nullptr;
04663   }
04664   case Intrinsic::memset: {
04665     // Assert for address < 256 since we support only user defined address
04666     // spaces.
04667     assert(cast<PointerType>(I.getArgOperand(0)->getType())->getAddressSpace()
04668            < 256 &&
04669            "Unknown address space");
04670     SDValue Op1 = getValue(I.getArgOperand(0));
04671     SDValue Op2 = getValue(I.getArgOperand(1));
04672     SDValue Op3 = getValue(I.getArgOperand(2));
04673     unsigned Align = cast<ConstantInt>(I.getArgOperand(3))->getZExtValue();
04674     if (!Align)
04675       Align = 1; // @llvm.memset defines 0 and 1 to both mean no alignment.
04676     bool isVol = cast<ConstantInt>(I.getArgOperand(4))->getZExtValue();
04677     DAG.setRoot(DAG.getMemset(getRoot(), sdl, Op1, Op2, Op3, Align, isVol,
04678                               MachinePointerInfo(I.getArgOperand(0))));
04679     return nullptr;
04680   }
04681   case Intrinsic::memmove: {
04682     // Assert for address < 256 since we support only user defined address
04683     // spaces.
04684     assert(cast<PointerType>(I.getArgOperand(0)->getType())->getAddressSpace()
04685            < 256 &&
04686            cast<PointerType>(I.getArgOperand(1)->getType())->getAddressSpace()
04687            < 256 &&
04688            "Unknown address space");
04689     SDValue Op1 = getValue(I.getArgOperand(0));
04690     SDValue Op2 = getValue(I.getArgOperand(1));
04691     SDValue Op3 = getValue(I.getArgOperand(2));
04692     unsigned Align = cast<ConstantInt>(I.getArgOperand(3))->getZExtValue();
04693     if (!Align)
04694       Align = 1; // @llvm.memmove defines 0 and 1 to both mean no alignment.
04695     bool isVol = cast<ConstantInt>(I.getArgOperand(4))->getZExtValue();
04696     DAG.setRoot(DAG.getMemmove(getRoot(), sdl, Op1, Op2, Op3, Align, isVol,
04697                                MachinePointerInfo(I.getArgOperand(0)),
04698                                MachinePointerInfo(I.getArgOperand(1))));
04699     return nullptr;
04700   }
04701   case Intrinsic::dbg_declare: {
04702     const DbgDeclareInst &DI = cast<DbgDeclareInst>(I);
04703     MDNode *Variable = DI.getVariable();
04704     MDNode *Expression = DI.getExpression();
04705     const Value *Address = DI.getAddress();
04706     DIVariable DIVar(Variable);
04707     assert((!DIVar || DIVar.isVariable()) &&
04708       "Variable in DbgDeclareInst should be either null or a DIVariable.");
04709     if (!Address || !DIVar) {
04710       DEBUG(dbgs() << "Dropping debug info for " << DI << "\n");
04711       return nullptr;
04712     }
04713 
04714     // Check if address has undef value.
04715     if (isa<UndefValue>(Address) ||
04716         (Address->use_empty() && !isa<Argument>(Address))) {
04717       DEBUG(dbgs() << "Dropping debug info for " << DI << "\n");
04718       return nullptr;
04719     }
04720 
04721     SDValue &N = NodeMap[Address];
04722     if (!N.getNode() && isa<Argument>(Address))
04723       // Check unused arguments map.
04724       N = UnusedArgNodeMap[Address];
04725     SDDbgValue *SDV;
04726     if (N.getNode()) {
04727       if (const BitCastInst *BCI = dyn_cast<BitCastInst>(Address))
04728         Address = BCI->getOperand(0);
04729       // Parameters are handled specially.
04730       bool isParameter =
04731         (DIVariable(Variable).getTag() == dwarf::DW_TAG_arg_variable ||
04732          isa<Argument>(Address));
04733 
04734       const AllocaInst *AI = dyn_cast<AllocaInst>(Address);
04735 
04736       if (isParameter && !AI) {
04737         FrameIndexSDNode *FINode = dyn_cast<FrameIndexSDNode>(N.getNode());
04738         if (FINode)
04739           // Byval parameter.  We have a frame index at this point.
04740           SDV = DAG.getFrameIndexDbgValue(
04741               Variable, Expression, FINode->getIndex(), 0, dl, SDNodeOrder);
04742         else {
04743           // Address is an argument, so try to emit its dbg value using
04744           // virtual register info from the FuncInfo.ValueMap.
04745           EmitFuncArgumentDbgValue(Address, Variable, Expression, 0, false, N);
04746           return nullptr;
04747         }
04748       } else if (AI)
04749         SDV = DAG.getDbgValue(Variable, Expression, N.getNode(), N.getResNo(),
04750                               true, 0, dl, SDNodeOrder);
04751       else {
04752         // Can't do anything with other non-AI cases yet.
04753         DEBUG(dbgs() << "Dropping debug info for " << DI << "\n");
04754         DEBUG(dbgs() << "non-AllocaInst issue for Address: \n\t");
04755         DEBUG(Address->dump());
04756         return nullptr;
04757       }
04758       DAG.AddDbgValue(SDV, N.getNode(), isParameter);
04759     } else {
04760       // If Address is an argument then try to emit its dbg value using
04761       // virtual register info from the FuncInfo.ValueMap.
04762       if (!EmitFuncArgumentDbgValue(Address, Variable, Expression, 0, false,
04763                                     N)) {
04764         // If variable is pinned by a alloca in dominating bb then
04765         // use StaticAllocaMap.
04766         if (const AllocaInst *AI = dyn_cast<AllocaInst>(Address)) {
04767           if (AI->getParent() != DI.getParent()) {
04768             DenseMap<const AllocaInst*, int>::iterator SI =
04769               FuncInfo.StaticAllocaMap.find(AI);
04770             if (SI != FuncInfo.StaticAllocaMap.end()) {
04771               SDV = DAG.getFrameIndexDbgValue(Variable, Expression, SI->second,
04772                                               0, dl, SDNodeOrder);
04773               DAG.AddDbgValue(SDV, nullptr, false);
04774               return nullptr;
04775             }
04776           }
04777         }
04778         DEBUG(dbgs() << "Dropping debug info for " << DI << "\n");
04779       }
04780     }
04781     return nullptr;
04782   }
04783   case Intrinsic::dbg_value: {
04784     const DbgValueInst &DI = cast<DbgValueInst>(I);
04785     DIVariable DIVar(DI.getVariable());
04786     assert((!DIVar || DIVar.isVariable()) &&
04787       "Variable in DbgValueInst should be either null or a DIVariable.");
04788     if (!DIVar)
04789       return nullptr;
04790 
04791     MDNode *Variable = DI.getVariable();
04792     MDNode *Expression = DI.getExpression();
04793     uint64_t Offset = DI.getOffset();
04794     const Value *V = DI.getValue();
04795     if (!V)
04796       return nullptr;
04797 
04798     SDDbgValue *SDV;
04799     if (isa<ConstantInt>(V) || isa<ConstantFP>(V) || isa<UndefValue>(V)) {
04800       SDV = DAG.getConstantDbgValue(Variable, Expression, V, Offset, dl,
04801                                     SDNodeOrder);
04802       DAG.AddDbgValue(SDV, nullptr, false);
04803     } else {
04804       // Do not use getValue() in here; we don't want to generate code at
04805       // this point if it hasn't been done yet.
04806       SDValue N = NodeMap[V];
04807       if (!N.getNode() && isa<Argument>(V))
04808         // Check unused arguments map.
04809         N = UnusedArgNodeMap[V];
04810       if (N.getNode()) {
04811         // A dbg.value for an alloca is always indirect.
04812         bool IsIndirect = isa<AllocaInst>(V) || Offset != 0;
04813         if (!EmitFuncArgumentDbgValue(V, Variable, Expression, Offset,
04814                                       IsIndirect, N)) {
04815           SDV = DAG.getDbgValue(Variable, Expression, N.getNode(), N.getResNo(),
04816                                 IsIndirect, Offset, dl, SDNodeOrder);
04817           DAG.AddDbgValue(SDV, N.getNode(), false);
04818         }
04819       } else if (!V->use_empty() ) {
04820         // Do not call getValue(V) yet, as we don't want to generate code.
04821         // Remember it for later.
04822         DanglingDebugInfo DDI(&DI, dl, SDNodeOrder);
04823         DanglingDebugInfoMap[V] = DDI;
04824       } else {
04825         // We may expand this to cover more cases.  One case where we have no
04826         // data available is an unreferenced parameter.
04827         DEBUG(dbgs() << "Dropping debug info for " << DI << "\n");
04828       }
04829     }
04830 
04831     // Build a debug info table entry.
04832     if (const BitCastInst *BCI = dyn_cast<BitCastInst>(V))
04833       V = BCI->getOperand(0);
04834     const AllocaInst *AI = dyn_cast<AllocaInst>(V);
04835     // Don't handle byval struct arguments or VLAs, for example.
04836     if (!AI) {
04837       DEBUG(dbgs() << "Dropping debug location info for:\n  " << DI << "\n");
04838       DEBUG(dbgs() << "  Last seen at:\n    " << *V << "\n");
04839       return nullptr;
04840     }
04841     DenseMap<const AllocaInst*, int>::iterator SI =
04842       FuncInfo.StaticAllocaMap.find(AI);
04843     if (SI == FuncInfo.StaticAllocaMap.end())
04844       return nullptr; // VLAs.
04845     return nullptr;
04846   }
04847 
04848   case Intrinsic::eh_typeid_for: {
04849     // Find the type id for the given typeinfo.
04850     GlobalValue *GV = ExtractTypeInfo(I.getArgOperand(0));
04851     unsigned TypeID = DAG.getMachineFunction().getMMI().getTypeIDFor(GV);
04852     Res = DAG.getConstant(TypeID, MVT::i32);
04853     setValue(&I, Res);
04854     return nullptr;
04855   }
04856 
04857   case Intrinsic::eh_return_i32:
04858   case Intrinsic::eh_return_i64:
04859     DAG.getMachineFunction().getMMI().setCallsEHReturn(true);
04860     DAG.setRoot(DAG.getNode(ISD::EH_RETURN, sdl,
04861                             MVT::Other,
04862                             getControlRoot(),
04863                             getValue(I.getArgOperand(0)),
04864                             getValue(I.getArgOperand(1))));
04865     return nullptr;
04866   case Intrinsic::eh_unwind_init:
04867     DAG.getMachineFunction().getMMI().setCallsUnwindInit(true);
04868     return nullptr;
04869   case Intrinsic::eh_dwarf_cfa: {
04870     SDValue CfaArg = DAG.getSExtOrTrunc(getValue(I.getArgOperand(0)), sdl,
04871                                         TLI.getPointerTy());
04872     SDValue Offset = DAG.getNode(ISD::ADD, sdl,
04873                                  CfaArg.getValueType(),
04874                                  DAG.getNode(ISD::FRAME_TO_ARGS_OFFSET, sdl,
04875                                              CfaArg.getValueType()),
04876                                  CfaArg);
04877     SDValue FA = DAG.getNode(ISD::FRAMEADDR, sdl, TLI.getPointerTy(),
04878                              DAG.getConstant(0, TLI.getPointerTy()));
04879     setValue(&I, DAG.getNode(ISD::ADD, sdl, FA.getValueType(),
04880                              FA, Offset));
04881     return nullptr;
04882   }
04883   case Intrinsic::eh_sjlj_callsite: {
04884     MachineModuleInfo &MMI = DAG.getMachineFunction().getMMI();
04885     ConstantInt *CI = dyn_cast<ConstantInt>(I.getArgOperand(0));
04886     assert(CI && "Non-constant call site value in eh.sjlj.callsite!");
04887     assert(MMI.getCurrentCallSite() == 0 && "Overlapping call sites!");
04888 
04889     MMI.setCurrentCallSite(CI->getZExtValue());
04890     return nullptr;
04891   }
04892   case Intrinsic::eh_sjlj_functioncontext: {
04893     // Get and store the index of the function context.
04894     MachineFrameInfo *MFI = DAG.getMachineFunction().getFrameInfo();
04895     AllocaInst *FnCtx =
04896       cast<AllocaInst>(I.getArgOperand(0)->stripPointerCasts());
04897     int FI = FuncInfo.StaticAllocaMap[FnCtx];
04898     MFI->setFunctionContextIndex(FI);
04899     return nullptr;
04900   }
04901   case Intrinsic::eh_sjlj_setjmp: {
04902     SDValue Ops[2];
04903     Ops[0] = getRoot();
04904     Ops[1] = getValue(I.getArgOperand(0));
04905     SDValue Op = DAG.getNode(ISD::EH_SJLJ_SETJMP, sdl,
04906                              DAG.getVTList(MVT::i32, MVT::Other), Ops);
04907     setValue(&I, Op.getValue(0));
04908     DAG.setRoot(Op.getValue(1));
04909     return nullptr;
04910   }
04911   case Intrinsic::eh_sjlj_longjmp: {
04912     DAG.setRoot(DAG.getNode(ISD::EH_SJLJ_LONGJMP, sdl, MVT::Other,
04913                             getRoot(), getValue(I.getArgOperand(0))));
04914     return nullptr;
04915   }
04916 
04917   case Intrinsic::x86_mmx_pslli_w:
04918   case Intrinsic::x86_mmx_pslli_d:
04919   case Intrinsic::x86_mmx_pslli_q:
04920   case Intrinsic::x86_mmx_psrli_w:
04921   case Intrinsic::x86_mmx_psrli_d:
04922   case Intrinsic::x86_mmx_psrli_q:
04923   case Intrinsic::x86_mmx_psrai_w:
04924   case Intrinsic::x86_mmx_psrai_d: {
04925     SDValue ShAmt = getValue(I.getArgOperand(1));
04926     if (isa<ConstantSDNode>(ShAmt)) {
04927       visitTargetIntrinsic(I, Intrinsic);
04928       return nullptr;
04929     }
04930     unsigned NewIntrinsic = 0;
04931     EVT ShAmtVT = MVT::v2i32;
04932     switch (Intrinsic) {
04933     case Intrinsic::x86_mmx_pslli_w:
04934       NewIntrinsic = Intrinsic::x86_mmx_psll_w;
04935       break;
04936     case Intrinsic::x86_mmx_pslli_d:
04937       NewIntrinsic = Intrinsic::x86_mmx_psll_d;
04938       break;
04939     case Intrinsic::x86_mmx_pslli_q:
04940       NewIntrinsic = Intrinsic::x86_mmx_psll_q;
04941       break;
04942     case Intrinsic::x86_mmx_psrli_w:
04943       NewIntrinsic = Intrinsic::x86_mmx_psrl_w;
04944       break;
04945     case Intrinsic::x86_mmx_psrli_d:
04946       NewIntrinsic = Intrinsic::x86_mmx_psrl_d;
04947       break;
04948     case Intrinsic::x86_mmx_psrli_q:
04949       NewIntrinsic = Intrinsic::x86_mmx_psrl_q;
04950       break;
04951     case Intrinsic::x86_mmx_psrai_w:
04952       NewIntrinsic = Intrinsic::x86_mmx_psra_w;
04953       break;
04954     case Intrinsic::x86_mmx_psrai_d:
04955       NewIntrinsic = Intrinsic::x86_mmx_psra_d;
04956       break;
04957     default: llvm_unreachable("Impossible intrinsic");  // Can't reach here.
04958     }
04959 
04960     // The vector shift intrinsics with scalars uses 32b shift amounts but
04961     // the sse2/mmx shift instructions reads 64 bits. Set the upper 32 bits
04962     // to be zero.
04963     // We must do this early because v2i32 is not a legal type.
04964     SDValue ShOps[2];
04965     ShOps[0] = ShAmt;
04966     ShOps[1] = DAG.getConstant(0, MVT::i32);
04967     ShAmt =  DAG.getNode(ISD::BUILD_VECTOR, sdl, ShAmtVT, ShOps);
04968     EVT DestVT = TLI.getValueType(I.getType());
04969     ShAmt = DAG.getNode(ISD::BITCAST, sdl, DestVT, ShAmt);
04970     Res = DAG.getNode(ISD::INTRINSIC_WO_CHAIN, sdl, DestVT,
04971                        DAG.getConstant(NewIntrinsic, MVT::i32),
04972                        getValue(I.getArgOperand(0)), ShAmt);
04973     setValue(&I, Res);
04974     return nullptr;
04975   }
04976   case Intrinsic::x86_avx_vinsertf128_pd_256:
04977   case Intrinsic::x86_avx_vinsertf128_ps_256:
04978   case Intrinsic::x86_avx_vinsertf128_si_256:
04979   case Intrinsic::x86_avx2_vinserti128: {
04980     EVT DestVT = TLI.getValueType(I.getType());
04981     EVT ElVT = TLI.getValueType(I.getArgOperand(1)->getType());
04982     uint64_t Idx = (cast<ConstantInt>(I.getArgOperand(2))->getZExtValue() & 1) *
04983                    ElVT.getVectorNumElements();
04984     Res =
04985         DAG.getNode(ISD::INSERT_SUBVECTOR, sdl, DestVT,
04986                     getValue(I.getArgOperand(0)), getValue(I.getArgOperand(1)),
04987                     DAG.getConstant(Idx, TLI.getVectorIdxTy()));
04988     setValue(&I, Res);
04989     return nullptr;
04990   }
04991   case Intrinsic::x86_avx_vextractf128_pd_256:
04992   case Intrinsic::x86_avx_vextractf128_ps_256:
04993   case Intrinsic::x86_avx_vextractf128_si_256:
04994   case Intrinsic::x86_avx2_vextracti128: {
04995     EVT DestVT = TLI.getValueType(I.getType());
04996     uint64_t Idx = (cast<ConstantInt>(I.getArgOperand(1))->getZExtValue() & 1) *
04997                    DestVT.getVectorNumElements();
04998     Res = DAG.getNode(ISD::EXTRACT_SUBVECTOR, sdl, DestVT,
04999                       getValue(I.getArgOperand(0)),
05000                       DAG.getConstant(Idx, TLI.getVectorIdxTy()));
05001     setValue(&I, Res);
05002     return nullptr;
05003   }
05004   case Intrinsic::convertff:
05005   case Intrinsic::convertfsi:
05006   case Intrinsic::convertfui:
05007   case Intrinsic::convertsif:
05008   case Intrinsic::convertuif:
05009   case Intrinsic::convertss:
05010   case Intrinsic::convertsu:
05011   case Intrinsic::convertus:
05012   case Intrinsic::convertuu: {
05013     ISD::CvtCode Code = ISD::CVT_INVALID;
05014     switch (Intrinsic) {
05015     default: llvm_unreachable("Impossible intrinsic");  // Can't reach here.
05016     case Intrinsic::convertff:  Code = ISD::CVT_FF; break;
05017     case Intrinsic::convertfsi: Code = ISD::CVT_FS; break;
05018     case Intrinsic::convertfui: Code = ISD::CVT_FU; break;
05019     case Intrinsic::convertsif: Code = ISD::CVT_SF; break;
05020     case Intrinsic::convertuif: Code = ISD::CVT_UF; break;
05021     case Intrinsic::convertss:  Code = ISD::CVT_SS; break;
05022     case Intrinsic::convertsu:  Code = ISD::CVT_SU; break;
05023     case Intrinsic::convertus:  Code = ISD::CVT_US; break;
05024     case Intrinsic::convertuu:  Code = ISD::CVT_UU; break;
05025     }
05026     EVT DestVT = TLI.getValueType(I.getType());
05027     const Value *Op1 = I.getArgOperand(0);
05028     Res = DAG.getConvertRndSat(DestVT, sdl, getValue(Op1),
05029                                DAG.getValueType(DestVT),
05030                                DAG.getValueType(getValue(Op1).getValueType()),
05031                                getValue(I.getArgOperand(1)),
05032                                getValue(I.getArgOperand(2)),
05033                                Code);
05034     setValue(&I, Res);
05035     return nullptr;
05036   }
05037   case Intrinsic::powi:
05038     setValue(&I, ExpandPowI(sdl, getValue(I.getArgOperand(0)),
05039                             getValue(I.getArgOperand(1)), DAG));
05040     return nullptr;
05041   case Intrinsic::log:
05042     setValue(&I, expandLog(sdl, getValue(I.getArgOperand(0)), DAG, TLI));
05043     return nullptr;
05044   case Intrinsic::log2:
05045     setValue(&I, expandLog2(sdl, getValue(I.getArgOperand(0)), DAG, TLI));
05046     return nullptr;
05047   case Intrinsic::log10:
05048     setValue(&I, expandLog10(sdl, getValue(I.getArgOperand(0)), DAG, TLI));
05049     return nullptr;
05050   case Intrinsic::exp:
05051     setValue(&I, expandExp(sdl, getValue(I.getArgOperand(0)), DAG, TLI));
05052     return nullptr;
05053   case Intrinsic::exp2:
05054     setValue(&I, expandExp2(sdl, getValue(I.getArgOperand(0)), DAG, TLI));
05055     return nullptr;
05056   case Intrinsic::pow:
05057     setValue(&I, expandPow(sdl, getValue(I.getArgOperand(0)),
05058                            getValue(I.getArgOperand(1)), DAG, TLI));
05059     return nullptr;
05060   case Intrinsic::sqrt:
05061   case Intrinsic::fabs:
05062   case Intrinsic::sin:
05063   case Intrinsic::cos:
05064   case Intrinsic::floor:
05065   case Intrinsic::ceil:
05066   case Intrinsic::trunc:
05067   case Intrinsic::rint:
05068   case Intrinsic::nearbyint:
05069   case Intrinsic::round: {
05070     unsigned Opcode;
05071     switch (Intrinsic) {
05072     default: llvm_unreachable("Impossible intrinsic");  // Can't reach here.
05073     case Intrinsic::sqrt:      Opcode = ISD::FSQRT;      break;
05074     case Intrinsic::fabs:      Opcode = ISD::FABS;       break;
05075     case Intrinsic::sin:       Opcode = ISD::FSIN;       break;
05076     case Intrinsic::cos:       Opcode = ISD::FCOS;       break;
05077     case Intrinsic::floor:     Opcode = ISD::FFLOOR;     break;
05078     case Intrinsic::ceil:      Opcode = ISD::FCEIL;      break;
05079     case Intrinsic::trunc:     Opcode = ISD::FTRUNC;     break;
05080     case Intrinsic::rint:      Opcode = ISD::FRINT;      break;
05081     case Intrinsic::nearbyint: Opcode = ISD::FNEARBYINT; break;
05082     case Intrinsic::round:     Opcode = ISD::FROUND;     break;
05083     }
05084 
05085     setValue(&I, DAG.getNode(Opcode, sdl,
05086                              getValue(I.getArgOperand(0)).getValueType(),
05087                              getValue(I.getArgOperand(0))));
05088     return nullptr;
05089   }
05090   case Intrinsic::minnum:
05091     setValue(&I, DAG.getNode(ISD::FMINNUM, sdl,
05092                              getValue(I.getArgOperand(0)).getValueType(),
05093                              getValue(I.getArgOperand(0)),
05094                              getValue(I.getArgOperand(1))));
05095     return nullptr;
05096   case Intrinsic::maxnum:
05097     setValue(&I, DAG.getNode(ISD::FMAXNUM, sdl,
05098                              getValue(I.getArgOperand(0)).getValueType(),
05099                              getValue(I.getArgOperand(0)),
05100                              getValue(I.getArgOperand(1))));
05101     return nullptr;
05102   case Intrinsic::copysign:
05103     setValue(&I, DAG.getNode(ISD::FCOPYSIGN, sdl,
05104                              getValue(I.getArgOperand(0)).getValueType(),
05105                              getValue(I.getArgOperand(0)),
05106                              getValue(I.getArgOperand(1))));
05107     return nullptr;
05108   case Intrinsic::fma:
05109     setValue(&I, DAG.getNode(ISD::FMA, sdl,
05110                              getValue(I.getArgOperand(0)).getValueType(),
05111                              getValue(I.getArgOperand(0)),
05112                              getValue(I.getArgOperand(1)),
05113                              getValue(I.getArgOperand(2))));
05114     return nullptr;
05115   case Intrinsic::fmuladd: {
05116     EVT VT = TLI.getValueType(I.getType());
05117     if (TM.Options.AllowFPOpFusion != FPOpFusion::Strict &&
05118         TLI.isFMAFasterThanFMulAndFAdd(VT)) {
05119       setValue(&I, DAG.getNode(ISD::FMA, sdl,
05120                                getValue(I.getArgOperand(0)).getValueType(),
05121                                getValue(I.getArgOperand(0)),
05122                                getValue(I.getArgOperand(1)),
05123                                getValue(I.getArgOperand(2))));
05124     } else {
05125       SDValue Mul = DAG.getNode(ISD::FMUL, sdl,
05126                                 getValue(I.getArgOperand(0)).getValueType(),
05127                                 getValue(I.getArgOperand(0)),
05128                                 getValue(I.getArgOperand(1)));
05129       SDValue Add = DAG.getNode(ISD::FADD, sdl,
05130                                 getValue(I.getArgOperand(0)).getValueType(),
05131                                 Mul,
05132                                 getValue(I.getArgOperand(2)));
05133       setValue(&I, Add);
05134     }
05135     return nullptr;
05136   }
05137   case Intrinsic::convert_to_fp16:
05138     setValue(&I, DAG.getNode(ISD::BITCAST, sdl, MVT::i16,
05139                              DAG.getNode(ISD::FP_ROUND, sdl, MVT::f16,
05140                                          getValue(I.getArgOperand(0)),
05141                                          DAG.getTargetConstant(0, MVT::i32))));
05142     return nullptr;
05143   case Intrinsic::convert_from_fp16:
05144     setValue(&I,
05145              DAG.getNode(ISD::FP_EXTEND, sdl, TLI.getValueType(I.getType()),
05146                          DAG.getNode(ISD::BITCAST, sdl, MVT::f16,
05147                                      getValue(I.getArgOperand(0)))));
05148     return nullptr;
05149   case Intrinsic::pcmarker: {
05150     SDValue Tmp = getValue(I.getArgOperand(0));
05151     DAG.setRoot(DAG.getNode(ISD::PCMARKER, sdl, MVT::Other, getRoot(), Tmp));
05152     return nullptr;
05153   }
05154   case Intrinsic::readcyclecounter: {
05155     SDValue Op = getRoot();
05156     Res = DAG.getNode(ISD::READCYCLECOUNTER, sdl,
05157                       DAG.getVTList(MVT::i64, MVT::Other), Op);
05158     setValue(&I, Res);
05159     DAG.setRoot(Res.getValue(1));
05160     return nullptr;
05161   }
05162   case Intrinsic::bswap:
05163     setValue(&I, DAG.getNode(ISD::BSWAP, sdl,
05164                              getValue(I.getArgOperand(0)).getValueType(),
05165                              getValue(I.getArgOperand(0))));
05166     return nullptr;
05167   case Intrinsic::cttz: {
05168     SDValue Arg = getValue(I.getArgOperand(0));
05169     ConstantInt *CI = cast<ConstantInt>(I.getArgOperand(1));
05170     EVT Ty = Arg.getValueType();
05171     setValue(&I, DAG.getNode(CI->isZero() ? ISD::CTTZ : ISD::CTTZ_ZERO_UNDEF,
05172                              sdl, Ty, Arg));
05173     return nullptr;
05174   }
05175   case Intrinsic::ctlz: {
05176     SDValue Arg = getValue(I.getArgOperand(0));
05177     ConstantInt *CI = cast<ConstantInt>(I.getArgOperand(1));
05178     EVT Ty = Arg.getValueType();
05179     setValue(&I, DAG.getNode(CI->isZero() ? ISD::CTLZ : ISD::CTLZ_ZERO_UNDEF,
05180                              sdl, Ty, Arg));
05181     return nullptr;
05182   }
05183   case Intrinsic::ctpop: {
05184     SDValue Arg = getValue(I.getArgOperand(0));
05185     EVT Ty = Arg.getValueType();
05186     setValue(&I, DAG.getNode(ISD::CTPOP, sdl, Ty, Arg));
05187     return nullptr;
05188   }
05189   case Intrinsic::stacksave: {
05190     SDValue Op = getRoot();
05191     Res = DAG.getNode(ISD::STACKSAVE, sdl,
05192                       DAG.getVTList(TLI.getPointerTy(), MVT::Other), Op);
05193     setValue(&I, Res);
05194     DAG.setRoot(Res.getValue(1));
05195     return nullptr;
05196   }
05197   case Intrinsic::stackrestore: {
05198     Res = getValue(I.getArgOperand(0));
05199     DAG.setRoot(DAG.getNode(ISD::STACKRESTORE, sdl, MVT::Other, getRoot(), Res));
05200     return nullptr;
05201   }
05202   case Intrinsic::stackprotector: {
05203     // Emit code into the DAG to store the stack guard onto the stack.
05204     MachineFunction &MF = DAG.getMachineFunction();
05205     MachineFrameInfo *MFI = MF.getFrameInfo();
05206     EVT PtrTy = TLI.getPointerTy();
05207     SDValue Src, Chain = getRoot();
05208     const Value *Ptr = cast<LoadInst>(I.getArgOperand(0))->getPointerOperand();
05209     const GlobalVariable *GV = dyn_cast<GlobalVariable>(Ptr);
05210 
05211     // See if Ptr is a bitcast. If it is, look through it and see if we can get
05212     // global variable __stack_chk_guard.
05213     if (!GV)
05214       if (const Operator *BC = dyn_cast<Operator>(Ptr))
05215         if (BC->getOpcode() == Instruction::BitCast)
05216           GV = dyn_cast<GlobalVariable>(BC->getOperand(0));
05217 
05218     if (GV && TLI.useLoadStackGuardNode()) {
05219       // Emit a LOAD_STACK_GUARD node.
05220       MachineSDNode *Node = DAG.getMachineNode(TargetOpcode::LOAD_STACK_GUARD,
05221                                                sdl, PtrTy, Chain);
05222       MachinePointerInfo MPInfo(GV);
05223       MachineInstr::mmo_iterator MemRefs = MF.allocateMemRefsArray(1);
05224       unsigned Flags = MachineMemOperand::MOLoad |
05225                        MachineMemOperand::MOInvariant;
05226       *MemRefs = MF.getMachineMemOperand(MPInfo, Flags,
05227                                          PtrTy.getSizeInBits() / 8,
05228                                          DAG.getEVTAlignment(PtrTy));
05229       Node->setMemRefs(MemRefs, MemRefs + 1);
05230 
05231       // Copy the guard value to a virtual register so that it can be
05232       // retrieved in the epilogue.
05233       Src = SDValue(Node, 0);
05234       const TargetRegisterClass *RC =
05235           TLI.getRegClassFor(Src.getSimpleValueType());
05236       unsigned Reg = MF.getRegInfo().createVirtualRegister(RC);
05237 
05238       SPDescriptor.setGuardReg(Reg);
05239       Chain = DAG.getCopyToReg(Chain, sdl, Reg, Src);
05240     } else {
05241       Src = getValue(I.getArgOperand(0));   // The guard's value.
05242     }
05243 
05244     AllocaInst *Slot = cast<AllocaInst>(I.getArgOperand(1));
05245 
05246     int FI = FuncInfo.StaticAllocaMap[Slot];
05247     MFI->setStackProtectorIndex(FI);
05248 
05249     SDValue FIN = DAG.getFrameIndex(FI, PtrTy);
05250 
05251     // Store the stack protector onto the stack.
05252     Res = DAG.getStore(Chain, sdl, Src, FIN,
05253                        MachinePointerInfo::getFixedStack(FI),
05254                        true, false, 0);
05255     setValue(&I, Res);
05256     DAG.setRoot(Res);
05257     return nullptr;
05258   }
05259   case Intrinsic::objectsize: {
05260     // If we don't know by now, we're never going to know.
05261     ConstantInt *CI = dyn_cast<ConstantInt>(I.getArgOperand(1));
05262 
05263     assert(CI && "Non-constant type in __builtin_object_size?");
05264 
05265     SDValue Arg = getValue(I.getCalledValue());
05266     EVT Ty = Arg.getValueType();
05267 
05268     if (CI->isZero())
05269       Res = DAG.getConstant(-1ULL, Ty);
05270     else
05271       Res = DAG.getConstant(0, Ty);
05272 
05273     setValue(&I, Res);
05274     return nullptr;
05275   }
05276   case Intrinsic::annotation:
05277   case Intrinsic::ptr_annotation:
05278     // Drop the intrinsic, but forward the value
05279     setValue(&I, getValue(I.getOperand(0)));
05280     return nullptr;
05281   case Intrinsic::assume:
05282   case Intrinsic::var_annotation:
05283     // Discard annotate attributes and assumptions
05284     return nullptr;
05285 
05286   case Intrinsic::init_trampoline: {
05287     const Function *F = cast<Function>(I.getArgOperand(1)->stripPointerCasts());
05288 
05289     SDValue Ops[6];
05290     Ops[0] = getRoot();
05291     Ops[1] = getValue(I.getArgOperand(0));
05292     Ops[2] = getValue(I.getArgOperand(1));
05293     Ops[3] = getValue(I.getArgOperand(2));
05294     Ops[4] = DAG.getSrcValue(I.getArgOperand(0));
05295     Ops[5] = DAG.getSrcValue(F);
05296 
05297     Res = DAG.getNode(ISD::INIT_TRAMPOLINE, sdl, MVT::Other, Ops);
05298 
05299     DAG.setRoot(Res);
05300     return nullptr;
05301   }
05302   case Intrinsic::adjust_trampoline: {
05303     setValue(&I, DAG.getNode(ISD::ADJUST_TRAMPOLINE, sdl,
05304                              TLI.getPointerTy(),
05305                              getValue(I.getArgOperand(0))));
05306     return nullptr;
05307   }
05308   case Intrinsic::gcroot:
05309     if (GFI) {
05310       const Value *Alloca = I.getArgOperand(0)->stripPointerCasts();
05311       const Constant *TypeMap = cast<Constant>(I.getArgOperand(1));
05312 
05313       FrameIndexSDNode *FI = cast<FrameIndexSDNode>(getValue(Alloca).getNode());
05314       GFI->addStackRoot(FI->getIndex(), TypeMap);
05315     }
05316     return nullptr;
05317   case Intrinsic::gcread:
05318   case Intrinsic::gcwrite:
05319     llvm_unreachable("GC failed to lower gcread/gcwrite intrinsics!");
05320   case Intrinsic::flt_rounds:
05321     setValue(&I, DAG.getNode(ISD::FLT_ROUNDS_, sdl, MVT::i32));
05322     return nullptr;
05323 
05324   case Intrinsic::expect: {
05325     // Just replace __builtin_expect(exp, c) with EXP.
05326     setValue(&I, getValue(I.getArgOperand(0)));
05327     return nullptr;
05328   }
05329 
05330   case Intrinsic::debugtrap:
05331   case Intrinsic::trap: {
05332     StringRef TrapFuncName = TM.Options.getTrapFunctionName();
05333     if (TrapFuncName.empty()) {
05334       ISD::NodeType Op = (Intrinsic == Intrinsic::trap) ?
05335         ISD::TRAP : ISD::DEBUGTRAP;
05336       DAG.setRoot(DAG.getNode(Op, sdl,MVT::Other, getRoot()));
05337       return nullptr;
05338     }
05339     TargetLowering::ArgListTy Args;
05340 
05341     TargetLowering::CallLoweringInfo CLI(DAG);
05342     CLI.setDebugLoc(sdl).setChain(getRoot())
05343       .setCallee(CallingConv::C, I.getType(),
05344                  DAG.getExternalSymbol(TrapFuncName.data(), TLI.getPointerTy()),
05345                  std::move(Args), 0);
05346 
05347     std::pair<SDValue, SDValue> Result = TLI.LowerCallTo(CLI);
05348     DAG.setRoot(Result.second);
05349     return nullptr;
05350   }
05351 
05352   case Intrinsic::uadd_with_overflow:
05353   case Intrinsic::sadd_with_overflow:
05354   case Intrinsic::usub_with_overflow:
05355   case Intrinsic::ssub_with_overflow:
05356   case Intrinsic::umul_with_overflow:
05357   case Intrinsic::smul_with_overflow: {
05358     ISD::NodeType Op;
05359     switch (Intrinsic) {
05360     default: llvm_unreachable("Impossible intrinsic");  // Can't reach here.
05361     case Intrinsic::uadd_with_overflow: Op = ISD::UADDO; break;
05362     case Intrinsic::sadd_with_overflow: Op = ISD::SADDO; break;
05363     case Intrinsic::usub_with_overflow: Op = ISD::USUBO; break;
05364     case Intrinsic::ssub_with_overflow: Op = ISD::SSUBO; break;
05365     case Intrinsic::umul_with_overflow: Op = ISD::UMULO; break;
05366     case Intrinsic::smul_with_overflow: Op = ISD::SMULO; break;
05367     }
05368     SDValue Op1 = getValue(I.getArgOperand(0));
05369     SDValue Op2 = getValue(I.getArgOperand(1));
05370 
05371     SDVTList VTs = DAG.getVTList(Op1.getValueType(), MVT::i1);
05372     setValue(&I, DAG.getNode(Op, sdl, VTs, Op1, Op2));
05373     return nullptr;
05374   }
05375   case Intrinsic::prefetch: {
05376     SDValue Ops[5];
05377     unsigned rw = cast<ConstantInt>(I.getArgOperand(1))->getZExtValue();
05378     Ops[0] = getRoot();
05379     Ops[1] = getValue(I.getArgOperand(0));
05380     Ops[2] = getValue(I.getArgOperand(1));
05381     Ops[3] = getValue(I.getArgOperand(2));
05382     Ops[4] = getValue(I.getArgOperand(3));
05383     DAG.setRoot(DAG.getMemIntrinsicNode(ISD::PREFETCH, sdl,
05384                                         DAG.getVTList(MVT::Other), Ops,
05385                                         EVT::getIntegerVT(*Context, 8),
05386                                         MachinePointerInfo(I.getArgOperand(0)),
05387                                         0, /* align */
05388                                         false, /* volatile */
05389                                         rw==0, /* read */
05390                                         rw==1)); /* write */
05391     return nullptr;
05392   }
05393   case Intrinsic::lifetime_start:
05394   case Intrinsic::lifetime_end: {
05395     bool IsStart = (Intrinsic == Intrinsic::lifetime_start);
05396     // Stack coloring is not enabled in O0, discard region information.
05397     if (TM.getOptLevel() == CodeGenOpt::None)
05398       return nullptr;
05399 
05400     SmallVector<Value *, 4> Allocas;
05401     GetUnderlyingObjects(I.getArgOperand(1), Allocas, DL);
05402 
05403     for (SmallVectorImpl<Value*>::iterator Object = Allocas.begin(),
05404            E = Allocas.end(); Object != E; ++Object) {
05405       AllocaInst *LifetimeObject = dyn_cast_or_null<AllocaInst>(*Object);
05406 
05407       // Could not find an Alloca.
05408       if (!LifetimeObject)
05409         continue;
05410 
05411       // First check that the Alloca is static, otherwise it won't have a
05412       // valid frame index.
05413       auto SI = FuncInfo.StaticAllocaMap.find(LifetimeObject);
05414       if (SI == FuncInfo.StaticAllocaMap.end())
05415         return nullptr;
05416 
05417       int FI = SI->second;
05418 
05419       SDValue Ops[2];
05420       Ops[0] = getRoot();
05421       Ops[1] = DAG.getFrameIndex(FI, TLI.getPointerTy(), true);
05422       unsigned Opcode = (IsStart ? ISD::LIFETIME_START : ISD::LIFETIME_END);
05423 
05424       Res = DAG.getNode(Opcode, sdl, MVT::Other, Ops);
05425       DAG.setRoot(Res);
05426     }
05427     return nullptr;
05428   }
05429   case Intrinsic::invariant_start:
05430     // Discard region information.
05431     setValue(&I, DAG.getUNDEF(TLI.getPointerTy()));
05432     return nullptr;
05433   case Intrinsic::invariant_end:
05434     // Discard region information.
05435     return nullptr;
05436   case Intrinsic::stackprotectorcheck: {
05437     // Do not actually emit anything for this basic block. Instead we initialize
05438     // the stack protector descriptor and export the guard variable so we can
05439     // access it in FinishBasicBlock.
05440     const BasicBlock *BB = I.getParent();
05441     SPDescriptor.initialize(BB, FuncInfo.MBBMap[BB], I);
05442     ExportFromCurrentBlock(SPDescriptor.getGuard());
05443 
05444     // Flush our exports since we are going to process a terminator.
05445     (void)getControlRoot();
05446     return nullptr;
05447   }
05448   case Intrinsic::clear_cache:
05449     return TLI.getClearCacheBuiltinName();
05450   case Intrinsic::donothing:
05451     // ignore
05452     return nullptr;
05453   case Intrinsic::experimental_stackmap: {
05454     visitStackmap(I);
05455     return nullptr;
05456   }
05457   case Intrinsic::experimental_patchpoint_void:
05458   case Intrinsic::experimental_patchpoint_i64: {
05459     visitPatchpoint(&I);
05460     return nullptr;
05461   }
05462   }
05463 }
05464 
05465 std::pair<SDValue, SDValue>
05466 SelectionDAGBuilder::lowerInvokable(TargetLowering::CallLoweringInfo &CLI,
05467                                     MachineBasicBlock *LandingPad) {
05468   MachineModuleInfo &MMI = DAG.getMachineFunction().getMMI();
05469   MCSymbol *BeginLabel = nullptr;
05470 
05471   if (LandingPad) {
05472     // Insert a label before the invoke call to mark the try range.  This can be
05473     // used to detect deletion of the invoke via the MachineModuleInfo.
05474     BeginLabel = MMI.getContext().CreateTempSymbol();
05475 
05476     // For SjLj, keep track of which landing pads go with which invokes
05477     // so as to maintain the ordering of pads in the LSDA.
05478     unsigned CallSiteIndex = MMI.getCurrentCallSite();
05479     if (CallSiteIndex) {
05480       MMI.setCallSiteBeginLabel(BeginLabel, CallSiteIndex);
05481       LPadToCallSiteMap[LandingPad].push_back(CallSiteIndex);
05482 
05483       // Now that the call site is handled, stop tracking it.
05484       MMI.setCurrentCallSite(0);
05485     }
05486 
05487     // Both PendingLoads and PendingExports must be flushed here;
05488     // this call might not return.
05489     (void)getRoot();
05490     DAG.setRoot(DAG.getEHLabel(getCurSDLoc(), getControlRoot(), BeginLabel));
05491 
05492     CLI.setChain(getRoot());
05493   }
05494 
05495   const TargetLowering *TLI = TM.getSubtargetImpl()->getTargetLowering();
05496   std::pair<SDValue, SDValue> Result = TLI->LowerCallTo(CLI);
05497 
05498   assert((CLI.IsTailCall || Result.second.getNode()) &&
05499          "Non-null chain expected with non-tail call!");
05500   assert((Result.second.getNode() || !Result.first.getNode()) &&
05501          "Null value expected with tail call!");
05502 
05503   if (!Result.second.getNode()) {
05504     // As a special case, a null chain means that a tail call has been emitted
05505     // and the DAG root is already updated.
05506     HasTailCall = true;
05507 
05508     // Since there's no actual continuation from this block, nothing can be
05509     // relying on us setting vregs for them.
05510     PendingExports.clear();
05511   } else {
05512     DAG.setRoot(Result.second);
05513   }
05514 
05515   if (LandingPad) {
05516     // Insert a label at the end of the invoke call to mark the try range.  This
05517     // can be used to detect deletion of the invoke via the MachineModuleInfo.
05518     MCSymbol *EndLabel = MMI.getContext().CreateTempSymbol();
05519     DAG.setRoot(DAG.getEHLabel(getCurSDLoc(), getRoot(), EndLabel));
05520 
05521     // Inform MachineModuleInfo of range.
05522     MMI.addInvoke(LandingPad, BeginLabel, EndLabel);
05523   }
05524 
05525   return Result;
05526 }
05527 
05528 void SelectionDAGBuilder::LowerCallTo(ImmutableCallSite CS, SDValue Callee,
05529                                       bool isTailCall,
05530                                       MachineBasicBlock *LandingPad) {
05531   PointerType *PT = cast<PointerType>(CS.getCalledValue()->getType());
05532   FunctionType *FTy = cast<FunctionType>(PT->getElementType());
05533   Type *RetTy = FTy->getReturnType();
05534 
05535   TargetLowering::ArgListTy Args;
05536   TargetLowering::ArgListEntry Entry;
05537   Args.reserve(CS.arg_size());
05538 
05539   for (ImmutableCallSite::arg_iterator i = CS.arg_begin(), e = CS.arg_end();
05540        i != e; ++i) {
05541     const Value *V = *i;
05542 
05543     // Skip empty types
05544     if (V->getType()->isEmptyTy())
05545       continue;
05546 
05547     SDValue ArgNode = getValue(V);
05548     Entry.Node = ArgNode; Entry.Ty = V->getType();
05549 
05550     // Skip the first return-type Attribute to get to params.
05551     Entry.setAttributes(&CS, i - CS.arg_begin() + 1);
05552     Args.push_back(Entry);
05553   }
05554 
05555   // Check if target-independent constraints permit a tail call here.
05556   // Target-dependent constraints are checked within TLI->LowerCallTo.
05557   if (isTailCall && !isInTailCallPosition(CS, DAG.getTarget()))
05558     isTailCall = false;
05559 
05560   TargetLowering::CallLoweringInfo CLI(DAG);
05561   CLI.setDebugLoc(getCurSDLoc()).setChain(getRoot())
05562     .setCallee(RetTy, FTy, Callee, std::move(Args), CS)
05563     .setTailCall(isTailCall);
05564   std::pair<SDValue,SDValue> Result = lowerInvokable(CLI, LandingPad);
05565 
05566   if (Result.first.getNode())
05567     setValue(CS.getInstruction(), Result.first);
05568 }
05569 
05570 /// IsOnlyUsedInZeroEqualityComparison - Return true if it only matters that the
05571 /// value is equal or not-equal to zero.
05572 static bool IsOnlyUsedInZeroEqualityComparison(const Value *V) {
05573   for (const User *U : V->users()) {
05574     if (const ICmpInst *IC = dyn_cast<ICmpInst>(U))
05575       if (IC->isEquality())
05576         if (const Constant *C = dyn_cast<Constant>(IC->getOperand(1)))
05577           if (C->isNullValue())
05578             continue;
05579     // Unknown instruction.
05580     return false;
05581   }
05582   return true;
05583 }
05584 
05585 static SDValue getMemCmpLoad(const Value *PtrVal, MVT LoadVT,
05586                              Type *LoadTy,
05587                              SelectionDAGBuilder &Builder) {
05588 
05589   // Check to see if this load can be trivially constant folded, e.g. if the
05590   // input is from a string literal.
05591   if (const Constant *LoadInput = dyn_cast<Constant>(PtrVal)) {
05592     // Cast pointer to the type we really want to load.
05593     LoadInput = ConstantExpr::getBitCast(const_cast<Constant *>(LoadInput),
05594                                          PointerType::getUnqual(LoadTy));
05595 
05596     if (const Constant *LoadCst =
05597           ConstantFoldLoadFromConstPtr(const_cast<Constant *>(LoadInput),
05598                                        Builder.DL))
05599       return Builder.getValue(LoadCst);
05600   }
05601 
05602   // Otherwise, we have to emit the load.  If the pointer is to unfoldable but
05603   // still constant memory, the input chain can be the entry node.
05604   SDValue Root;
05605   bool ConstantMemory = false;
05606 
05607   // Do not serialize (non-volatile) loads of constant memory with anything.
05608   if (Builder.AA->pointsToConstantMemory(PtrVal)) {
05609     Root = Builder.DAG.getEntryNode();
05610     ConstantMemory = true;
05611   } else {
05612     // Do not serialize non-volatile loads against each other.
05613     Root = Builder.DAG.getRoot();
05614   }
05615 
05616   SDValue Ptr = Builder.getValue(PtrVal);
05617   SDValue LoadVal = Builder.DAG.getLoad(LoadVT, Builder.getCurSDLoc(), Root,
05618                                         Ptr, MachinePointerInfo(PtrVal),
05619                                         false /*volatile*/,
05620                                         false /*nontemporal*/,
05621                                         false /*isinvariant*/, 1 /* align=1 */);
05622 
05623   if (!ConstantMemory)
05624     Builder.PendingLoads.push_back(LoadVal.getValue(1));
05625   return LoadVal;
05626 }
05627 
05628 /// processIntegerCallValue - Record the value for an instruction that
05629 /// produces an integer result, converting the type where necessary.
05630 void SelectionDAGBuilder::processIntegerCallValue(const Instruction &I,
05631                                                   SDValue Value,
05632                                                   bool IsSigned) {
05633   EVT VT = DAG.getTargetLoweringInfo().getValueType(I.getType(), true);
05634   if (IsSigned)
05635     Value = DAG.getSExtOrTrunc(Value, getCurSDLoc(), VT);
05636   else
05637     Value = DAG.getZExtOrTrunc(Value, getCurSDLoc(), VT);
05638   setValue(&I, Value);
05639 }
05640 
05641 /// visitMemCmpCall - See if we can lower a call to memcmp in an optimized form.
05642 /// If so, return true and lower it, otherwise return false and it will be
05643 /// lowered like a normal call.
05644 bool SelectionDAGBuilder::visitMemCmpCall(const CallInst &I) {
05645   // Verify that the prototype makes sense.  int memcmp(void*,void*,size_t)
05646   if (I.getNumArgOperands() != 3)
05647     return false;
05648 
05649   const Value *LHS = I.getArgOperand(0), *RHS = I.getArgOperand(1);
05650   if (!LHS->getType()->isPointerTy() || !RHS->getType()->isPointerTy() ||
05651       !I.getArgOperand(2)->getType()->isIntegerTy() ||
05652       !I.getType()->isIntegerTy())
05653     return false;
05654 
05655   const Value *Size = I.getArgOperand(2);
05656   const ConstantInt *CSize = dyn_cast<ConstantInt>(Size);
05657   if (CSize && CSize->getZExtValue() == 0) {
05658     EVT CallVT = DAG.getTargetLoweringInfo().getValueType(I.getType(), true);
05659     setValue(&I, DAG.getConstant(0, CallVT));
05660     return true;
05661   }
05662 
05663   const TargetSelectionDAGInfo &TSI = DAG.getSelectionDAGInfo();
05664   std::pair<SDValue, SDValue> Res =
05665     TSI.EmitTargetCodeForMemcmp(DAG, getCurSDLoc(), DAG.getRoot(),
05666                                 getValue(LHS), getValue(RHS), getValue(Size),
05667                                 MachinePointerInfo(LHS),
05668                                 MachinePointerInfo(RHS));
05669   if (Res.first.getNode()) {
05670     processIntegerCallValue(I, Res.first, true);
05671     PendingLoads.push_back(Res.second);
05672     return true;
05673   }
05674 
05675   // memcmp(S1,S2,2) != 0 -> (*(short*)LHS != *(short*)RHS)  != 0
05676   // memcmp(S1,S2,4) != 0 -> (*(int*)LHS != *(int*)RHS)  != 0
05677   if (CSize && IsOnlyUsedInZeroEqualityComparison(&I)) {
05678     bool ActuallyDoIt = true;
05679     MVT LoadVT;
05680     Type *LoadTy;
05681     switch (CSize->getZExtValue()) {
05682     default:
05683       LoadVT = MVT::Other;
05684       LoadTy = nullptr;
05685       ActuallyDoIt = false;
05686       break;
05687     case 2:
05688       LoadVT = MVT::i16;
05689       LoadTy = Type::getInt16Ty(CSize->getContext());
05690       break;
05691     case 4:
05692       LoadVT = MVT::i32;
05693       LoadTy = Type::getInt32Ty(CSize->getContext());
05694       break;
05695     case 8:
05696       LoadVT = MVT::i64;
05697       LoadTy = Type::getInt64Ty(CSize->getContext());
05698       break;
05699         /*
05700     case 16:
05701       LoadVT = MVT::v4i32;
05702       LoadTy = Type::getInt32Ty(CSize->getContext());
05703       LoadTy = VectorType::get(LoadTy, 4);
05704       break;
05705          */
05706     }
05707 
05708     // This turns into unaligned loads.  We only do this if the target natively
05709     // supports the MVT we'll be loading or if it is small enough (<= 4) that
05710     // we'll only produce a small number of byte loads.
05711 
05712     // Require that we can find a legal MVT, and only do this if the target
05713     // supports unaligned loads of that type.  Expanding into byte loads would
05714     // bloat the code.
05715     const TargetLowering &TLI = DAG.getTargetLoweringInfo();
05716     if (ActuallyDoIt && CSize->getZExtValue() > 4) {
05717       unsigned DstAS = LHS->getType()->getPointerAddressSpace();
05718       unsigned SrcAS = RHS->getType()->getPointerAddressSpace();
05719       // TODO: Handle 5 byte compare as 4-byte + 1 byte.
05720       // TODO: Handle 8 byte compare on x86-32 as two 32-bit loads.
05721       // TODO: Check alignment of src and dest ptrs.
05722       if (!TLI.isTypeLegal(LoadVT) ||
05723           !TLI.allowsMisalignedMemoryAccesses(LoadVT, SrcAS) ||
05724           !TLI.allowsMisalignedMemoryAccesses(LoadVT, DstAS))
05725         ActuallyDoIt = false;
05726     }
05727 
05728     if (ActuallyDoIt) {
05729       SDValue LHSVal = getMemCmpLoad(LHS, LoadVT, LoadTy, *this);
05730       SDValue RHSVal = getMemCmpLoad(RHS, LoadVT, LoadTy, *this);
05731 
05732       SDValue Res = DAG.getSetCC(getCurSDLoc(), MVT::i1, LHSVal, RHSVal,
05733                                  ISD::SETNE);
05734       processIntegerCallValue(I, Res, false);
05735       return true;
05736     }
05737   }
05738 
05739 
05740   return false;
05741 }
05742 
05743 /// visitMemChrCall -- See if we can lower a memchr call into an optimized
05744 /// form.  If so, return true and lower it, otherwise return false and it
05745 /// will be lowered like a normal call.
05746 bool SelectionDAGBuilder::visitMemChrCall(const CallInst &I) {
05747   // Verify that the prototype makes sense.  void *memchr(void *, int, size_t)
05748   if (I.getNumArgOperands() != 3)
05749     return false;
05750 
05751   const Value *Src = I.getArgOperand(0);
05752   const Value *Char = I.getArgOperand(1);
05753   const Value *Length = I.getArgOperand(2);
05754   if (!Src->getType()->isPointerTy() ||
05755       !Char->getType()->isIntegerTy() ||
05756       !Length->getType()->isIntegerTy() ||
05757       !I.getType()->isPointerTy())
05758     return false;
05759 
05760   const TargetSelectionDAGInfo &TSI = DAG.getSelectionDAGInfo();
05761   std::pair<SDValue, SDValue> Res =
05762     TSI.EmitTargetCodeForMemchr(DAG, getCurSDLoc(), DAG.getRoot(),
05763                                 getValue(Src), getValue(Char), getValue(Length),
05764                                 MachinePointerInfo(Src));
05765   if (Res.first.getNode()) {
05766     setValue(&I, Res.first);
05767     PendingLoads.push_back(Res.second);
05768     return true;
05769   }
05770 
05771   return false;
05772 }
05773 
05774 /// visitStrCpyCall -- See if we can lower a strcpy or stpcpy call into an
05775 /// optimized form.  If so, return true and lower it, otherwise return false
05776 /// and it will be lowered like a normal call.
05777 bool SelectionDAGBuilder::visitStrCpyCall(const CallInst &I, bool isStpcpy) {
05778   // Verify that the prototype makes sense.  char *strcpy(char *, char *)
05779   if (I.getNumArgOperands() != 2)
05780     return false;
05781 
05782   const Value *Arg0 = I.getArgOperand(0), *Arg1 = I.getArgOperand(1);
05783   if (!Arg0->getType()->isPointerTy() ||
05784       !Arg1->getType()->isPointerTy() ||
05785       !I.getType()->isPointerTy())
05786     return false;
05787 
05788   const TargetSelectionDAGInfo &TSI = DAG.getSelectionDAGInfo();
05789   std::pair<SDValue, SDValue> Res =
05790     TSI.EmitTargetCodeForStrcpy(DAG, getCurSDLoc(), getRoot(),
05791                                 getValue(Arg0), getValue(Arg1),
05792                                 MachinePointerInfo(Arg0),
05793                                 MachinePointerInfo(Arg1), isStpcpy);
05794   if (Res.first.getNode()) {
05795     setValue(&I, Res.first);
05796     DAG.setRoot(Res.second);
05797     return true;
05798   }
05799 
05800   return false;
05801 }
05802 
05803 /// visitStrCmpCall - See if we can lower a call to strcmp in an optimized form.
05804 /// If so, return true and lower it, otherwise return false and it will be
05805 /// lowered like a normal call.
05806 bool SelectionDAGBuilder::visitStrCmpCall(const CallInst &I) {
05807   // Verify that the prototype makes sense.  int strcmp(void*,void*)
05808   if (I.getNumArgOperands() != 2)
05809     return false;
05810 
05811   const Value *Arg0 = I.getArgOperand(0), *Arg1 = I.getArgOperand(1);
05812   if (!Arg0->getType()->isPointerTy() ||
05813       !Arg1->getType()->isPointerTy() ||
05814       !I.getType()->isIntegerTy())
05815     return false;
05816 
05817   const TargetSelectionDAGInfo &TSI = DAG.getSelectionDAGInfo();
05818   std::pair<SDValue, SDValue> Res =
05819     TSI.EmitTargetCodeForStrcmp(DAG, getCurSDLoc(), DAG.getRoot(),
05820                                 getValue(Arg0), getValue(Arg1),
05821                                 MachinePointerInfo(Arg0),
05822                                 MachinePointerInfo(Arg1));
05823   if (Res.first.getNode()) {
05824     processIntegerCallValue(I, Res.first, true);
05825     PendingLoads.push_back(Res.second);
05826     return true;
05827   }
05828 
05829   return false;
05830 }
05831 
05832 /// visitStrLenCall -- See if we can lower a strlen call into an optimized
05833 /// form.  If so, return true and lower it, otherwise return false and it
05834 /// will be lowered like a normal call.
05835 bool SelectionDAGBuilder::visitStrLenCall(const CallInst &I) {
05836   // Verify that the prototype makes sense.  size_t strlen(char *)
05837   if (I.getNumArgOperands() != 1)
05838     return false;
05839 
05840   const Value *Arg0 = I.getArgOperand(0);
05841   if (!Arg0->getType()->isPointerTy() || !I.getType()->isIntegerTy())
05842     return false;
05843 
05844   const TargetSelectionDAGInfo &TSI = DAG.getSelectionDAGInfo();
05845   std::pair<SDValue, SDValue> Res =
05846     TSI.EmitTargetCodeForStrlen(DAG, getCurSDLoc(), DAG.getRoot(),
05847                                 getValue(Arg0), MachinePointerInfo(Arg0));
05848   if (Res.first.getNode()) {
05849     processIntegerCallValue(I, Res.first, false);
05850     PendingLoads.push_back(Res.second);
05851     return true;
05852   }
05853 
05854   return false;
05855 }
05856 
05857 /// visitStrNLenCall -- See if we can lower a strnlen call into an optimized
05858 /// form.  If so, return true and lower it, otherwise return false and it
05859 /// will be lowered like a normal call.
05860 bool SelectionDAGBuilder::visitStrNLenCall(const CallInst &I) {
05861   // Verify that the prototype makes sense.  size_t strnlen(char *, size_t)
05862   if (I.getNumArgOperands() != 2)
05863     return false;
05864 
05865   const Value *Arg0 = I.getArgOperand(0), *Arg1 = I.getArgOperand(1);
05866   if (!Arg0->getType()->isPointerTy() ||
05867       !Arg1->getType()->isIntegerTy() ||
05868       !I.getType()->isIntegerTy())
05869     return false;
05870 
05871   const TargetSelectionDAGInfo &TSI = DAG.getSelectionDAGInfo();
05872   std::pair<SDValue, SDValue> Res =
05873     TSI.EmitTargetCodeForStrnlen(DAG, getCurSDLoc(), DAG.getRoot(),
05874                                  getValue(Arg0), getValue(Arg1),
05875                                  MachinePointerInfo(Arg0));
05876   if (Res.first.getNode()) {
05877     processIntegerCallValue(I, Res.first, false);
05878     PendingLoads.push_back(Res.second);
05879     return true;
05880   }
05881 
05882   return false;
05883 }
05884 
05885 /// visitUnaryFloatCall - If a call instruction is a unary floating-point
05886 /// operation (as expected), translate it to an SDNode with the specified opcode
05887 /// and return true.
05888 bool SelectionDAGBuilder::visitUnaryFloatCall(const CallInst &I,
05889                                               unsigned Opcode) {
05890   // Sanity check that it really is a unary floating-point call.
05891   if (I.getNumArgOperands() != 1 ||
05892       !I.getArgOperand(0)->getType()->isFloatingPointTy() ||
05893       I.getType() != I.getArgOperand(0)->getType() ||
05894       !I.onlyReadsMemory())
05895     return false;
05896 
05897   SDValue Tmp = getValue(I.getArgOperand(0));
05898   setValue(&I, DAG.getNode(Opcode, getCurSDLoc(), Tmp.getValueType(), Tmp));
05899   return true;
05900 }
05901 
05902 /// visitBinaryFloatCall - If a call instruction is a binary floating-point
05903 /// operation (as expected), translate it to an SDNode with the specified opcode
05904 /// and return true.
05905 bool SelectionDAGBuilder::visitBinaryFloatCall(const CallInst &I,
05906                                                unsigned Opcode) {
05907   // Sanity check that it really is a binary floating-point call.
05908   if (I.getNumArgOperands() != 2 ||
05909       !I.getArgOperand(0)->getType()->isFloatingPointTy() ||
05910       I.getType() != I.getArgOperand(0)->getType() ||
05911       I.getType() != I.getArgOperand(1)->getType() ||
05912       !I.onlyReadsMemory())
05913     return false;
05914 
05915   SDValue Tmp0 = getValue(I.getArgOperand(0));
05916   SDValue Tmp1 = getValue(I.getArgOperand(1));
05917   EVT VT = Tmp0.getValueType();
05918   setValue(&I, DAG.getNode(Opcode, getCurSDLoc(), VT, Tmp0, Tmp1));
05919   return true;
05920 }
05921 
05922 void SelectionDAGBuilder::visitCall(const CallInst &I) {
05923   // Handle inline assembly differently.
05924   if (isa<InlineAsm>(I.getCalledValue())) {
05925     visitInlineAsm(&I);
05926     return;
05927   }
05928 
05929   MachineModuleInfo &MMI = DAG.getMachineFunction().getMMI();
05930   ComputeUsesVAFloatArgument(I, &MMI);
05931 
05932   const char *RenameFn = nullptr;
05933   if (Function *F = I.getCalledFunction()) {
05934     if (F->isDeclaration()) {
05935       if (const TargetIntrinsicInfo *II = TM.getIntrinsicInfo()) {
05936         if (unsigned IID = II->getIntrinsicID(F)) {
05937           RenameFn = visitIntrinsicCall(I, IID);
05938           if (!RenameFn)
05939             return;
05940         }
05941       }
05942       if (unsigned IID = F->getIntrinsicID()) {
05943         RenameFn = visitIntrinsicCall(I, IID);
05944         if (!RenameFn)
05945           return;
05946       }
05947     }
05948 
05949     // Check for well-known libc/libm calls.  If the function is internal, it
05950     // can't be a library call.
05951     LibFunc::Func Func;
05952     if (!F->hasLocalLinkage() && F->hasName() &&
05953         LibInfo->getLibFunc(F->getName(), Func) &&
05954         LibInfo->hasOptimizedCodeGen(Func)) {
05955       switch (Func) {
05956       default: break;
05957       case LibFunc::copysign:
05958       case LibFunc::copysignf:
05959       case LibFunc::copysignl:
05960         if (I.getNumArgOperands() == 2 &&   // Basic sanity checks.
05961             I.getArgOperand(0)->getType()->isFloatingPointTy() &&
05962             I.getType() == I.getArgOperand(0)->getType() &&
05963             I.getType() == I.getArgOperand(1)->getType() &&
05964             I.onlyReadsMemory()) {
05965           SDValue LHS = getValue(I.getArgOperand(0));
05966           SDValue RHS = getValue(I.getArgOperand(1));
05967           setValue(&I, DAG.getNode(ISD::FCOPYSIGN, getCurSDLoc(),
05968                                    LHS.getValueType(), LHS, RHS));
05969           return;
05970         }
05971         break;
05972       case LibFunc::fabs:
05973       case LibFunc::fabsf:
05974       case LibFunc::fabsl:
05975         if (visitUnaryFloatCall(I, ISD::FABS))
05976           return;
05977         break;
05978       case LibFunc::fmin:
05979       case LibFunc::fminf:
05980       case LibFunc::fminl:
05981         if (visitBinaryFloatCall(I, ISD::FMINNUM))
05982           return;
05983         break;
05984       case LibFunc::fmax:
05985       case LibFunc::fmaxf:
05986       case LibFunc::fmaxl:
05987         if (visitBinaryFloatCall(I, ISD::FMAXNUM))
05988           return;
05989         break;
05990       case LibFunc::sin:
05991       case LibFunc::sinf:
05992       case LibFunc::sinl:
05993         if (visitUnaryFloatCall(I, ISD::FSIN))
05994           return;
05995         break;
05996       case LibFunc::cos:
05997       case LibFunc::cosf:
05998       case LibFunc::cosl:
05999         if (visitUnaryFloatCall(I, ISD::FCOS))
06000           return;
06001         break;
06002       case LibFunc::sqrt:
06003       case LibFunc::sqrtf:
06004       case LibFunc::sqrtl:
06005       case LibFunc::sqrt_finite:
06006       case LibFunc::sqrtf_finite:
06007       case LibFunc::sqrtl_finite:
06008         if (visitUnaryFloatCall(I, ISD::FSQRT))
06009           return;
06010         break;
06011       case LibFunc::floor:
06012       case LibFunc::floorf:
06013       case LibFunc::floorl:
06014         if (visitUnaryFloatCall(I, ISD::FFLOOR))
06015           return;
06016         break;
06017       case LibFunc::nearbyint:
06018       case LibFunc::nearbyintf:
06019       case LibFunc::nearbyintl:
06020         if (visitUnaryFloatCall(I, ISD::FNEARBYINT))
06021           return;
06022         break;
06023       case LibFunc::ceil:
06024       case LibFunc::ceilf:
06025       case LibFunc::ceill:
06026         if (visitUnaryFloatCall(I, ISD::FCEIL))
06027           return;
06028         break;
06029       case LibFunc::rint:
06030       case LibFunc::rintf:
06031       case LibFunc::rintl:
06032         if (visitUnaryFloatCall(I, ISD::FRINT))
06033           return;
06034         break;
06035       case LibFunc::round:
06036       case LibFunc::roundf:
06037       case LibFunc::roundl:
06038         if (visitUnaryFloatCall(I, ISD::FROUND))
06039           return;
06040         break;
06041       case LibFunc::trunc:
06042       case LibFunc::truncf:
06043       case LibFunc::truncl:
06044         if (visitUnaryFloatCall(I, ISD::FTRUNC))
06045           return;
06046         break;
06047       case LibFunc::log2:
06048       case LibFunc::log2f:
06049       case LibFunc::log2l:
06050         if (visitUnaryFloatCall(I, ISD::FLOG2))
06051           return;
06052         break;
06053       case LibFunc::exp2:
06054       case LibFunc::exp2f:
06055       case LibFunc::exp2l:
06056         if (visitUnaryFloatCall(I, ISD::FEXP2))
06057           return;
06058         break;
06059       case LibFunc::memcmp:
06060         if (visitMemCmpCall(I))
06061           return;
06062         break;
06063       case LibFunc::memchr:
06064         if (visitMemChrCall(I))
06065           return;
06066         break;
06067       case LibFunc::strcpy:
06068         if (visitStrCpyCall(I, false))
06069           return;
06070         break;
06071       case LibFunc::stpcpy:
06072         if (visitStrCpyCall(I, true))
06073           return;
06074         break;
06075       case LibFunc::strcmp:
06076         if (visitStrCmpCall(I))
06077           return;
06078         break;
06079       case LibFunc::strlen:
06080         if (visitStrLenCall(I))
06081           return;
06082         break;
06083       case LibFunc::strnlen:
06084         if (visitStrNLenCall(I))
06085           return;
06086         break;
06087       }
06088     }
06089   }
06090 
06091   SDValue Callee;
06092   if (!RenameFn)
06093     Callee = getValue(I.getCalledValue());
06094   else
06095     Callee = DAG.getExternalSymbol(RenameFn,
06096                                    DAG.getTargetLoweringInfo().getPointerTy());
06097 
06098   // Check if we can potentially perform a tail call. More detailed checking is
06099   // be done within LowerCallTo, after more information about the call is known.
06100   LowerCallTo(&I, Callee, I.isTailCall());
06101 }
06102 
06103 namespace {
06104 
06105 /// AsmOperandInfo - This contains information for each constraint that we are
06106 /// lowering.
06107 class SDISelAsmOperandInfo : public TargetLowering::AsmOperandInfo {
06108 public:
06109   /// CallOperand - If this is the result output operand or a clobber
06110   /// this is null, otherwise it is the incoming operand to the CallInst.
06111   /// This gets modified as the asm is processed.
06112   SDValue CallOperand;
06113 
06114   /// AssignedRegs - If this is a register or register class operand, this
06115   /// contains the set of register corresponding to the operand.
06116   RegsForValue AssignedRegs;
06117 
06118   explicit SDISelAsmOperandInfo(const TargetLowering::AsmOperandInfo &info)
06119     : TargetLowering::AsmOperandInfo(info), CallOperand(nullptr,0) {
06120   }
06121 
06122   /// getCallOperandValEVT - Return the EVT of the Value* that this operand
06123   /// corresponds to.  If there is no Value* for this operand, it returns
06124   /// MVT::Other.
06125   EVT getCallOperandValEVT(LLVMContext &Context,
06126                            const TargetLowering &TLI,
06127                            const DataLayout *DL) const {
06128     if (!CallOperandVal) return MVT::Other;
06129 
06130     if (isa<BasicBlock>(CallOperandVal))
06131       return TLI.getPointerTy();
06132 
06133     llvm::Type *OpTy = CallOperandVal->getType();
06134 
06135     // FIXME: code duplicated from TargetLowering::ParseConstraints().
06136     // If this is an indirect operand, the operand is a pointer to the
06137     // accessed type.
06138     if (isIndirect) {
06139       llvm::PointerType *PtrTy = dyn_cast<PointerType>(OpTy);
06140       if (!PtrTy)
06141         report_fatal_error("Indirect operand for inline asm not a pointer!");
06142       OpTy = PtrTy->getElementType();
06143     }
06144 
06145     // Look for vector wrapped in a struct. e.g. { <16 x i8> }.
06146     if (StructType *STy = dyn_cast<StructType>(OpTy))
06147       if (STy->getNumElements() == 1)
06148         OpTy = STy->getElementType(0);
06149 
06150     // If OpTy is not a single value, it may be a struct/union that we
06151     // can tile with integers.
06152     if (!OpTy->isSingleValueType() && OpTy->isSized()) {
06153       unsigned BitSize = DL->