LLVM  mainline
SelectionDAGBuilder.cpp
Go to the documentation of this file.
00001 //===-- SelectionDAGBuilder.cpp - Selection-DAG building ------------------===//
00002 //
00003 //                     The LLVM Compiler Infrastructure
00004 //
00005 // This file is distributed under the University of Illinois Open Source
00006 // License. See LICENSE.TXT for details.
00007 //
00008 //===----------------------------------------------------------------------===//
00009 //
00010 // This implements routines for translating from LLVM IR into SelectionDAG IR.
00011 //
00012 //===----------------------------------------------------------------------===//
00013 
00014 #include "SelectionDAGBuilder.h"
00015 #include "SDNodeDbgValue.h"
00016 #include "llvm/ADT/BitVector.h"
00017 #include "llvm/ADT/Optional.h"
00018 #include "llvm/ADT/SmallSet.h"
00019 #include "llvm/ADT/Statistic.h"
00020 #include "llvm/Analysis/AliasAnalysis.h"
00021 #include "llvm/Analysis/BranchProbabilityInfo.h"
00022 #include "llvm/Analysis/ConstantFolding.h"
00023 #include "llvm/Analysis/TargetLibraryInfo.h"
00024 #include "llvm/Analysis/ValueTracking.h"
00025 #include "llvm/Analysis/VectorUtils.h"
00026 #include "llvm/CodeGen/FastISel.h"
00027 #include "llvm/CodeGen/FunctionLoweringInfo.h"
00028 #include "llvm/CodeGen/GCMetadata.h"
00029 #include "llvm/CodeGen/GCStrategy.h"
00030 #include "llvm/CodeGen/MachineFrameInfo.h"
00031 #include "llvm/CodeGen/MachineFunction.h"
00032 #include "llvm/CodeGen/MachineInstrBuilder.h"
00033 #include "llvm/CodeGen/MachineJumpTableInfo.h"
00034 #include "llvm/CodeGen/MachineModuleInfo.h"
00035 #include "llvm/CodeGen/MachineRegisterInfo.h"
00036 #include "llvm/CodeGen/SelectionDAG.h"
00037 #include "llvm/CodeGen/StackMaps.h"
00038 #include "llvm/CodeGen/WinEHFuncInfo.h"
00039 #include "llvm/IR/CallingConv.h"
00040 #include "llvm/IR/Constants.h"
00041 #include "llvm/IR/DataLayout.h"
00042 #include "llvm/IR/DebugInfo.h"
00043 #include "llvm/IR/DerivedTypes.h"
00044 #include "llvm/IR/Function.h"
00045 #include "llvm/IR/GetElementPtrTypeIterator.h"
00046 #include "llvm/IR/GlobalVariable.h"
00047 #include "llvm/IR/InlineAsm.h"
00048 #include "llvm/IR/Instructions.h"
00049 #include "llvm/IR/IntrinsicInst.h"
00050 #include "llvm/IR/Intrinsics.h"
00051 #include "llvm/IR/LLVMContext.h"
00052 #include "llvm/IR/Module.h"
00053 #include "llvm/IR/Statepoint.h"
00054 #include "llvm/MC/MCSymbol.h"
00055 #include "llvm/Support/CommandLine.h"
00056 #include "llvm/Support/Debug.h"
00057 #include "llvm/Support/ErrorHandling.h"
00058 #include "llvm/Support/MathExtras.h"
00059 #include "llvm/Support/raw_ostream.h"
00060 #include "llvm/Target/TargetFrameLowering.h"
00061 #include "llvm/Target/TargetInstrInfo.h"
00062 #include "llvm/Target/TargetIntrinsicInfo.h"
00063 #include "llvm/Target/TargetLowering.h"
00064 #include "llvm/Target/TargetOptions.h"
00065 #include "llvm/Target/TargetSelectionDAGInfo.h"
00066 #include "llvm/Target/TargetSubtargetInfo.h"
00067 #include <algorithm>
00068 #include <utility>
00069 using namespace llvm;
00070 
00071 #define DEBUG_TYPE "isel"
00072 
00073 /// LimitFloatPrecision - Generate low-precision inline sequences for
00074 /// some float libcalls (6, 8 or 12 bits).
00075 static unsigned LimitFloatPrecision;
00076 
00077 static cl::opt<unsigned, true>
00078 LimitFPPrecision("limit-float-precision",
00079                  cl::desc("Generate low-precision inline sequences "
00080                           "for some float libcalls"),
00081                  cl::location(LimitFloatPrecision),
00082                  cl::init(0));
00083 
00084 static cl::opt<bool>
00085 EnableFMFInDAG("enable-fmf-dag", cl::init(true), cl::Hidden,
00086                 cl::desc("Enable fast-math-flags for DAG nodes"));
00087 
00088 // Limit the width of DAG chains. This is important in general to prevent
00089 // DAG-based analysis from blowing up. For example, alias analysis and
00090 // load clustering may not complete in reasonable time. It is difficult to
00091 // recognize and avoid this situation within each individual analysis, and
00092 // future analyses are likely to have the same behavior. Limiting DAG width is
00093 // the safe approach and will be especially important with global DAGs.
00094 //
00095 // MaxParallelChains default is arbitrarily high to avoid affecting
00096 // optimization, but could be lowered to improve compile time. Any ld-ld-st-st
00097 // sequence over this should have been converted to llvm.memcpy by the
00098 // frontend. It easy to induce this behavior with .ll code such as:
00099 // %buffer = alloca [4096 x i8]
00100 // %data = load [4096 x i8]* %argPtr
00101 // store [4096 x i8] %data, [4096 x i8]* %buffer
00102 static const unsigned MaxParallelChains = 64;
00103 
00104 static SDValue getCopyFromPartsVector(SelectionDAG &DAG, SDLoc DL,
00105                                       const SDValue *Parts, unsigned NumParts,
00106                                       MVT PartVT, EVT ValueVT, const Value *V);
00107 
00108 /// getCopyFromParts - Create a value that contains the specified legal parts
00109 /// combined into the value they represent.  If the parts combine to a type
00110 /// larger then ValueVT then AssertOp can be used to specify whether the extra
00111 /// bits are known to be zero (ISD::AssertZext) or sign extended from ValueVT
00112 /// (ISD::AssertSext).
00113 static SDValue getCopyFromParts(SelectionDAG &DAG, SDLoc DL,
00114                                 const SDValue *Parts,
00115                                 unsigned NumParts, MVT PartVT, EVT ValueVT,
00116                                 const Value *V,
00117                                 ISD::NodeType AssertOp = ISD::DELETED_NODE) {
00118   if (ValueVT.isVector())
00119     return getCopyFromPartsVector(DAG, DL, Parts, NumParts,
00120                                   PartVT, ValueVT, V);
00121 
00122   assert(NumParts > 0 && "No parts to assemble!");
00123   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
00124   SDValue Val = Parts[0];
00125 
00126   if (NumParts > 1) {
00127     // Assemble the value from multiple parts.
00128     if (ValueVT.isInteger()) {
00129       unsigned PartBits = PartVT.getSizeInBits();
00130       unsigned ValueBits = ValueVT.getSizeInBits();
00131 
00132       // Assemble the power of 2 part.
00133       unsigned RoundParts = NumParts & (NumParts - 1) ?
00134         1 << Log2_32(NumParts) : NumParts;
00135       unsigned RoundBits = PartBits * RoundParts;
00136       EVT RoundVT = RoundBits == ValueBits ?
00137         ValueVT : EVT::getIntegerVT(*DAG.getContext(), RoundBits);
00138       SDValue Lo, Hi;
00139 
00140       EVT HalfVT = EVT::getIntegerVT(*DAG.getContext(), RoundBits/2);
00141 
00142       if (RoundParts > 2) {
00143         Lo = getCopyFromParts(DAG, DL, Parts, RoundParts / 2,
00144                               PartVT, HalfVT, V);
00145         Hi = getCopyFromParts(DAG, DL, Parts + RoundParts / 2,
00146                               RoundParts / 2, PartVT, HalfVT, V);
00147       } else {
00148         Lo = DAG.getNode(ISD::BITCAST, DL, HalfVT, Parts[0]);
00149         Hi = DAG.getNode(ISD::BITCAST, DL, HalfVT, Parts[1]);
00150       }
00151 
00152       if (DAG.getDataLayout().isBigEndian())
00153         std::swap(Lo, Hi);
00154 
00155       Val = DAG.getNode(ISD::BUILD_PAIR, DL, RoundVT, Lo, Hi);
00156 
00157       if (RoundParts < NumParts) {
00158         // Assemble the trailing non-power-of-2 part.
00159         unsigned OddParts = NumParts - RoundParts;
00160         EVT OddVT = EVT::getIntegerVT(*DAG.getContext(), OddParts * PartBits);
00161         Hi = getCopyFromParts(DAG, DL,
00162                               Parts + RoundParts, OddParts, PartVT, OddVT, V);
00163 
00164         // Combine the round and odd parts.
00165         Lo = Val;
00166         if (DAG.getDataLayout().isBigEndian())
00167           std::swap(Lo, Hi);
00168         EVT TotalVT = EVT::getIntegerVT(*DAG.getContext(), NumParts * PartBits);
00169         Hi = DAG.getNode(ISD::ANY_EXTEND, DL, TotalVT, Hi);
00170         Hi =
00171             DAG.getNode(ISD::SHL, DL, TotalVT, Hi,
00172                         DAG.getConstant(Lo.getValueType().getSizeInBits(), DL,
00173                                         TLI.getPointerTy(DAG.getDataLayout())));
00174         Lo = DAG.getNode(ISD::ZERO_EXTEND, DL, TotalVT, Lo);
00175         Val = DAG.getNode(ISD::OR, DL, TotalVT, Lo, Hi);
00176       }
00177     } else if (PartVT.isFloatingPoint()) {
00178       // FP split into multiple FP parts (for ppcf128)
00179       assert(ValueVT == EVT(MVT::ppcf128) && PartVT == MVT::f64 &&
00180              "Unexpected split");
00181       SDValue Lo, Hi;
00182       Lo = DAG.getNode(ISD::BITCAST, DL, EVT(MVT::f64), Parts[0]);
00183       Hi = DAG.getNode(ISD::BITCAST, DL, EVT(MVT::f64), Parts[1]);
00184       if (TLI.hasBigEndianPartOrdering(ValueVT, DAG.getDataLayout()))
00185         std::swap(Lo, Hi);
00186       Val = DAG.getNode(ISD::BUILD_PAIR, DL, ValueVT, Lo, Hi);
00187     } else {
00188       // FP split into integer parts (soft fp)
00189       assert(ValueVT.isFloatingPoint() && PartVT.isInteger() &&
00190              !PartVT.isVector() && "Unexpected split");
00191       EVT IntVT = EVT::getIntegerVT(*DAG.getContext(), ValueVT.getSizeInBits());
00192       Val = getCopyFromParts(DAG, DL, Parts, NumParts, PartVT, IntVT, V);
00193     }
00194   }
00195 
00196   // There is now one part, held in Val.  Correct it to match ValueVT.
00197   EVT PartEVT = Val.getValueType();
00198 
00199   if (PartEVT == ValueVT)
00200     return Val;
00201 
00202   if (PartEVT.isInteger() && ValueVT.isFloatingPoint() &&
00203       ValueVT.bitsLT(PartEVT)) {
00204     // For an FP value in an integer part, we need to truncate to the right
00205     // width first.
00206     PartEVT = EVT::getIntegerVT(*DAG.getContext(),  ValueVT.getSizeInBits());
00207     Val = DAG.getNode(ISD::TRUNCATE, DL, PartEVT, Val);
00208   }
00209 
00210   if (PartEVT.isInteger() && ValueVT.isInteger()) {
00211     if (ValueVT.bitsLT(PartEVT)) {
00212       // For a truncate, see if we have any information to
00213       // indicate whether the truncated bits will always be
00214       // zero or sign-extension.
00215       if (AssertOp != ISD::DELETED_NODE)
00216         Val = DAG.getNode(AssertOp, DL, PartEVT, Val,
00217                           DAG.getValueType(ValueVT));
00218       return DAG.getNode(ISD::TRUNCATE, DL, ValueVT, Val);
00219     }
00220     return DAG.getNode(ISD::ANY_EXTEND, DL, ValueVT, Val);
00221   }
00222 
00223   if (PartEVT.isFloatingPoint() && ValueVT.isFloatingPoint()) {
00224     // FP_ROUND's are always exact here.
00225     if (ValueVT.bitsLT(Val.getValueType()))
00226       return DAG.getNode(
00227           ISD::FP_ROUND, DL, ValueVT, Val,
00228           DAG.getTargetConstant(1, DL, TLI.getPointerTy(DAG.getDataLayout())));
00229 
00230     return DAG.getNode(ISD::FP_EXTEND, DL, ValueVT, Val);
00231   }
00232 
00233   if (PartEVT.getSizeInBits() == ValueVT.getSizeInBits())
00234     return DAG.getNode(ISD::BITCAST, DL, ValueVT, Val);
00235 
00236   llvm_unreachable("Unknown mismatch!");
00237 }
00238 
00239 static void diagnosePossiblyInvalidConstraint(LLVMContext &Ctx, const Value *V,
00240                                               const Twine &ErrMsg) {
00241   const Instruction *I = dyn_cast_or_null<Instruction>(V);
00242   if (!V)
00243     return Ctx.emitError(ErrMsg);
00244 
00245   const char *AsmError = ", possible invalid constraint for vector type";
00246   if (const CallInst *CI = dyn_cast<CallInst>(I))
00247     if (isa<InlineAsm>(CI->getCalledValue()))
00248       return Ctx.emitError(I, ErrMsg + AsmError);
00249 
00250   return Ctx.emitError(I, ErrMsg);
00251 }
00252 
00253 /// getCopyFromPartsVector - Create a value that contains the specified legal
00254 /// parts combined into the value they represent.  If the parts combine to a
00255 /// type larger then ValueVT then AssertOp can be used to specify whether the
00256 /// extra bits are known to be zero (ISD::AssertZext) or sign extended from
00257 /// ValueVT (ISD::AssertSext).
00258 static SDValue getCopyFromPartsVector(SelectionDAG &DAG, SDLoc DL,
00259                                       const SDValue *Parts, unsigned NumParts,
00260                                       MVT PartVT, EVT ValueVT, const Value *V) {
00261   assert(ValueVT.isVector() && "Not a vector value");
00262   assert(NumParts > 0 && "No parts to assemble!");
00263   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
00264   SDValue Val = Parts[0];
00265 
00266   // Handle a multi-element vector.
00267   if (NumParts > 1) {
00268     EVT IntermediateVT;
00269     MVT RegisterVT;
00270     unsigned NumIntermediates;
00271     unsigned NumRegs =
00272     TLI.getVectorTypeBreakdown(*DAG.getContext(), ValueVT, IntermediateVT,
00273                                NumIntermediates, RegisterVT);
00274     assert(NumRegs == NumParts && "Part count doesn't match vector breakdown!");
00275     NumParts = NumRegs; // Silence a compiler warning.
00276     assert(RegisterVT == PartVT && "Part type doesn't match vector breakdown!");
00277     assert(RegisterVT.getSizeInBits() ==
00278            Parts[0].getSimpleValueType().getSizeInBits() &&
00279            "Part type sizes don't match!");
00280 
00281     // Assemble the parts into intermediate operands.
00282     SmallVector<SDValue, 8> Ops(NumIntermediates);
00283     if (NumIntermediates == NumParts) {
00284       // If the register was not expanded, truncate or copy the value,
00285       // as appropriate.
00286       for (unsigned i = 0; i != NumParts; ++i)
00287         Ops[i] = getCopyFromParts(DAG, DL, &Parts[i], 1,
00288                                   PartVT, IntermediateVT, V);
00289     } else if (NumParts > 0) {
00290       // If the intermediate type was expanded, build the intermediate
00291       // operands from the parts.
00292       assert(NumParts % NumIntermediates == 0 &&
00293              "Must expand into a divisible number of parts!");
00294       unsigned Factor = NumParts / NumIntermediates;
00295       for (unsigned i = 0; i != NumIntermediates; ++i)
00296         Ops[i] = getCopyFromParts(DAG, DL, &Parts[i * Factor], Factor,
00297                                   PartVT, IntermediateVT, V);
00298     }
00299 
00300     // Build a vector with BUILD_VECTOR or CONCAT_VECTORS from the
00301     // intermediate operands.
00302     Val = DAG.getNode(IntermediateVT.isVector() ? ISD::CONCAT_VECTORS
00303                                                 : ISD::BUILD_VECTOR,
00304                       DL, ValueVT, Ops);
00305   }
00306 
00307   // There is now one part, held in Val.  Correct it to match ValueVT.
00308   EVT PartEVT = Val.getValueType();
00309 
00310   if (PartEVT == ValueVT)
00311     return Val;
00312 
00313   if (PartEVT.isVector()) {
00314     // If the element type of the source/dest vectors are the same, but the
00315     // parts vector has more elements than the value vector, then we have a
00316     // vector widening case (e.g. <2 x float> -> <4 x float>).  Extract the
00317     // elements we want.
00318     if (PartEVT.getVectorElementType() == ValueVT.getVectorElementType()) {
00319       assert(PartEVT.getVectorNumElements() > ValueVT.getVectorNumElements() &&
00320              "Cannot narrow, it would be a lossy transformation");
00321       return DAG.getNode(
00322           ISD::EXTRACT_SUBVECTOR, DL, ValueVT, Val,
00323           DAG.getConstant(0, DL, TLI.getVectorIdxTy(DAG.getDataLayout())));
00324     }
00325 
00326     // Vector/Vector bitcast.
00327     if (ValueVT.getSizeInBits() == PartEVT.getSizeInBits())
00328       return DAG.getNode(ISD::BITCAST, DL, ValueVT, Val);
00329 
00330     assert(PartEVT.getVectorNumElements() == ValueVT.getVectorNumElements() &&
00331       "Cannot handle this kind of promotion");
00332     // Promoted vector extract
00333     return DAG.getAnyExtOrTrunc(Val, DL, ValueVT);
00334 
00335   }
00336 
00337   // Trivial bitcast if the types are the same size and the destination
00338   // vector type is legal.
00339   if (PartEVT.getSizeInBits() == ValueVT.getSizeInBits() &&
00340       TLI.isTypeLegal(ValueVT))
00341     return DAG.getNode(ISD::BITCAST, DL, ValueVT, Val);
00342 
00343   // Handle cases such as i8 -> <1 x i1>
00344   if (ValueVT.getVectorNumElements() != 1) {
00345     diagnosePossiblyInvalidConstraint(*DAG.getContext(), V,
00346                                       "non-trivial scalar-to-vector conversion");
00347     return DAG.getUNDEF(ValueVT);
00348   }
00349 
00350   if (ValueVT.getVectorNumElements() == 1 &&
00351       ValueVT.getVectorElementType() != PartEVT)
00352     Val = DAG.getAnyExtOrTrunc(Val, DL, ValueVT.getScalarType());
00353 
00354   return DAG.getNode(ISD::BUILD_VECTOR, DL, ValueVT, Val);
00355 }
00356 
00357 static void getCopyToPartsVector(SelectionDAG &DAG, SDLoc dl,
00358                                  SDValue Val, SDValue *Parts, unsigned NumParts,
00359                                  MVT PartVT, const Value *V);
00360 
00361 /// getCopyToParts - Create a series of nodes that contain the specified value
00362 /// split into legal parts.  If the parts contain more bits than Val, then, for
00363 /// integers, ExtendKind can be used to specify how to generate the extra bits.
00364 static void getCopyToParts(SelectionDAG &DAG, SDLoc DL,
00365                            SDValue Val, SDValue *Parts, unsigned NumParts,
00366                            MVT PartVT, const Value *V,
00367                            ISD::NodeType ExtendKind = ISD::ANY_EXTEND) {
00368   EVT ValueVT = Val.getValueType();
00369 
00370   // Handle the vector case separately.
00371   if (ValueVT.isVector())
00372     return getCopyToPartsVector(DAG, DL, Val, Parts, NumParts, PartVT, V);
00373 
00374   unsigned PartBits = PartVT.getSizeInBits();
00375   unsigned OrigNumParts = NumParts;
00376   assert(DAG.getTargetLoweringInfo().isTypeLegal(PartVT) &&
00377          "Copying to an illegal type!");
00378 
00379   if (NumParts == 0)
00380     return;
00381 
00382   assert(!ValueVT.isVector() && "Vector case handled elsewhere");
00383   EVT PartEVT = PartVT;
00384   if (PartEVT == ValueVT) {
00385     assert(NumParts == 1 && "No-op copy with multiple parts!");
00386     Parts[0] = Val;
00387     return;
00388   }
00389 
00390   if (NumParts * PartBits > ValueVT.getSizeInBits()) {
00391     // If the parts cover more bits than the value has, promote the value.
00392     if (PartVT.isFloatingPoint() && ValueVT.isFloatingPoint()) {
00393       assert(NumParts == 1 && "Do not know what to promote to!");
00394       Val = DAG.getNode(ISD::FP_EXTEND, DL, PartVT, Val);
00395     } else {
00396       if (ValueVT.isFloatingPoint()) {
00397         // FP values need to be bitcast, then extended if they are being put
00398         // into a larger container.
00399         ValueVT = EVT::getIntegerVT(*DAG.getContext(),  ValueVT.getSizeInBits());
00400         Val = DAG.getNode(ISD::BITCAST, DL, ValueVT, Val);
00401       }
00402       assert((PartVT.isInteger() || PartVT == MVT::x86mmx) &&
00403              ValueVT.isInteger() &&
00404              "Unknown mismatch!");
00405       ValueVT = EVT::getIntegerVT(*DAG.getContext(), NumParts * PartBits);
00406       Val = DAG.getNode(ExtendKind, DL, ValueVT, Val);
00407       if (PartVT == MVT::x86mmx)
00408         Val = DAG.getNode(ISD::BITCAST, DL, PartVT, Val);
00409     }
00410   } else if (PartBits == ValueVT.getSizeInBits()) {
00411     // Different types of the same size.
00412     assert(NumParts == 1 && PartEVT != ValueVT);
00413     Val = DAG.getNode(ISD::BITCAST, DL, PartVT, Val);
00414   } else if (NumParts * PartBits < ValueVT.getSizeInBits()) {
00415     // If the parts cover less bits than value has, truncate the value.
00416     assert((PartVT.isInteger() || PartVT == MVT::x86mmx) &&
00417            ValueVT.isInteger() &&
00418            "Unknown mismatch!");
00419     ValueVT = EVT::getIntegerVT(*DAG.getContext(), NumParts * PartBits);
00420     Val = DAG.getNode(ISD::TRUNCATE, DL, ValueVT, Val);
00421     if (PartVT == MVT::x86mmx)
00422       Val = DAG.getNode(ISD::BITCAST, DL, PartVT, Val);
00423   }
00424 
00425   // The value may have changed - recompute ValueVT.
00426   ValueVT = Val.getValueType();
00427   assert(NumParts * PartBits == ValueVT.getSizeInBits() &&
00428          "Failed to tile the value with PartVT!");
00429 
00430   if (NumParts == 1) {
00431     if (PartEVT != ValueVT)
00432       diagnosePossiblyInvalidConstraint(*DAG.getContext(), V,
00433                                         "scalar-to-vector conversion failed");
00434 
00435     Parts[0] = Val;
00436     return;
00437   }
00438 
00439   // Expand the value into multiple parts.
00440   if (NumParts & (NumParts - 1)) {
00441     // The number of parts is not a power of 2.  Split off and copy the tail.
00442     assert(PartVT.isInteger() && ValueVT.isInteger() &&
00443            "Do not know what to expand to!");
00444     unsigned RoundParts = 1 << Log2_32(NumParts);
00445     unsigned RoundBits = RoundParts * PartBits;
00446     unsigned OddParts = NumParts - RoundParts;
00447     SDValue OddVal = DAG.getNode(ISD::SRL, DL, ValueVT, Val,
00448                                  DAG.getIntPtrConstant(RoundBits, DL));
00449     getCopyToParts(DAG, DL, OddVal, Parts + RoundParts, OddParts, PartVT, V);
00450 
00451     if (DAG.getDataLayout().isBigEndian())
00452       // The odd parts were reversed by getCopyToParts - unreverse them.
00453       std::reverse(Parts + RoundParts, Parts + NumParts);
00454 
00455     NumParts = RoundParts;
00456     ValueVT = EVT::getIntegerVT(*DAG.getContext(), NumParts * PartBits);
00457     Val = DAG.getNode(ISD::TRUNCATE, DL, ValueVT, Val);
00458   }
00459 
00460   // The number of parts is a power of 2.  Repeatedly bisect the value using
00461   // EXTRACT_ELEMENT.
00462   Parts[0] = DAG.getNode(ISD::BITCAST, DL,
00463                          EVT::getIntegerVT(*DAG.getContext(),
00464                                            ValueVT.getSizeInBits()),
00465                          Val);
00466 
00467   for (unsigned StepSize = NumParts; StepSize > 1; StepSize /= 2) {
00468     for (unsigned i = 0; i < NumParts; i += StepSize) {
00469       unsigned ThisBits = StepSize * PartBits / 2;
00470       EVT ThisVT = EVT::getIntegerVT(*DAG.getContext(), ThisBits);
00471       SDValue &Part0 = Parts[i];
00472       SDValue &Part1 = Parts[i+StepSize/2];
00473 
00474       Part1 = DAG.getNode(ISD::EXTRACT_ELEMENT, DL,
00475                           ThisVT, Part0, DAG.getIntPtrConstant(1, DL));
00476       Part0 = DAG.getNode(ISD::EXTRACT_ELEMENT, DL,
00477                           ThisVT, Part0, DAG.getIntPtrConstant(0, DL));
00478 
00479       if (ThisBits == PartBits && ThisVT != PartVT) {
00480         Part0 = DAG.getNode(ISD::BITCAST, DL, PartVT, Part0);
00481         Part1 = DAG.getNode(ISD::BITCAST, DL, PartVT, Part1);
00482       }
00483     }
00484   }
00485 
00486   if (DAG.getDataLayout().isBigEndian())
00487     std::reverse(Parts, Parts + OrigNumParts);
00488 }
00489 
00490 
00491 /// getCopyToPartsVector - Create a series of nodes that contain the specified
00492 /// value split into legal parts.
00493 static void getCopyToPartsVector(SelectionDAG &DAG, SDLoc DL,
00494                                  SDValue Val, SDValue *Parts, unsigned NumParts,
00495                                  MVT PartVT, const Value *V) {
00496   EVT ValueVT = Val.getValueType();
00497   assert(ValueVT.isVector() && "Not a vector");
00498   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
00499 
00500   if (NumParts == 1) {
00501     EVT PartEVT = PartVT;
00502     if (PartEVT == ValueVT) {
00503       // Nothing to do.
00504     } else if (PartVT.getSizeInBits() == ValueVT.getSizeInBits()) {
00505       // Bitconvert vector->vector case.
00506       Val = DAG.getNode(ISD::BITCAST, DL, PartVT, Val);
00507     } else if (PartVT.isVector() &&
00508                PartEVT.getVectorElementType() == ValueVT.getVectorElementType() &&
00509                PartEVT.getVectorNumElements() > ValueVT.getVectorNumElements()) {
00510       EVT ElementVT = PartVT.getVectorElementType();
00511       // Vector widening case, e.g. <2 x float> -> <4 x float>.  Shuffle in
00512       // undef elements.
00513       SmallVector<SDValue, 16> Ops;
00514       for (unsigned i = 0, e = ValueVT.getVectorNumElements(); i != e; ++i)
00515         Ops.push_back(DAG.getNode(
00516             ISD::EXTRACT_VECTOR_ELT, DL, ElementVT, Val,
00517             DAG.getConstant(i, DL, TLI.getVectorIdxTy(DAG.getDataLayout()))));
00518 
00519       for (unsigned i = ValueVT.getVectorNumElements(),
00520            e = PartVT.getVectorNumElements(); i != e; ++i)
00521         Ops.push_back(DAG.getUNDEF(ElementVT));
00522 
00523       Val = DAG.getNode(ISD::BUILD_VECTOR, DL, PartVT, Ops);
00524 
00525       // FIXME: Use CONCAT for 2x -> 4x.
00526 
00527       //SDValue UndefElts = DAG.getUNDEF(VectorTy);
00528       //Val = DAG.getNode(ISD::CONCAT_VECTORS, DL, PartVT, Val, UndefElts);
00529     } else if (PartVT.isVector() &&
00530                PartEVT.getVectorElementType().bitsGE(
00531                  ValueVT.getVectorElementType()) &&
00532                PartEVT.getVectorNumElements() == ValueVT.getVectorNumElements()) {
00533 
00534       // Promoted vector extract
00535       Val = DAG.getAnyExtOrTrunc(Val, DL, PartVT);
00536     } else{
00537       // Vector -> scalar conversion.
00538       assert(ValueVT.getVectorNumElements() == 1 &&
00539              "Only trivial vector-to-scalar conversions should get here!");
00540       Val = DAG.getNode(
00541           ISD::EXTRACT_VECTOR_ELT, DL, PartVT, Val,
00542           DAG.getConstant(0, DL, TLI.getVectorIdxTy(DAG.getDataLayout())));
00543 
00544       Val = DAG.getAnyExtOrTrunc(Val, DL, PartVT);
00545     }
00546 
00547     Parts[0] = Val;
00548     return;
00549   }
00550 
00551   // Handle a multi-element vector.
00552   EVT IntermediateVT;
00553   MVT RegisterVT;
00554   unsigned NumIntermediates;
00555   unsigned NumRegs = TLI.getVectorTypeBreakdown(*DAG.getContext(), ValueVT,
00556                                                 IntermediateVT,
00557                                                 NumIntermediates, RegisterVT);
00558   unsigned NumElements = ValueVT.getVectorNumElements();
00559 
00560   assert(NumRegs == NumParts && "Part count doesn't match vector breakdown!");
00561   NumParts = NumRegs; // Silence a compiler warning.
00562   assert(RegisterVT == PartVT && "Part type doesn't match vector breakdown!");
00563 
00564   // Split the vector into intermediate operands.
00565   SmallVector<SDValue, 8> Ops(NumIntermediates);
00566   for (unsigned i = 0; i != NumIntermediates; ++i) {
00567     if (IntermediateVT.isVector())
00568       Ops[i] =
00569           DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, IntermediateVT, Val,
00570                       DAG.getConstant(i * (NumElements / NumIntermediates), DL,
00571                                       TLI.getVectorIdxTy(DAG.getDataLayout())));
00572     else
00573       Ops[i] = DAG.getNode(
00574           ISD::EXTRACT_VECTOR_ELT, DL, IntermediateVT, Val,
00575           DAG.getConstant(i, DL, TLI.getVectorIdxTy(DAG.getDataLayout())));
00576   }
00577 
00578   // Split the intermediate operands into legal parts.
00579   if (NumParts == NumIntermediates) {
00580     // If the register was not expanded, promote or copy the value,
00581     // as appropriate.
00582     for (unsigned i = 0; i != NumParts; ++i)
00583       getCopyToParts(DAG, DL, Ops[i], &Parts[i], 1, PartVT, V);
00584   } else if (NumParts > 0) {
00585     // If the intermediate type was expanded, split each the value into
00586     // legal parts.
00587     assert(NumIntermediates != 0 && "division by zero");
00588     assert(NumParts % NumIntermediates == 0 &&
00589            "Must expand into a divisible number of parts!");
00590     unsigned Factor = NumParts / NumIntermediates;
00591     for (unsigned i = 0; i != NumIntermediates; ++i)
00592       getCopyToParts(DAG, DL, Ops[i], &Parts[i*Factor], Factor, PartVT, V);
00593   }
00594 }
00595 
00596 RegsForValue::RegsForValue() {}
00597 
00598 RegsForValue::RegsForValue(const SmallVector<unsigned, 4> &regs, MVT regvt,
00599                            EVT valuevt)
00600     : ValueVTs(1, valuevt), RegVTs(1, regvt), Regs(regs) {}
00601 
00602 RegsForValue::RegsForValue(LLVMContext &Context, const TargetLowering &TLI,
00603                            const DataLayout &DL, unsigned Reg, Type *Ty) {
00604   ComputeValueVTs(TLI, DL, Ty, ValueVTs);
00605 
00606   for (EVT ValueVT : ValueVTs) {
00607     unsigned NumRegs = TLI.getNumRegisters(Context, ValueVT);
00608     MVT RegisterVT = TLI.getRegisterType(Context, ValueVT);
00609     for (unsigned i = 0; i != NumRegs; ++i)
00610       Regs.push_back(Reg + i);
00611     RegVTs.push_back(RegisterVT);
00612     Reg += NumRegs;
00613   }
00614 }
00615 
00616 /// getCopyFromRegs - Emit a series of CopyFromReg nodes that copies from
00617 /// this value and returns the result as a ValueVT value.  This uses
00618 /// Chain/Flag as the input and updates them for the output Chain/Flag.
00619 /// If the Flag pointer is NULL, no flag is used.
00620 SDValue RegsForValue::getCopyFromRegs(SelectionDAG &DAG,
00621                                       FunctionLoweringInfo &FuncInfo,
00622                                       SDLoc dl,
00623                                       SDValue &Chain, SDValue *Flag,
00624                                       const Value *V) const {
00625   // A Value with type {} or [0 x %t] needs no registers.
00626   if (ValueVTs.empty())
00627     return SDValue();
00628 
00629   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
00630 
00631   // Assemble the legal parts into the final values.
00632   SmallVector<SDValue, 4> Values(ValueVTs.size());
00633   SmallVector<SDValue, 8> Parts;
00634   for (unsigned Value = 0, Part = 0, e = ValueVTs.size(); Value != e; ++Value) {
00635     // Copy the legal parts from the registers.
00636     EVT ValueVT = ValueVTs[Value];
00637     unsigned NumRegs = TLI.getNumRegisters(*DAG.getContext(), ValueVT);
00638     MVT RegisterVT = RegVTs[Value];
00639 
00640     Parts.resize(NumRegs);
00641     for (unsigned i = 0; i != NumRegs; ++i) {
00642       SDValue P;
00643       if (!Flag) {
00644         P = DAG.getCopyFromReg(Chain, dl, Regs[Part+i], RegisterVT);
00645       } else {
00646         P = DAG.getCopyFromReg(Chain, dl, Regs[Part+i], RegisterVT, *Flag);
00647         *Flag = P.getValue(2);
00648       }
00649 
00650       Chain = P.getValue(1);
00651       Parts[i] = P;
00652 
00653       // If the source register was virtual and if we know something about it,
00654       // add an assert node.
00655       if (!TargetRegisterInfo::isVirtualRegister(Regs[Part+i]) ||
00656           !RegisterVT.isInteger() || RegisterVT.isVector())
00657         continue;
00658 
00659       const FunctionLoweringInfo::LiveOutInfo *LOI =
00660         FuncInfo.GetLiveOutRegInfo(Regs[Part+i]);
00661       if (!LOI)
00662         continue;
00663 
00664       unsigned RegSize = RegisterVT.getSizeInBits();
00665       unsigned NumSignBits = LOI->NumSignBits;
00666       unsigned NumZeroBits = LOI->KnownZero.countLeadingOnes();
00667 
00668       if (NumZeroBits == RegSize) {
00669         // The current value is a zero.
00670         // Explicitly express that as it would be easier for
00671         // optimizations to kick in.
00672         Parts[i] = DAG.getConstant(0, dl, RegisterVT);
00673         continue;
00674       }
00675 
00676       // FIXME: We capture more information than the dag can represent.  For
00677       // now, just use the tightest assertzext/assertsext possible.
00678       bool isSExt = true;
00679       EVT FromVT(MVT::Other);
00680       if (NumSignBits == RegSize)
00681         isSExt = true, FromVT = MVT::i1;   // ASSERT SEXT 1
00682       else if (NumZeroBits >= RegSize-1)
00683         isSExt = false, FromVT = MVT::i1;  // ASSERT ZEXT 1
00684       else if (NumSignBits > RegSize-8)
00685         isSExt = true, FromVT = MVT::i8;   // ASSERT SEXT 8
00686       else if (NumZeroBits >= RegSize-8)
00687         isSExt = false, FromVT = MVT::i8;  // ASSERT ZEXT 8
00688       else if (NumSignBits > RegSize-16)
00689         isSExt = true, FromVT = MVT::i16;  // ASSERT SEXT 16
00690       else if (NumZeroBits >= RegSize-16)
00691         isSExt = false, FromVT = MVT::i16; // ASSERT ZEXT 16
00692       else if (NumSignBits > RegSize-32)
00693         isSExt = true, FromVT = MVT::i32;  // ASSERT SEXT 32
00694       else if (NumZeroBits >= RegSize-32)
00695         isSExt = false, FromVT = MVT::i32; // ASSERT ZEXT 32
00696       else
00697         continue;
00698 
00699       // Add an assertion node.
00700       assert(FromVT != MVT::Other);
00701       Parts[i] = DAG.getNode(isSExt ? ISD::AssertSext : ISD::AssertZext, dl,
00702                              RegisterVT, P, DAG.getValueType(FromVT));
00703     }
00704 
00705     Values[Value] = getCopyFromParts(DAG, dl, Parts.begin(),
00706                                      NumRegs, RegisterVT, ValueVT, V);
00707     Part += NumRegs;
00708     Parts.clear();
00709   }
00710 
00711   return DAG.getNode(ISD::MERGE_VALUES, dl, DAG.getVTList(ValueVTs), Values);
00712 }
00713 
00714 /// getCopyToRegs - Emit a series of CopyToReg nodes that copies the
00715 /// specified value into the registers specified by this object.  This uses
00716 /// Chain/Flag as the input and updates them for the output Chain/Flag.
00717 /// If the Flag pointer is NULL, no flag is used.
00718 void RegsForValue::getCopyToRegs(SDValue Val, SelectionDAG &DAG, SDLoc dl,
00719                                  SDValue &Chain, SDValue *Flag, const Value *V,
00720                                  ISD::NodeType PreferredExtendType) const {
00721   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
00722   ISD::NodeType ExtendKind = PreferredExtendType;
00723 
00724   // Get the list of the values's legal parts.
00725   unsigned NumRegs = Regs.size();
00726   SmallVector<SDValue, 8> Parts(NumRegs);
00727   for (unsigned Value = 0, Part = 0, e = ValueVTs.size(); Value != e; ++Value) {
00728     EVT ValueVT = ValueVTs[Value];
00729     unsigned NumParts = TLI.getNumRegisters(*DAG.getContext(), ValueVT);
00730     MVT RegisterVT = RegVTs[Value];
00731 
00732     if (ExtendKind == ISD::ANY_EXTEND && TLI.isZExtFree(Val, RegisterVT))
00733       ExtendKind = ISD::ZERO_EXTEND;
00734 
00735     getCopyToParts(DAG, dl, Val.getValue(Val.getResNo() + Value),
00736                    &Parts[Part], NumParts, RegisterVT, V, ExtendKind);
00737     Part += NumParts;
00738   }
00739 
00740   // Copy the parts into the registers.
00741   SmallVector<SDValue, 8> Chains(NumRegs);
00742   for (unsigned i = 0; i != NumRegs; ++i) {
00743     SDValue Part;
00744     if (!Flag) {
00745       Part = DAG.getCopyToReg(Chain, dl, Regs[i], Parts[i]);
00746     } else {
00747       Part = DAG.getCopyToReg(Chain, dl, Regs[i], Parts[i], *Flag);
00748       *Flag = Part.getValue(1);
00749     }
00750 
00751     Chains[i] = Part.getValue(0);
00752   }
00753 
00754   if (NumRegs == 1 || Flag)
00755     // If NumRegs > 1 && Flag is used then the use of the last CopyToReg is
00756     // flagged to it. That is the CopyToReg nodes and the user are considered
00757     // a single scheduling unit. If we create a TokenFactor and return it as
00758     // chain, then the TokenFactor is both a predecessor (operand) of the
00759     // user as well as a successor (the TF operands are flagged to the user).
00760     // c1, f1 = CopyToReg
00761     // c2, f2 = CopyToReg
00762     // c3     = TokenFactor c1, c2
00763     // ...
00764     //        = op c3, ..., f2
00765     Chain = Chains[NumRegs-1];
00766   else
00767     Chain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Chains);
00768 }
00769 
00770 /// AddInlineAsmOperands - Add this value to the specified inlineasm node
00771 /// operand list.  This adds the code marker and includes the number of
00772 /// values added into it.
00773 void RegsForValue::AddInlineAsmOperands(unsigned Code, bool HasMatching,
00774                                         unsigned MatchingIdx, SDLoc dl,
00775                                         SelectionDAG &DAG,
00776                                         std::vector<SDValue> &Ops) const {
00777   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
00778 
00779   unsigned Flag = InlineAsm::getFlagWord(Code, Regs.size());
00780   if (HasMatching)
00781     Flag = InlineAsm::getFlagWordForMatchingOp(Flag, MatchingIdx);
00782   else if (!Regs.empty() &&
00783            TargetRegisterInfo::isVirtualRegister(Regs.front())) {
00784     // Put the register class of the virtual registers in the flag word.  That
00785     // way, later passes can recompute register class constraints for inline
00786     // assembly as well as normal instructions.
00787     // Don't do this for tied operands that can use the regclass information
00788     // from the def.
00789     const MachineRegisterInfo &MRI = DAG.getMachineFunction().getRegInfo();
00790     const TargetRegisterClass *RC = MRI.getRegClass(Regs.front());
00791     Flag = InlineAsm::getFlagWordForRegClass(Flag, RC->getID());
00792   }
00793 
00794   SDValue Res = DAG.getTargetConstant(Flag, dl, MVT::i32);
00795   Ops.push_back(Res);
00796 
00797   unsigned SP = TLI.getStackPointerRegisterToSaveRestore();
00798   for (unsigned Value = 0, Reg = 0, e = ValueVTs.size(); Value != e; ++Value) {
00799     unsigned NumRegs = TLI.getNumRegisters(*DAG.getContext(), ValueVTs[Value]);
00800     MVT RegisterVT = RegVTs[Value];
00801     for (unsigned i = 0; i != NumRegs; ++i) {
00802       assert(Reg < Regs.size() && "Mismatch in # registers expected");
00803       unsigned TheReg = Regs[Reg++];
00804       Ops.push_back(DAG.getRegister(TheReg, RegisterVT));
00805 
00806       if (TheReg == SP && Code == InlineAsm::Kind_Clobber) {
00807         // If we clobbered the stack pointer, MFI should know about it.
00808         assert(DAG.getMachineFunction().getFrameInfo()->
00809             hasOpaqueSPAdjustment());
00810       }
00811     }
00812   }
00813 }
00814 
00815 void SelectionDAGBuilder::init(GCFunctionInfo *gfi, AliasAnalysis &aa,
00816                                const TargetLibraryInfo *li) {
00817   AA = &aa;
00818   GFI = gfi;
00819   LibInfo = li;
00820   DL = &DAG.getDataLayout();
00821   Context = DAG.getContext();
00822   LPadToCallSiteMap.clear();
00823 }
00824 
00825 /// clear - Clear out the current SelectionDAG and the associated
00826 /// state and prepare this SelectionDAGBuilder object to be used
00827 /// for a new block. This doesn't clear out information about
00828 /// additional blocks that are needed to complete switch lowering
00829 /// or PHI node updating; that information is cleared out as it is
00830 /// consumed.
00831 void SelectionDAGBuilder::clear() {
00832   NodeMap.clear();
00833   UnusedArgNodeMap.clear();
00834   PendingLoads.clear();
00835   PendingExports.clear();
00836   CurInst = nullptr;
00837   HasTailCall = false;
00838   SDNodeOrder = LowestSDNodeOrder;
00839   StatepointLowering.clear();
00840 }
00841 
00842 /// clearDanglingDebugInfo - Clear the dangling debug information
00843 /// map. This function is separated from the clear so that debug
00844 /// information that is dangling in a basic block can be properly
00845 /// resolved in a different basic block. This allows the
00846 /// SelectionDAG to resolve dangling debug information attached
00847 /// to PHI nodes.
00848 void SelectionDAGBuilder::clearDanglingDebugInfo() {
00849   DanglingDebugInfoMap.clear();
00850 }
00851 
00852 /// getRoot - Return the current virtual root of the Selection DAG,
00853 /// flushing any PendingLoad items. This must be done before emitting
00854 /// a store or any other node that may need to be ordered after any
00855 /// prior load instructions.
00856 ///
00857 SDValue SelectionDAGBuilder::getRoot() {
00858   if (PendingLoads.empty())
00859     return DAG.getRoot();
00860 
00861   if (PendingLoads.size() == 1) {
00862     SDValue Root = PendingLoads[0];
00863     DAG.setRoot(Root);
00864     PendingLoads.clear();
00865     return Root;
00866   }
00867 
00868   // Otherwise, we have to make a token factor node.
00869   SDValue Root = DAG.getNode(ISD::TokenFactor, getCurSDLoc(), MVT::Other,
00870                              PendingLoads);
00871   PendingLoads.clear();
00872   DAG.setRoot(Root);
00873   return Root;
00874 }
00875 
00876 /// getControlRoot - Similar to getRoot, but instead of flushing all the
00877 /// PendingLoad items, flush all the PendingExports items. It is necessary
00878 /// to do this before emitting a terminator instruction.
00879 ///
00880 SDValue SelectionDAGBuilder::getControlRoot() {
00881   SDValue Root = DAG.getRoot();
00882 
00883   if (PendingExports.empty())
00884     return Root;
00885 
00886   // Turn all of the CopyToReg chains into one factored node.
00887   if (Root.getOpcode() != ISD::EntryToken) {
00888     unsigned i = 0, e = PendingExports.size();
00889     for (; i != e; ++i) {
00890       assert(PendingExports[i].getNode()->getNumOperands() > 1);
00891       if (PendingExports[i].getNode()->getOperand(0) == Root)
00892         break;  // Don't add the root if we already indirectly depend on it.
00893     }
00894 
00895     if (i == e)
00896       PendingExports.push_back(Root);
00897   }
00898 
00899   Root = DAG.getNode(ISD::TokenFactor, getCurSDLoc(), MVT::Other,
00900                      PendingExports);
00901   PendingExports.clear();
00902   DAG.setRoot(Root);
00903   return Root;
00904 }
00905 
00906 void SelectionDAGBuilder::visit(const Instruction &I) {
00907   // Set up outgoing PHI node register values before emitting the terminator.
00908   if (isa<TerminatorInst>(&I))
00909     HandlePHINodesInSuccessorBlocks(I.getParent());
00910 
00911   ++SDNodeOrder;
00912 
00913   CurInst = &I;
00914 
00915   visit(I.getOpcode(), I);
00916 
00917   if (!isa<TerminatorInst>(&I) && !HasTailCall &&
00918       !isStatepoint(&I)) // statepoints handle their exports internally
00919     CopyToExportRegsIfNeeded(&I);
00920 
00921   CurInst = nullptr;
00922 }
00923 
00924 void SelectionDAGBuilder::visitPHI(const PHINode &) {
00925   llvm_unreachable("SelectionDAGBuilder shouldn't visit PHI nodes!");
00926 }
00927 
00928 void SelectionDAGBuilder::visit(unsigned Opcode, const User &I) {
00929   // Note: this doesn't use InstVisitor, because it has to work with
00930   // ConstantExpr's in addition to instructions.
00931   switch (Opcode) {
00932   default: llvm_unreachable("Unknown instruction type encountered!");
00933     // Build the switch statement using the Instruction.def file.
00934 #define HANDLE_INST(NUM, OPCODE, CLASS) \
00935     case Instruction::OPCODE: visit##OPCODE((const CLASS&)I); break;
00936 #include "llvm/IR/Instruction.def"
00937   }
00938 }
00939 
00940 // resolveDanglingDebugInfo - if we saw an earlier dbg_value referring to V,
00941 // generate the debug data structures now that we've seen its definition.
00942 void SelectionDAGBuilder::resolveDanglingDebugInfo(const Value *V,
00943                                                    SDValue Val) {
00944   DanglingDebugInfo &DDI = DanglingDebugInfoMap[V];
00945   if (DDI.getDI()) {
00946     const DbgValueInst *DI = DDI.getDI();
00947     DebugLoc dl = DDI.getdl();
00948     unsigned DbgSDNodeOrder = DDI.getSDNodeOrder();
00949     DILocalVariable *Variable = DI->getVariable();
00950     DIExpression *Expr = DI->getExpression();
00951     assert(Variable->isValidLocationForIntrinsic(dl) &&
00952            "Expected inlined-at fields to agree");
00953     uint64_t Offset = DI->getOffset();
00954     SDDbgValue *SDV;
00955     if (Val.getNode()) {
00956       if (!EmitFuncArgumentDbgValue(V, Variable, Expr, dl, Offset, false,
00957                                     Val)) {
00958         SDV = DAG.getDbgValue(Variable, Expr, Val.getNode(), Val.getResNo(),
00959                               false, Offset, dl, DbgSDNodeOrder);
00960         DAG.AddDbgValue(SDV, Val.getNode(), false);
00961       }
00962     } else
00963       DEBUG(dbgs() << "Dropping debug info for " << *DI << "\n");
00964     DanglingDebugInfoMap[V] = DanglingDebugInfo();
00965   }
00966 }
00967 
00968 /// getCopyFromRegs - If there was virtual register allocated for the value V
00969 /// emit CopyFromReg of the specified type Ty. Return empty SDValue() otherwise.
00970 SDValue SelectionDAGBuilder::getCopyFromRegs(const Value *V, Type *Ty) {
00971   DenseMap<const Value *, unsigned>::iterator It = FuncInfo.ValueMap.find(V);
00972   SDValue Result;
00973 
00974   if (It != FuncInfo.ValueMap.end()) {
00975     unsigned InReg = It->second;
00976     RegsForValue RFV(*DAG.getContext(), DAG.getTargetLoweringInfo(),
00977                      DAG.getDataLayout(), InReg, Ty);
00978     SDValue Chain = DAG.getEntryNode();
00979     Result = RFV.getCopyFromRegs(DAG, FuncInfo, getCurSDLoc(), Chain, nullptr, V);
00980     resolveDanglingDebugInfo(V, Result);
00981   }
00982 
00983   return Result;
00984 }
00985 
00986 /// getValue - Return an SDValue for the given Value.
00987 SDValue SelectionDAGBuilder::getValue(const Value *V) {
00988   // If we already have an SDValue for this value, use it. It's important
00989   // to do this first, so that we don't create a CopyFromReg if we already
00990   // have a regular SDValue.
00991   SDValue &N = NodeMap[V];
00992   if (N.getNode()) return N;
00993 
00994   // If there's a virtual register allocated and initialized for this
00995   // value, use it.
00996   SDValue copyFromReg = getCopyFromRegs(V, V->getType());
00997   if (copyFromReg.getNode()) {
00998     return copyFromReg;
00999   }
01000 
01001   // Otherwise create a new SDValue and remember it.
01002   SDValue Val = getValueImpl(V);
01003   NodeMap[V] = Val;
01004   resolveDanglingDebugInfo(V, Val);
01005   return Val;
01006 }
01007 
01008 // Return true if SDValue exists for the given Value
01009 bool SelectionDAGBuilder::findValue(const Value *V) const {
01010   return (NodeMap.find(V) != NodeMap.end()) ||
01011     (FuncInfo.ValueMap.find(V) != FuncInfo.ValueMap.end());
01012 }
01013 
01014 /// getNonRegisterValue - Return an SDValue for the given Value, but
01015 /// don't look in FuncInfo.ValueMap for a virtual register.
01016 SDValue SelectionDAGBuilder::getNonRegisterValue(const Value *V) {
01017   // If we already have an SDValue for this value, use it.
01018   SDValue &N = NodeMap[V];
01019   if (N.getNode()) {
01020     if (isa<ConstantSDNode>(N) || isa<ConstantFPSDNode>(N)) {
01021       // Remove the debug location from the node as the node is about to be used
01022       // in a location which may differ from the original debug location.  This
01023       // is relevant to Constant and ConstantFP nodes because they can appear
01024       // as constant expressions inside PHI nodes.
01025       N->setDebugLoc(DebugLoc());
01026     }
01027     return N;
01028   }
01029 
01030   // Otherwise create a new SDValue and remember it.
01031   SDValue Val = getValueImpl(V);
01032   NodeMap[V] = Val;
01033   resolveDanglingDebugInfo(V, Val);
01034   return Val;
01035 }
01036 
01037 /// getValueImpl - Helper function for getValue and getNonRegisterValue.
01038 /// Create an SDValue for the given value.
01039 SDValue SelectionDAGBuilder::getValueImpl(const Value *V) {
01040   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
01041 
01042   if (const Constant *C = dyn_cast<Constant>(V)) {
01043     EVT VT = TLI.getValueType(DAG.getDataLayout(), V->getType(), true);
01044 
01045     if (const ConstantInt *CI = dyn_cast<ConstantInt>(C))
01046       return DAG.getConstant(*CI, getCurSDLoc(), VT);
01047 
01048     if (const GlobalValue *GV = dyn_cast<GlobalValue>(C))
01049       return DAG.getGlobalAddress(GV, getCurSDLoc(), VT);
01050 
01051     if (isa<ConstantPointerNull>(C)) {
01052       unsigned AS = V->getType()->getPointerAddressSpace();
01053       return DAG.getConstant(0, getCurSDLoc(),
01054                              TLI.getPointerTy(DAG.getDataLayout(), AS));
01055     }
01056 
01057     if (const ConstantFP *CFP = dyn_cast<ConstantFP>(C))
01058       return DAG.getConstantFP(*CFP, getCurSDLoc(), VT);
01059 
01060     if (isa<UndefValue>(C) && !V->getType()->isAggregateType())
01061       return DAG.getUNDEF(VT);
01062 
01063     if (const ConstantExpr *CE = dyn_cast<ConstantExpr>(C)) {
01064       visit(CE->getOpcode(), *CE);
01065       SDValue N1 = NodeMap[V];
01066       assert(N1.getNode() && "visit didn't populate the NodeMap!");
01067       return N1;
01068     }
01069 
01070     if (isa<ConstantStruct>(C) || isa<ConstantArray>(C)) {
01071       SmallVector<SDValue, 4> Constants;
01072       for (User::const_op_iterator OI = C->op_begin(), OE = C->op_end();
01073            OI != OE; ++OI) {
01074         SDNode *Val = getValue(*OI).getNode();
01075         // If the operand is an empty aggregate, there are no values.
01076         if (!Val) continue;
01077         // Add each leaf value from the operand to the Constants list
01078         // to form a flattened list of all the values.
01079         for (unsigned i = 0, e = Val->getNumValues(); i != e; ++i)
01080           Constants.push_back(SDValue(Val, i));
01081       }
01082 
01083       return DAG.getMergeValues(Constants, getCurSDLoc());
01084     }
01085 
01086     if (const ConstantDataSequential *CDS =
01087           dyn_cast<ConstantDataSequential>(C)) {
01088       SmallVector<SDValue, 4> Ops;
01089       for (unsigned i = 0, e = CDS->getNumElements(); i != e; ++i) {
01090         SDNode *Val = getValue(CDS->getElementAsConstant(i)).getNode();
01091         // Add each leaf value from the operand to the Constants list
01092         // to form a flattened list of all the values.
01093         for (unsigned i = 0, e = Val->getNumValues(); i != e; ++i)
01094           Ops.push_back(SDValue(Val, i));
01095       }
01096 
01097       if (isa<ArrayType>(CDS->getType()))
01098         return DAG.getMergeValues(Ops, getCurSDLoc());
01099       return NodeMap[V] = DAG.getNode(ISD::BUILD_VECTOR, getCurSDLoc(),
01100                                       VT, Ops);
01101     }
01102 
01103     if (C->getType()->isStructTy() || C->getType()->isArrayTy()) {
01104       assert((isa<ConstantAggregateZero>(C) || isa<UndefValue>(C)) &&
01105              "Unknown struct or array constant!");
01106 
01107       SmallVector<EVT, 4> ValueVTs;
01108       ComputeValueVTs(TLI, DAG.getDataLayout(), C->getType(), ValueVTs);
01109       unsigned NumElts = ValueVTs.size();
01110       if (NumElts == 0)
01111         return SDValue(); // empty struct
01112       SmallVector<SDValue, 4> Constants(NumElts);
01113       for (unsigned i = 0; i != NumElts; ++i) {
01114         EVT EltVT = ValueVTs[i];
01115         if (isa<UndefValue>(C))
01116           Constants[i] = DAG.getUNDEF(EltVT);
01117         else if (EltVT.isFloatingPoint())
01118           Constants[i] = DAG.getConstantFP(0, getCurSDLoc(), EltVT);
01119         else
01120           Constants[i] = DAG.getConstant(0, getCurSDLoc(), EltVT);
01121       }
01122 
01123       return DAG.getMergeValues(Constants, getCurSDLoc());
01124     }
01125 
01126     if (const BlockAddress *BA = dyn_cast<BlockAddress>(C))
01127       return DAG.getBlockAddress(BA, VT);
01128 
01129     VectorType *VecTy = cast<VectorType>(V->getType());
01130     unsigned NumElements = VecTy->getNumElements();
01131 
01132     // Now that we know the number and type of the elements, get that number of
01133     // elements into the Ops array based on what kind of constant it is.
01134     SmallVector<SDValue, 16> Ops;
01135     if (const ConstantVector *CV = dyn_cast<ConstantVector>(C)) {
01136       for (unsigned i = 0; i != NumElements; ++i)
01137         Ops.push_back(getValue(CV->getOperand(i)));
01138     } else {
01139       assert(isa<ConstantAggregateZero>(C) && "Unknown vector constant!");
01140       EVT EltVT =
01141           TLI.getValueType(DAG.getDataLayout(), VecTy->getElementType());
01142 
01143       SDValue Op;
01144       if (EltVT.isFloatingPoint())
01145         Op = DAG.getConstantFP(0, getCurSDLoc(), EltVT);
01146       else
01147         Op = DAG.getConstant(0, getCurSDLoc(), EltVT);
01148       Ops.assign(NumElements, Op);
01149     }
01150 
01151     // Create a BUILD_VECTOR node.
01152     return NodeMap[V] = DAG.getNode(ISD::BUILD_VECTOR, getCurSDLoc(), VT, Ops);
01153   }
01154 
01155   // If this is a static alloca, generate it as the frameindex instead of
01156   // computation.
01157   if (const AllocaInst *AI = dyn_cast<AllocaInst>(V)) {
01158     DenseMap<const AllocaInst*, int>::iterator SI =
01159       FuncInfo.StaticAllocaMap.find(AI);
01160     if (SI != FuncInfo.StaticAllocaMap.end())
01161       return DAG.getFrameIndex(SI->second,
01162                                TLI.getPointerTy(DAG.getDataLayout()));
01163   }
01164 
01165   // If this is an instruction which fast-isel has deferred, select it now.
01166   if (const Instruction *Inst = dyn_cast<Instruction>(V)) {
01167     unsigned InReg = FuncInfo.InitializeRegForValue(Inst);
01168     RegsForValue RFV(*DAG.getContext(), TLI, DAG.getDataLayout(), InReg,
01169                      Inst->getType());
01170     SDValue Chain = DAG.getEntryNode();
01171     return RFV.getCopyFromRegs(DAG, FuncInfo, getCurSDLoc(), Chain, nullptr, V);
01172   }
01173 
01174   llvm_unreachable("Can't get register for value!");
01175 }
01176 
01177 void SelectionDAGBuilder::visitCatchPad(const CatchPadInst &I) {
01178   auto Pers = classifyEHPersonality(FuncInfo.Fn->getPersonalityFn());
01179   bool IsMSVCCXX = Pers == EHPersonality::MSVC_CXX;
01180   bool IsCoreCLR = Pers == EHPersonality::CoreCLR;
01181   MachineBasicBlock *CatchPadMBB = FuncInfo.MBB;
01182   // In MSVC C++ and CoreCLR, catchblocks are funclets and need prologues.
01183   if (IsMSVCCXX || IsCoreCLR)
01184     CatchPadMBB->setIsEHFuncletEntry();
01185 
01186   DAG.setRoot(DAG.getNode(ISD::CATCHPAD, getCurSDLoc(), MVT::Other, getControlRoot()));
01187 }
01188 
01189 void SelectionDAGBuilder::visitCatchRet(const CatchReturnInst &I) {
01190   // Update machine-CFG edge.
01191   MachineBasicBlock *TargetMBB = FuncInfo.MBBMap[I.getSuccessor()];
01192   FuncInfo.MBB->addSuccessor(TargetMBB);
01193 
01194   auto Pers = classifyEHPersonality(FuncInfo.Fn->getPersonalityFn());
01195   bool IsSEH = isAsynchronousEHPersonality(Pers);
01196   if (IsSEH) {
01197     // If this is not a fall-through branch or optimizations are switched off,
01198     // emit the branch.
01199     if (TargetMBB != NextBlock(FuncInfo.MBB) ||
01200         TM.getOptLevel() == CodeGenOpt::None)
01201       DAG.setRoot(DAG.getNode(ISD::BR, getCurSDLoc(), MVT::Other,
01202                               getControlRoot(), DAG.getBasicBlock(TargetMBB)));
01203     return;
01204   }
01205 
01206   // Figure out the funclet membership for the catchret's successor.
01207   // This will be used by the FuncletLayout pass to determine how to order the
01208   // BB's.
01209   // A 'catchret' returns to the outer scope's color.
01210   Value *ParentPad = I.getCatchSwitchParentPad();
01211   const BasicBlock *SuccessorColor;
01212   if (isa<ConstantTokenNone>(ParentPad))
01213     SuccessorColor = &FuncInfo.Fn->getEntryBlock();
01214   else
01215     SuccessorColor = cast<Instruction>(ParentPad)->getParent();
01216   assert(SuccessorColor && "No parent funclet for catchret!");
01217   MachineBasicBlock *SuccessorColorMBB = FuncInfo.MBBMap[SuccessorColor];
01218   assert(SuccessorColorMBB && "No MBB for SuccessorColor!");
01219 
01220   // Create the terminator node.
01221   SDValue Ret = DAG.getNode(ISD::CATCHRET, getCurSDLoc(), MVT::Other,
01222                             getControlRoot(), DAG.getBasicBlock(TargetMBB),
01223                             DAG.getBasicBlock(SuccessorColorMBB));
01224   DAG.setRoot(Ret);
01225 }
01226 
01227 void SelectionDAGBuilder::visitCleanupPad(const CleanupPadInst &CPI) {
01228   // Don't emit any special code for the cleanuppad instruction. It just marks
01229   // the start of a funclet.
01230   FuncInfo.MBB->setIsEHFuncletEntry();
01231   FuncInfo.MBB->setIsCleanupFuncletEntry();
01232 }
01233 
01234 /// When an invoke or a cleanupret unwinds to the next EH pad, there are
01235 /// many places it could ultimately go. In the IR, we have a single unwind
01236 /// destination, but in the machine CFG, we enumerate all the possible blocks.
01237 /// This function skips over imaginary basic blocks that hold catchswitch
01238 /// instructions, and finds all the "real" machine
01239 /// basic block destinations. As those destinations may not be successors of
01240 /// EHPadBB, here we also calculate the edge probability to those destinations.
01241 /// The passed-in Prob is the edge probability to EHPadBB.
01242 static void findUnwindDestinations(
01243     FunctionLoweringInfo &FuncInfo, const BasicBlock *EHPadBB,
01244     BranchProbability Prob,
01245     SmallVectorImpl<std::pair<MachineBasicBlock *, BranchProbability>>
01246         &UnwindDests) {
01247   EHPersonality Personality =
01248     classifyEHPersonality(FuncInfo.Fn->getPersonalityFn());
01249   bool IsMSVCCXX = Personality == EHPersonality::MSVC_CXX;
01250   bool IsCoreCLR = Personality == EHPersonality::CoreCLR;
01251 
01252   while (EHPadBB) {
01253     const Instruction *Pad = EHPadBB->getFirstNonPHI();
01254     BasicBlock *NewEHPadBB = nullptr;
01255     if (isa<LandingPadInst>(Pad)) {
01256       // Stop on landingpads. They are not funclets.
01257       UnwindDests.emplace_back(FuncInfo.MBBMap[EHPadBB], Prob);
01258       break;
01259     } else if (isa<CleanupPadInst>(Pad)) {
01260       // Stop on cleanup pads. Cleanups are always funclet entries for all known
01261       // personalities.
01262       UnwindDests.emplace_back(FuncInfo.MBBMap[EHPadBB], Prob);
01263       UnwindDests.back().first->setIsEHFuncletEntry();
01264       break;
01265     } else if (auto *CatchSwitch = dyn_cast<CatchSwitchInst>(Pad)) {
01266       // Add the catchpad handlers to the possible destinations.
01267       for (const BasicBlock *CatchPadBB : CatchSwitch->handlers()) {
01268         UnwindDests.emplace_back(FuncInfo.MBBMap[CatchPadBB], Prob);
01269         // For MSVC++ and the CLR, catchblocks are funclets and need prologues.
01270         if (IsMSVCCXX || IsCoreCLR)
01271           UnwindDests.back().first->setIsEHFuncletEntry();
01272       }
01273       NewEHPadBB = CatchSwitch->getUnwindDest();
01274     } else {
01275       continue;
01276     }
01277 
01278     BranchProbabilityInfo *BPI = FuncInfo.BPI;
01279     if (BPI && NewEHPadBB)
01280       Prob *= BPI->getEdgeProbability(EHPadBB, NewEHPadBB);
01281     EHPadBB = NewEHPadBB;
01282   }
01283 }
01284 
01285 void SelectionDAGBuilder::visitCleanupRet(const CleanupReturnInst &I) {
01286   // Update successor info.
01287   SmallVector<std::pair<MachineBasicBlock *, BranchProbability>, 1> UnwindDests;
01288   auto UnwindDest = I.getUnwindDest();
01289   BranchProbabilityInfo *BPI = FuncInfo.BPI;
01290   BranchProbability UnwindDestProb =
01291       (BPI && UnwindDest)
01292           ? BPI->getEdgeProbability(FuncInfo.MBB->getBasicBlock(), UnwindDest)
01293           : BranchProbability::getZero();
01294   findUnwindDestinations(FuncInfo, UnwindDest, UnwindDestProb, UnwindDests);
01295   for (auto &UnwindDest : UnwindDests) {
01296     UnwindDest.first->setIsEHPad();
01297     addSuccessorWithProb(FuncInfo.MBB, UnwindDest.first, UnwindDest.second);
01298   }
01299   FuncInfo.MBB->normalizeSuccProbs();
01300 
01301   // Create the terminator node.
01302   SDValue Ret =
01303       DAG.getNode(ISD::CLEANUPRET, getCurSDLoc(), MVT::Other, getControlRoot());
01304   DAG.setRoot(Ret);
01305 }
01306 
01307 void SelectionDAGBuilder::visitCatchSwitch(const CatchSwitchInst &CSI) {
01308   report_fatal_error("visitCatchSwitch not yet implemented!");
01309 }
01310 
01311 void SelectionDAGBuilder::visitRet(const ReturnInst &I) {
01312   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
01313   auto &DL = DAG.getDataLayout();
01314   SDValue Chain = getControlRoot();
01315   SmallVector<ISD::OutputArg, 8> Outs;
01316   SmallVector<SDValue, 8> OutVals;
01317 
01318   if (!FuncInfo.CanLowerReturn) {
01319     unsigned DemoteReg = FuncInfo.DemoteRegister;
01320     const Function *F = I.getParent()->getParent();
01321 
01322     // Emit a store of the return value through the virtual register.
01323     // Leave Outs empty so that LowerReturn won't try to load return
01324     // registers the usual way.
01325     SmallVector<EVT, 1> PtrValueVTs;
01326     ComputeValueVTs(TLI, DL, PointerType::getUnqual(F->getReturnType()),
01327                     PtrValueVTs);
01328 
01329     SDValue RetPtr = DAG.getCopyFromReg(DAG.getEntryNode(), getCurSDLoc(),
01330                                         DemoteReg, PtrValueVTs[0]);
01331     SDValue RetOp = getValue(I.getOperand(0));
01332 
01333     SmallVector<EVT, 4> ValueVTs;
01334     SmallVector<uint64_t, 4> Offsets;
01335     ComputeValueVTs(TLI, DL, I.getOperand(0)->getType(), ValueVTs, &Offsets);
01336     unsigned NumValues = ValueVTs.size();
01337 
01338     // An aggregate return value cannot wrap around the address space, so
01339     // offsets to its parts don't wrap either.
01340     SDNodeFlags Flags;
01341     Flags.setNoUnsignedWrap(true);
01342 
01343     SmallVector<SDValue, 4> Chains(NumValues);
01344     for (unsigned i = 0; i != NumValues; ++i) {
01345       SDValue Add = DAG.getNode(ISD::ADD, getCurSDLoc(),
01346                                 RetPtr.getValueType(), RetPtr,
01347                                 DAG.getIntPtrConstant(Offsets[i],
01348                                                       getCurSDLoc()),
01349                                 &Flags);
01350       Chains[i] =
01351         DAG.getStore(Chain, getCurSDLoc(),
01352                      SDValue(RetOp.getNode(), RetOp.getResNo() + i),
01353                      // FIXME: better loc info would be nice.
01354                      Add, MachinePointerInfo(), false, false, 0);
01355     }
01356 
01357     Chain = DAG.getNode(ISD::TokenFactor, getCurSDLoc(),
01358                         MVT::Other, Chains);
01359   } else if (I.getNumOperands() != 0) {
01360     SmallVector<EVT, 4> ValueVTs;
01361     ComputeValueVTs(TLI, DL, I.getOperand(0)->getType(), ValueVTs);
01362     unsigned NumValues = ValueVTs.size();
01363     if (NumValues) {
01364       SDValue RetOp = getValue(I.getOperand(0));
01365 
01366       const Function *F = I.getParent()->getParent();
01367 
01368       ISD::NodeType ExtendKind = ISD::ANY_EXTEND;
01369       if (F->getAttributes().hasAttribute(AttributeSet::ReturnIndex,
01370                                           Attribute::SExt))
01371         ExtendKind = ISD::SIGN_EXTEND;
01372       else if (F->getAttributes().hasAttribute(AttributeSet::ReturnIndex,
01373                                                Attribute::ZExt))
01374         ExtendKind = ISD::ZERO_EXTEND;
01375 
01376       LLVMContext &Context = F->getContext();
01377       bool RetInReg = F->getAttributes().hasAttribute(AttributeSet::ReturnIndex,
01378                                                       Attribute::InReg);
01379 
01380       for (unsigned j = 0; j != NumValues; ++j) {
01381         EVT VT = ValueVTs[j];
01382 
01383         if (ExtendKind != ISD::ANY_EXTEND && VT.isInteger())
01384           VT = TLI.getTypeForExtArgOrReturn(Context, VT, ExtendKind);
01385 
01386         unsigned NumParts = TLI.getNumRegisters(Context, VT);
01387         MVT PartVT = TLI.getRegisterType(Context, VT);
01388         SmallVector<SDValue, 4> Parts(NumParts);
01389         getCopyToParts(DAG, getCurSDLoc(),
01390                        SDValue(RetOp.getNode(), RetOp.getResNo() + j),
01391                        &Parts[0], NumParts, PartVT, &I, ExtendKind);
01392 
01393         // 'inreg' on function refers to return value
01394         ISD::ArgFlagsTy Flags = ISD::ArgFlagsTy();
01395         if (RetInReg)
01396           Flags.setInReg();
01397 
01398         // Propagate extension type if any
01399         if (ExtendKind == ISD::SIGN_EXTEND)
01400           Flags.setSExt();
01401         else if (ExtendKind == ISD::ZERO_EXTEND)
01402           Flags.setZExt();
01403 
01404         for (unsigned i = 0; i < NumParts; ++i) {
01405           Outs.push_back(ISD::OutputArg(Flags, Parts[i].getValueType(),
01406                                         VT, /*isfixed=*/true, 0, 0));
01407           OutVals.push_back(Parts[i]);
01408         }
01409       }
01410     }
01411   }
01412 
01413   bool isVarArg = DAG.getMachineFunction().getFunction()->isVarArg();
01414   CallingConv::ID CallConv =
01415     DAG.getMachineFunction().getFunction()->getCallingConv();
01416   Chain = DAG.getTargetLoweringInfo().LowerReturn(
01417       Chain, CallConv, isVarArg, Outs, OutVals, getCurSDLoc(), DAG);
01418 
01419   // Verify that the target's LowerReturn behaved as expected.
01420   assert(Chain.getNode() && Chain.getValueType() == MVT::Other &&
01421          "LowerReturn didn't return a valid chain!");
01422 
01423   // Update the DAG with the new chain value resulting from return lowering.
01424   DAG.setRoot(Chain);
01425 }
01426 
01427 /// CopyToExportRegsIfNeeded - If the given value has virtual registers
01428 /// created for it, emit nodes to copy the value into the virtual
01429 /// registers.
01430 void SelectionDAGBuilder::CopyToExportRegsIfNeeded(const Value *V) {
01431   // Skip empty types
01432   if (V->getType()->isEmptyTy())
01433     return;
01434 
01435   DenseMap<const Value *, unsigned>::iterator VMI = FuncInfo.ValueMap.find(V);
01436   if (VMI != FuncInfo.ValueMap.end()) {
01437     assert(!V->use_empty() && "Unused value assigned virtual registers!");
01438     CopyValueToVirtualRegister(V, VMI->second);
01439   }
01440 }
01441 
01442 /// ExportFromCurrentBlock - If this condition isn't known to be exported from
01443 /// the current basic block, add it to ValueMap now so that we'll get a
01444 /// CopyTo/FromReg.
01445 void SelectionDAGBuilder::ExportFromCurrentBlock(const Value *V) {
01446   // No need to export constants.
01447   if (!isa<Instruction>(V) && !isa<Argument>(V)) return;
01448 
01449   // Already exported?
01450   if (FuncInfo.isExportedInst(V)) return;
01451 
01452   unsigned Reg = FuncInfo.InitializeRegForValue(V);
01453   CopyValueToVirtualRegister(V, Reg);
01454 }
01455 
01456 bool SelectionDAGBuilder::isExportableFromCurrentBlock(const Value *V,
01457                                                      const BasicBlock *FromBB) {
01458   // The operands of the setcc have to be in this block.  We don't know
01459   // how to export them from some other block.
01460   if (const Instruction *VI = dyn_cast<Instruction>(V)) {
01461     // Can export from current BB.
01462     if (VI->getParent() == FromBB)
01463       return true;
01464 
01465     // Is already exported, noop.
01466     return FuncInfo.isExportedInst(V);
01467   }
01468 
01469   // If this is an argument, we can export it if the BB is the entry block or
01470   // if it is already exported.
01471   if (isa<Argument>(V)) {
01472     if (FromBB == &FromBB->getParent()->getEntryBlock())
01473       return true;
01474 
01475     // Otherwise, can only export this if it is already exported.
01476     return FuncInfo.isExportedInst(V);
01477   }
01478 
01479   // Otherwise, constants can always be exported.
01480   return true;
01481 }
01482 
01483 /// Return branch probability calculated by BranchProbabilityInfo for IR blocks.
01484 BranchProbability
01485 SelectionDAGBuilder::getEdgeProbability(const MachineBasicBlock *Src,
01486                                         const MachineBasicBlock *Dst) const {
01487   BranchProbabilityInfo *BPI = FuncInfo.BPI;
01488   const BasicBlock *SrcBB = Src->getBasicBlock();
01489   const BasicBlock *DstBB = Dst->getBasicBlock();
01490   if (!BPI) {
01491     // If BPI is not available, set the default probability as 1 / N, where N is
01492     // the number of successors.
01493     auto SuccSize = std::max<uint32_t>(
01494         std::distance(succ_begin(SrcBB), succ_end(SrcBB)), 1);
01495     return BranchProbability(1, SuccSize);
01496   }
01497   return BPI->getEdgeProbability(SrcBB, DstBB);
01498 }
01499 
01500 void SelectionDAGBuilder::addSuccessorWithProb(MachineBasicBlock *Src,
01501                                                MachineBasicBlock *Dst,
01502                                                BranchProbability Prob) {
01503   if (!FuncInfo.BPI)
01504     Src->addSuccessorWithoutProb(Dst);
01505   else {
01506     if (Prob.isUnknown())
01507       Prob = getEdgeProbability(Src, Dst);
01508     Src->addSuccessor(Dst, Prob);
01509   }
01510 }
01511 
01512 static bool InBlock(const Value *V, const BasicBlock *BB) {
01513   if (const Instruction *I = dyn_cast<Instruction>(V))
01514     return I->getParent() == BB;
01515   return true;
01516 }
01517 
01518 /// EmitBranchForMergedCondition - Helper method for FindMergedConditions.
01519 /// This function emits a branch and is used at the leaves of an OR or an
01520 /// AND operator tree.
01521 ///
01522 void
01523 SelectionDAGBuilder::EmitBranchForMergedCondition(const Value *Cond,
01524                                                   MachineBasicBlock *TBB,
01525                                                   MachineBasicBlock *FBB,
01526                                                   MachineBasicBlock *CurBB,
01527                                                   MachineBasicBlock *SwitchBB,
01528                                                   BranchProbability TProb,
01529                                                   BranchProbability FProb) {
01530   const BasicBlock *BB = CurBB->getBasicBlock();
01531 
01532   // If the leaf of the tree is a comparison, merge the condition into
01533   // the caseblock.
01534   if (const CmpInst *BOp = dyn_cast<CmpInst>(Cond)) {
01535     // The operands of the cmp have to be in this block.  We don't know
01536     // how to export them from some other block.  If this is the first block
01537     // of the sequence, no exporting is needed.
01538     if (CurBB == SwitchBB ||
01539         (isExportableFromCurrentBlock(BOp->getOperand(0), BB) &&
01540          isExportableFromCurrentBlock(BOp->getOperand(1), BB))) {
01541       ISD::CondCode Condition;
01542       if (const ICmpInst *IC = dyn_cast<ICmpInst>(Cond)) {
01543         Condition = getICmpCondCode(IC->getPredicate());
01544       } else {
01545         const FCmpInst *FC = cast<FCmpInst>(Cond);
01546         Condition = getFCmpCondCode(FC->getPredicate());
01547         if (TM.Options.NoNaNsFPMath)
01548           Condition = getFCmpCodeWithoutNaN(Condition);
01549       }
01550 
01551       CaseBlock CB(Condition, BOp->getOperand(0), BOp->getOperand(1), nullptr,
01552                    TBB, FBB, CurBB, TProb, FProb);
01553       SwitchCases.push_back(CB);
01554       return;
01555     }
01556   }
01557 
01558   // Create a CaseBlock record representing this branch.
01559   CaseBlock CB(ISD::SETEQ, Cond, ConstantInt::getTrue(*DAG.getContext()),
01560                nullptr, TBB, FBB, CurBB, TProb, FProb);
01561   SwitchCases.push_back(CB);
01562 }
01563 
01564 /// FindMergedConditions - If Cond is an expression like
01565 void SelectionDAGBuilder::FindMergedConditions(const Value *Cond,
01566                                                MachineBasicBlock *TBB,
01567                                                MachineBasicBlock *FBB,
01568                                                MachineBasicBlock *CurBB,
01569                                                MachineBasicBlock *SwitchBB,
01570                                                Instruction::BinaryOps Opc,
01571                                                BranchProbability TProb,
01572                                                BranchProbability FProb) {
01573   // If this node is not part of the or/and tree, emit it as a branch.
01574   const Instruction *BOp = dyn_cast<Instruction>(Cond);
01575   if (!BOp || !(isa<BinaryOperator>(BOp) || isa<CmpInst>(BOp)) ||
01576       (unsigned)BOp->getOpcode() != Opc || !BOp->hasOneUse() ||
01577       BOp->getParent() != CurBB->getBasicBlock() ||
01578       !InBlock(BOp->getOperand(0), CurBB->getBasicBlock()) ||
01579       !InBlock(BOp->getOperand(1), CurBB->getBasicBlock())) {
01580     EmitBranchForMergedCondition(Cond, TBB, FBB, CurBB, SwitchBB,
01581                                  TProb, FProb);
01582     return;
01583   }
01584 
01585   //  Create TmpBB after CurBB.
01586   MachineFunction::iterator BBI(CurBB);
01587   MachineFunction &MF = DAG.getMachineFunction();
01588   MachineBasicBlock *TmpBB = MF.CreateMachineBasicBlock(CurBB->getBasicBlock());
01589   CurBB->getParent()->insert(++BBI, TmpBB);
01590 
01591   if (Opc == Instruction::Or) {
01592     // Codegen X | Y as:
01593     // BB1:
01594     //   jmp_if_X TBB
01595     //   jmp TmpBB
01596     // TmpBB:
01597     //   jmp_if_Y TBB
01598     //   jmp FBB
01599     //
01600 
01601     // We have flexibility in setting Prob for BB1 and Prob for TmpBB.
01602     // The requirement is that
01603     //   TrueProb for BB1 + (FalseProb for BB1 * TrueProb for TmpBB)
01604     //     = TrueProb for original BB.
01605     // Assuming the original probabilities are A and B, one choice is to set
01606     // BB1's probabilities to A/2 and A/2+B, and set TmpBB's probabilities to
01607     // A/(1+B) and 2B/(1+B). This choice assumes that
01608     //   TrueProb for BB1 == FalseProb for BB1 * TrueProb for TmpBB.
01609     // Another choice is to assume TrueProb for BB1 equals to TrueProb for
01610     // TmpBB, but the math is more complicated.
01611 
01612     auto NewTrueProb = TProb / 2;
01613     auto NewFalseProb = TProb / 2 + FProb;
01614     // Emit the LHS condition.
01615     FindMergedConditions(BOp->getOperand(0), TBB, TmpBB, CurBB, SwitchBB, Opc,
01616                          NewTrueProb, NewFalseProb);
01617 
01618     // Normalize A/2 and B to get A/(1+B) and 2B/(1+B).
01619     SmallVector<BranchProbability, 2> Probs{TProb / 2, FProb};
01620     BranchProbability::normalizeProbabilities(Probs.begin(), Probs.end());
01621     // Emit the RHS condition into TmpBB.
01622     FindMergedConditions(BOp->getOperand(1), TBB, FBB, TmpBB, SwitchBB, Opc,
01623                          Probs[0], Probs[1]);
01624   } else {
01625     assert(Opc == Instruction::And && "Unknown merge op!");
01626     // Codegen X & Y as:
01627     // BB1:
01628     //   jmp_if_X TmpBB
01629     //   jmp FBB
01630     // TmpBB:
01631     //   jmp_if_Y TBB
01632     //   jmp FBB
01633     //
01634     //  This requires creation of TmpBB after CurBB.
01635 
01636     // We have flexibility in setting Prob for BB1 and Prob for TmpBB.
01637     // The requirement is that
01638     //   FalseProb for BB1 + (TrueProb for BB1 * FalseProb for TmpBB)
01639     //     = FalseProb for original BB.
01640     // Assuming the original probabilities are A and B, one choice is to set
01641     // BB1's probabilities to A+B/2 and B/2, and set TmpBB's probabilities to
01642     // 2A/(1+A) and B/(1+A). This choice assumes that FalseProb for BB1 ==
01643     // TrueProb for BB1 * FalseProb for TmpBB.
01644 
01645     auto NewTrueProb = TProb + FProb / 2;
01646     auto NewFalseProb = FProb / 2;
01647     // Emit the LHS condition.
01648     FindMergedConditions(BOp->getOperand(0), TmpBB, FBB, CurBB, SwitchBB, Opc,
01649                          NewTrueProb, NewFalseProb);
01650 
01651     // Normalize A and B/2 to get 2A/(1+A) and B/(1+A).
01652     SmallVector<BranchProbability, 2> Probs{TProb, FProb / 2};
01653     BranchProbability::normalizeProbabilities(Probs.begin(), Probs.end());
01654     // Emit the RHS condition into TmpBB.
01655     FindMergedConditions(BOp->getOperand(1), TBB, FBB, TmpBB, SwitchBB, Opc,
01656                          Probs[0], Probs[1]);
01657   }
01658 }
01659 
01660 /// If the set of cases should be emitted as a series of branches, return true.
01661 /// If we should emit this as a bunch of and/or'd together conditions, return
01662 /// false.
01663 bool
01664 SelectionDAGBuilder::ShouldEmitAsBranches(const std::vector<CaseBlock> &Cases) {
01665   if (Cases.size() != 2) return true;
01666 
01667   // If this is two comparisons of the same values or'd or and'd together, they
01668   // will get folded into a single comparison, so don't emit two blocks.
01669   if ((Cases[0].CmpLHS == Cases[1].CmpLHS &&
01670        Cases[0].CmpRHS == Cases[1].CmpRHS) ||
01671       (Cases[0].CmpRHS == Cases[1].CmpLHS &&
01672        Cases[0].CmpLHS == Cases[1].CmpRHS)) {
01673     return false;
01674   }
01675 
01676   // Handle: (X != null) | (Y != null) --> (X|Y) != 0
01677   // Handle: (X == null) & (Y == null) --> (X|Y) == 0
01678   if (Cases[0].CmpRHS == Cases[1].CmpRHS &&
01679       Cases[0].CC == Cases[1].CC &&
01680       isa<Constant>(Cases[0].CmpRHS) &&
01681       cast<Constant>(Cases[0].CmpRHS)->isNullValue()) {
01682     if (Cases[0].CC == ISD::SETEQ && Cases[0].TrueBB == Cases[1].ThisBB)
01683       return false;
01684     if (Cases[0].CC == ISD::SETNE && Cases[0].FalseBB == Cases[1].ThisBB)
01685       return false;
01686   }
01687 
01688   return true;
01689 }
01690 
01691 void SelectionDAGBuilder::visitBr(const BranchInst &I) {
01692   MachineBasicBlock *BrMBB = FuncInfo.MBB;
01693 
01694   // Update machine-CFG edges.
01695   MachineBasicBlock *Succ0MBB = FuncInfo.MBBMap[I.getSuccessor(0)];
01696 
01697   if (I.isUnconditional()) {
01698     // Update machine-CFG edges.
01699     BrMBB->addSuccessor(Succ0MBB);
01700 
01701     // If this is not a fall-through branch or optimizations are switched off,
01702     // emit the branch.
01703     if (Succ0MBB != NextBlock(BrMBB) || TM.getOptLevel() == CodeGenOpt::None)
01704       DAG.setRoot(DAG.getNode(ISD::BR, getCurSDLoc(),
01705                               MVT::Other, getControlRoot(),
01706                               DAG.getBasicBlock(Succ0MBB)));
01707 
01708     return;
01709   }
01710 
01711   // If this condition is one of the special cases we handle, do special stuff
01712   // now.
01713   const Value *CondVal = I.getCondition();
01714   MachineBasicBlock *Succ1MBB = FuncInfo.MBBMap[I.getSuccessor(1)];
01715 
01716   // If this is a series of conditions that are or'd or and'd together, emit
01717   // this as a sequence of branches instead of setcc's with and/or operations.
01718   // As long as jumps are not expensive, this should improve performance.
01719   // For example, instead of something like:
01720   //     cmp A, B
01721   //     C = seteq
01722   //     cmp D, E
01723   //     F = setle
01724   //     or C, F
01725   //     jnz foo
01726   // Emit:
01727   //     cmp A, B
01728   //     je foo
01729   //     cmp D, E
01730   //     jle foo
01731   //
01732   if (const BinaryOperator *BOp = dyn_cast<BinaryOperator>(CondVal)) {
01733     Instruction::BinaryOps Opcode = BOp->getOpcode();
01734     if (!DAG.getTargetLoweringInfo().isJumpExpensive() && BOp->hasOneUse() &&
01735         !I.getMetadata(LLVMContext::MD_unpredictable) &&
01736         (Opcode == Instruction::And || Opcode == Instruction::Or)) {
01737       FindMergedConditions(BOp, Succ0MBB, Succ1MBB, BrMBB, BrMBB,
01738                            Opcode,
01739                            getEdgeProbability(BrMBB, Succ0MBB),
01740                            getEdgeProbability(BrMBB, Succ1MBB));
01741       // If the compares in later blocks need to use values not currently
01742       // exported from this block, export them now.  This block should always
01743       // be the first entry.
01744       assert(SwitchCases[0].ThisBB == BrMBB && "Unexpected lowering!");
01745 
01746       // Allow some cases to be rejected.
01747       if (ShouldEmitAsBranches(SwitchCases)) {
01748         for (unsigned i = 1, e = SwitchCases.size(); i != e; ++i) {
01749           ExportFromCurrentBlock(SwitchCases[i].CmpLHS);
01750           ExportFromCurrentBlock(SwitchCases[i].CmpRHS);
01751         }
01752 
01753         // Emit the branch for this block.
01754         visitSwitchCase(SwitchCases[0], BrMBB);
01755         SwitchCases.erase(SwitchCases.begin());
01756         return;
01757       }
01758 
01759       // Okay, we decided not to do this, remove any inserted MBB's and clear
01760       // SwitchCases.
01761       for (unsigned i = 1, e = SwitchCases.size(); i != e; ++i)
01762         FuncInfo.MF->erase(SwitchCases[i].ThisBB);
01763 
01764       SwitchCases.clear();
01765     }
01766   }
01767 
01768   // Create a CaseBlock record representing this branch.
01769   CaseBlock CB(ISD::SETEQ, CondVal, ConstantInt::getTrue(*DAG.getContext()),
01770                nullptr, Succ0MBB, Succ1MBB, BrMBB);
01771 
01772   // Use visitSwitchCase to actually insert the fast branch sequence for this
01773   // cond branch.
01774   visitSwitchCase(CB, BrMBB);
01775 }
01776 
01777 /// visitSwitchCase - Emits the necessary code to represent a single node in
01778 /// the binary search tree resulting from lowering a switch instruction.
01779 void SelectionDAGBuilder::visitSwitchCase(CaseBlock &CB,
01780                                           MachineBasicBlock *SwitchBB) {
01781   SDValue Cond;
01782   SDValue CondLHS = getValue(CB.CmpLHS);
01783   SDLoc dl = getCurSDLoc();
01784 
01785   // Build the setcc now.
01786   if (!CB.CmpMHS) {
01787     // Fold "(X == true)" to X and "(X == false)" to !X to
01788     // handle common cases produced by branch lowering.
01789     if (CB.CmpRHS == ConstantInt::getTrue(*DAG.getContext()) &&
01790         CB.CC == ISD::SETEQ)
01791       Cond = CondLHS;
01792     else if (CB.CmpRHS == ConstantInt::getFalse(*DAG.getContext()) &&
01793              CB.CC == ISD::SETEQ) {
01794       SDValue True = DAG.getConstant(1, dl, CondLHS.getValueType());
01795       Cond = DAG.getNode(ISD::XOR, dl, CondLHS.getValueType(), CondLHS, True);
01796     } else
01797       Cond = DAG.getSetCC(dl, MVT::i1, CondLHS, getValue(CB.CmpRHS), CB.CC);
01798   } else {
01799     assert(CB.CC == ISD::SETLE && "Can handle only LE ranges now");
01800 
01801     const APInt& Low = cast<ConstantInt>(CB.CmpLHS)->getValue();
01802     const APInt& High = cast<ConstantInt>(CB.CmpRHS)->getValue();
01803 
01804     SDValue CmpOp = getValue(CB.CmpMHS);
01805     EVT VT = CmpOp.getValueType();
01806 
01807     if (cast<ConstantInt>(CB.CmpLHS)->isMinValue(true)) {
01808       Cond = DAG.getSetCC(dl, MVT::i1, CmpOp, DAG.getConstant(High, dl, VT),
01809                           ISD::SETLE);
01810     } else {
01811       SDValue SUB = DAG.getNode(ISD::SUB, dl,
01812                                 VT, CmpOp, DAG.getConstant(Low, dl, VT));
01813       Cond = DAG.getSetCC(dl, MVT::i1, SUB,
01814                           DAG.getConstant(High-Low, dl, VT), ISD::SETULE);
01815     }
01816   }
01817 
01818   // Update successor info
01819   addSuccessorWithProb(SwitchBB, CB.TrueBB, CB.TrueProb);
01820   // TrueBB and FalseBB are always different unless the incoming IR is
01821   // degenerate. This only happens when running llc on weird IR.
01822   if (CB.TrueBB != CB.FalseBB)
01823     addSuccessorWithProb(SwitchBB, CB.FalseBB, CB.FalseProb);
01824   SwitchBB->normalizeSuccProbs();
01825 
01826   // If the lhs block is the next block, invert the condition so that we can
01827   // fall through to the lhs instead of the rhs block.
01828   if (CB.TrueBB == NextBlock(SwitchBB)) {
01829     std::swap(CB.TrueBB, CB.FalseBB);
01830     SDValue True = DAG.getConstant(1, dl, Cond.getValueType());
01831     Cond = DAG.getNode(ISD::XOR, dl, Cond.getValueType(), Cond, True);
01832   }
01833 
01834   SDValue BrCond = DAG.getNode(ISD::BRCOND, dl,
01835                                MVT::Other, getControlRoot(), Cond,
01836                                DAG.getBasicBlock(CB.TrueBB));
01837 
01838   // Insert the false branch. Do this even if it's a fall through branch,
01839   // this makes it easier to do DAG optimizations which require inverting
01840   // the branch condition.
01841   BrCond = DAG.getNode(ISD::BR, dl, MVT::Other, BrCond,
01842                        DAG.getBasicBlock(CB.FalseBB));
01843 
01844   DAG.setRoot(BrCond);
01845 }
01846 
01847 /// visitJumpTable - Emit JumpTable node in the current MBB
01848 void SelectionDAGBuilder::visitJumpTable(JumpTable &JT) {
01849   // Emit the code for the jump table
01850   assert(JT.Reg != -1U && "Should lower JT Header first!");
01851   EVT PTy = DAG.getTargetLoweringInfo().getPointerTy(DAG.getDataLayout());
01852   SDValue Index = DAG.getCopyFromReg(getControlRoot(), getCurSDLoc(),
01853                                      JT.Reg, PTy);
01854   SDValue Table = DAG.getJumpTable(JT.JTI, PTy);
01855   SDValue BrJumpTable = DAG.getNode(ISD::BR_JT, getCurSDLoc(),
01856                                     MVT::Other, Index.getValue(1),
01857                                     Table, Index);
01858   DAG.setRoot(BrJumpTable);
01859 }
01860 
01861 /// visitJumpTableHeader - This function emits necessary code to produce index
01862 /// in the JumpTable from switch case.
01863 void SelectionDAGBuilder::visitJumpTableHeader(JumpTable &JT,
01864                                                JumpTableHeader &JTH,
01865                                                MachineBasicBlock *SwitchBB) {
01866   SDLoc dl = getCurSDLoc();
01867 
01868   // Subtract the lowest switch case value from the value being switched on and
01869   // conditional branch to default mbb if the result is greater than the
01870   // difference between smallest and largest cases.
01871   SDValue SwitchOp = getValue(JTH.SValue);
01872   EVT VT = SwitchOp.getValueType();
01873   SDValue Sub = DAG.getNode(ISD::SUB, dl, VT, SwitchOp,
01874                             DAG.getConstant(JTH.First, dl, VT));
01875 
01876   // The SDNode we just created, which holds the value being switched on minus
01877   // the smallest case value, needs to be copied to a virtual register so it
01878   // can be used as an index into the jump table in a subsequent basic block.
01879   // This value may be smaller or larger than the target's pointer type, and
01880   // therefore require extension or truncating.
01881   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
01882   SwitchOp = DAG.getZExtOrTrunc(Sub, dl, TLI.getPointerTy(DAG.getDataLayout()));
01883 
01884   unsigned JumpTableReg =
01885       FuncInfo.CreateReg(TLI.getPointerTy(DAG.getDataLayout()));
01886   SDValue CopyTo = DAG.getCopyToReg(getControlRoot(), dl,
01887                                     JumpTableReg, SwitchOp);
01888   JT.Reg = JumpTableReg;
01889 
01890   // Emit the range check for the jump table, and branch to the default block
01891   // for the switch statement if the value being switched on exceeds the largest
01892   // case in the switch.
01893   SDValue CMP = DAG.getSetCC(
01894       dl, TLI.getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(),
01895                                  Sub.getValueType()),
01896       Sub, DAG.getConstant(JTH.Last - JTH.First, dl, VT), ISD::SETUGT);
01897 
01898   SDValue BrCond = DAG.getNode(ISD::BRCOND, dl,
01899                                MVT::Other, CopyTo, CMP,
01900                                DAG.getBasicBlock(JT.Default));
01901 
01902   // Avoid emitting unnecessary branches to the next block.
01903   if (JT.MBB != NextBlock(SwitchBB))
01904     BrCond = DAG.getNode(ISD::BR, dl, MVT::Other, BrCond,
01905                          DAG.getBasicBlock(JT.MBB));
01906 
01907   DAG.setRoot(BrCond);
01908 }
01909 
01910 /// Codegen a new tail for a stack protector check ParentMBB which has had its
01911 /// tail spliced into a stack protector check success bb.
01912 ///
01913 /// For a high level explanation of how this fits into the stack protector
01914 /// generation see the comment on the declaration of class
01915 /// StackProtectorDescriptor.
01916 void SelectionDAGBuilder::visitSPDescriptorParent(StackProtectorDescriptor &SPD,
01917                                                   MachineBasicBlock *ParentBB) {
01918 
01919   // First create the loads to the guard/stack slot for the comparison.
01920   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
01921   EVT PtrTy = TLI.getPointerTy(DAG.getDataLayout());
01922 
01923   MachineFrameInfo *MFI = ParentBB->getParent()->getFrameInfo();
01924   int FI = MFI->getStackProtectorIndex();
01925 
01926   const Value *IRGuard = SPD.getGuard();
01927   SDValue GuardPtr = getValue(IRGuard);
01928   SDValue StackSlotPtr = DAG.getFrameIndex(FI, PtrTy);
01929 
01930   unsigned Align = DL->getPrefTypeAlignment(IRGuard->getType());
01931 
01932   SDValue Guard;
01933   SDLoc dl = getCurSDLoc();
01934 
01935   // If GuardReg is set and useLoadStackGuardNode returns true, retrieve the
01936   // guard value from the virtual register holding the value. Otherwise, emit a
01937   // volatile load to retrieve the stack guard value.
01938   unsigned GuardReg = SPD.getGuardReg();
01939 
01940   if (GuardReg && TLI.useLoadStackGuardNode())
01941     Guard = DAG.getCopyFromReg(DAG.getEntryNode(), dl, GuardReg,
01942                                PtrTy);
01943   else
01944     Guard = DAG.getLoad(PtrTy, dl, DAG.getEntryNode(),
01945                         GuardPtr, MachinePointerInfo(IRGuard, 0),
01946                         true, false, false, Align);
01947 
01948   SDValue StackSlot = DAG.getLoad(
01949       PtrTy, dl, DAG.getEntryNode(), StackSlotPtr,
01950       MachinePointerInfo::getFixedStack(DAG.getMachineFunction(), FI), true,
01951       false, false, Align);
01952 
01953   // Perform the comparison via a subtract/getsetcc.
01954   EVT VT = Guard.getValueType();
01955   SDValue Sub = DAG.getNode(ISD::SUB, dl, VT, Guard, StackSlot);
01956 
01957   SDValue Cmp = DAG.getSetCC(dl, TLI.getSetCCResultType(DAG.getDataLayout(),
01958                                                         *DAG.getContext(),
01959                                                         Sub.getValueType()),
01960                              Sub, DAG.getConstant(0, dl, VT), ISD::SETNE);
01961 
01962   // If the sub is not 0, then we know the guard/stackslot do not equal, so
01963   // branch to failure MBB.
01964   SDValue BrCond = DAG.getNode(ISD::BRCOND, dl,
01965                                MVT::Other, StackSlot.getOperand(0),
01966                                Cmp, DAG.getBasicBlock(SPD.getFailureMBB()));
01967   // Otherwise branch to success MBB.
01968   SDValue Br = DAG.getNode(ISD::BR, dl,
01969                            MVT::Other, BrCond,
01970                            DAG.getBasicBlock(SPD.getSuccessMBB()));
01971 
01972   DAG.setRoot(Br);
01973 }
01974 
01975 /// Codegen the failure basic block for a stack protector check.
01976 ///
01977 /// A failure stack protector machine basic block consists simply of a call to
01978 /// __stack_chk_fail().
01979 ///
01980 /// For a high level explanation of how this fits into the stack protector
01981 /// generation see the comment on the declaration of class
01982 /// StackProtectorDescriptor.
01983 void
01984 SelectionDAGBuilder::visitSPDescriptorFailure(StackProtectorDescriptor &SPD) {
01985   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
01986   SDValue Chain =
01987       TLI.makeLibCall(DAG, RTLIB::STACKPROTECTOR_CHECK_FAIL, MVT::isVoid,
01988                       None, false, getCurSDLoc(), false, false).second;
01989   DAG.setRoot(Chain);
01990 }
01991 
01992 /// visitBitTestHeader - This function emits necessary code to produce value
01993 /// suitable for "bit tests"
01994 void SelectionDAGBuilder::visitBitTestHeader(BitTestBlock &B,
01995                                              MachineBasicBlock *SwitchBB) {
01996   SDLoc dl = getCurSDLoc();
01997 
01998   // Subtract the minimum value
01999   SDValue SwitchOp = getValue(B.SValue);
02000   EVT VT = SwitchOp.getValueType();
02001   SDValue Sub = DAG.getNode(ISD::SUB, dl, VT, SwitchOp,
02002                             DAG.getConstant(B.First, dl, VT));
02003 
02004   // Check range
02005   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
02006   SDValue RangeCmp = DAG.getSetCC(
02007       dl, TLI.getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(),
02008                                  Sub.getValueType()),
02009       Sub, DAG.getConstant(B.Range, dl, VT), ISD::SETUGT);
02010 
02011   // Determine the type of the test operands.
02012   bool UsePtrType = false;
02013   if (!TLI.isTypeLegal(VT))
02014     UsePtrType = true;
02015   else {
02016     for (unsigned i = 0, e = B.Cases.size(); i != e; ++i)
02017       if (!isUIntN(VT.getSizeInBits(), B.Cases[i].Mask)) {
02018         // Switch table case range are encoded into series of masks.
02019         // Just use pointer type, it's guaranteed to fit.
02020         UsePtrType = true;
02021         break;
02022       }
02023   }
02024   if (UsePtrType) {
02025     VT = TLI.getPointerTy(DAG.getDataLayout());
02026     Sub = DAG.getZExtOrTrunc(Sub, dl, VT);
02027   }
02028 
02029   B.RegVT = VT.getSimpleVT();
02030   B.Reg = FuncInfo.CreateReg(B.RegVT);
02031   SDValue CopyTo = DAG.getCopyToReg(getControlRoot(), dl, B.Reg, Sub);
02032 
02033   MachineBasicBlock* MBB = B.Cases[0].ThisBB;
02034 
02035   addSuccessorWithProb(SwitchBB, B.Default, B.DefaultProb);
02036   addSuccessorWithProb(SwitchBB, MBB, B.Prob);
02037   SwitchBB->normalizeSuccProbs();
02038 
02039   SDValue BrRange = DAG.getNode(ISD::BRCOND, dl,
02040                                 MVT::Other, CopyTo, RangeCmp,
02041                                 DAG.getBasicBlock(B.Default));
02042 
02043   // Avoid emitting unnecessary branches to the next block.
02044   if (MBB != NextBlock(SwitchBB))
02045     BrRange = DAG.getNode(ISD::BR, dl, MVT::Other, BrRange,
02046                           DAG.getBasicBlock(MBB));
02047 
02048   DAG.setRoot(BrRange);
02049 }
02050 
02051 /// visitBitTestCase - this function produces one "bit test"
02052 void SelectionDAGBuilder::visitBitTestCase(BitTestBlock &BB,
02053                                            MachineBasicBlock* NextMBB,
02054                                            BranchProbability BranchProbToNext,
02055                                            unsigned Reg,
02056                                            BitTestCase &B,
02057                                            MachineBasicBlock *SwitchBB) {
02058   SDLoc dl = getCurSDLoc();
02059   MVT VT = BB.RegVT;
02060   SDValue ShiftOp = DAG.getCopyFromReg(getControlRoot(), dl, Reg, VT);
02061   SDValue Cmp;
02062   unsigned PopCount = countPopulation(B.Mask);
02063   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
02064   if (PopCount == 1) {
02065     // Testing for a single bit; just compare the shift count with what it
02066     // would need to be to shift a 1 bit in that position.
02067     Cmp = DAG.getSetCC(
02068         dl, TLI.getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), VT),
02069         ShiftOp, DAG.getConstant(countTrailingZeros(B.Mask), dl, VT),
02070         ISD::SETEQ);
02071   } else if (PopCount == BB.Range) {
02072     // There is only one zero bit in the range, test for it directly.
02073     Cmp = DAG.getSetCC(
02074         dl, TLI.getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), VT),
02075         ShiftOp, DAG.getConstant(countTrailingOnes(B.Mask), dl, VT),
02076         ISD::SETNE);
02077   } else {
02078     // Make desired shift
02079     SDValue SwitchVal = DAG.getNode(ISD::SHL, dl, VT,
02080                                     DAG.getConstant(1, dl, VT), ShiftOp);
02081 
02082     // Emit bit tests and jumps
02083     SDValue AndOp = DAG.getNode(ISD::AND, dl,
02084                                 VT, SwitchVal, DAG.getConstant(B.Mask, dl, VT));
02085     Cmp = DAG.getSetCC(
02086         dl, TLI.getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), VT),
02087         AndOp, DAG.getConstant(0, dl, VT), ISD::SETNE);
02088   }
02089 
02090   // The branch probability from SwitchBB to B.TargetBB is B.ExtraProb.
02091   addSuccessorWithProb(SwitchBB, B.TargetBB, B.ExtraProb);
02092   // The branch probability from SwitchBB to NextMBB is BranchProbToNext.
02093   addSuccessorWithProb(SwitchBB, NextMBB, BranchProbToNext);
02094   // It is not guaranteed that the sum of B.ExtraProb and BranchProbToNext is
02095   // one as they are relative probabilities (and thus work more like weights),
02096   // and hence we need to normalize them to let the sum of them become one.
02097   SwitchBB->normalizeSuccProbs();
02098 
02099   SDValue BrAnd = DAG.getNode(ISD::BRCOND, dl,
02100                               MVT::Other, getControlRoot(),
02101                               Cmp, DAG.getBasicBlock(B.TargetBB));
02102 
02103   // Avoid emitting unnecessary branches to the next block.
02104   if (NextMBB != NextBlock(SwitchBB))
02105     BrAnd = DAG.getNode(ISD::BR, dl, MVT::Other, BrAnd,
02106                         DAG.getBasicBlock(NextMBB));
02107 
02108   DAG.setRoot(BrAnd);
02109 }
02110 
02111 void SelectionDAGBuilder::visitInvoke(const InvokeInst &I) {
02112   MachineBasicBlock *InvokeMBB = FuncInfo.MBB;
02113 
02114   // Retrieve successors. Look through artificial IR level blocks like
02115   // catchswitch for successors.
02116   MachineBasicBlock *Return = FuncInfo.MBBMap[I.getSuccessor(0)];
02117   const BasicBlock *EHPadBB = I.getSuccessor(1);
02118 
02119   const Value *Callee(I.getCalledValue());
02120   const Function *Fn = dyn_cast<Function>(Callee);
02121   if (isa<InlineAsm>(Callee))
02122     visitInlineAsm(&I);
02123   else if (Fn && Fn->isIntrinsic()) {
02124     switch (Fn->getIntrinsicID()) {
02125     default:
02126       llvm_unreachable("Cannot invoke this intrinsic");
02127     case Intrinsic::donothing:
02128       // Ignore invokes to @llvm.donothing: jump directly to the next BB.
02129       break;
02130     case Intrinsic::experimental_patchpoint_void:
02131     case Intrinsic::experimental_patchpoint_i64:
02132       visitPatchpoint(&I, EHPadBB);
02133       break;
02134     case Intrinsic::experimental_gc_statepoint:
02135       LowerStatepoint(ImmutableStatepoint(&I), EHPadBB);
02136       break;
02137     }
02138   } else
02139     LowerCallTo(&I, getValue(Callee), false, EHPadBB);
02140 
02141   // If the value of the invoke is used outside of its defining block, make it
02142   // available as a virtual register.
02143   // We already took care of the exported value for the statepoint instruction
02144   // during call to the LowerStatepoint.
02145   if (!isStatepoint(I)) {
02146     CopyToExportRegsIfNeeded(&I);
02147   }
02148 
02149   SmallVector<std::pair<MachineBasicBlock *, BranchProbability>, 1> UnwindDests;
02150   BranchProbabilityInfo *BPI = FuncInfo.BPI;
02151   BranchProbability EHPadBBProb =
02152       BPI ? BPI->getEdgeProbability(InvokeMBB->getBasicBlock(), EHPadBB)
02153           : BranchProbability::getZero();
02154   findUnwindDestinations(FuncInfo, EHPadBB, EHPadBBProb, UnwindDests);
02155 
02156   // Update successor info.
02157   addSuccessorWithProb(InvokeMBB, Return);
02158   for (auto &UnwindDest : UnwindDests) {
02159     UnwindDest.first->setIsEHPad();
02160     addSuccessorWithProb(InvokeMBB, UnwindDest.first, UnwindDest.second);
02161   }
02162   InvokeMBB->normalizeSuccProbs();
02163 
02164   // Drop into normal successor.
02165   DAG.setRoot(DAG.getNode(ISD::BR, getCurSDLoc(),
02166                           MVT::Other, getControlRoot(),
02167                           DAG.getBasicBlock(Return)));
02168 }
02169 
02170 void SelectionDAGBuilder::visitResume(const ResumeInst &RI) {
02171   llvm_unreachable("SelectionDAGBuilder shouldn't visit resume instructions!");
02172 }
02173 
02174 void SelectionDAGBuilder::visitLandingPad(const LandingPadInst &LP) {
02175   assert(FuncInfo.MBB->isEHPad() &&
02176          "Call to landingpad not in landing pad!");
02177 
02178   MachineBasicBlock *MBB = FuncInfo.MBB;
02179   MachineModuleInfo &MMI = DAG.getMachineFunction().getMMI();
02180   AddLandingPadInfo(LP, MMI, MBB);
02181 
02182   // If there aren't registers to copy the values into (e.g., during SjLj
02183   // exceptions), then don't bother to create these DAG nodes.
02184   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
02185   const Constant *PersonalityFn = FuncInfo.Fn->getPersonalityFn();
02186   if (TLI.getExceptionPointerRegister(PersonalityFn) == 0 &&
02187       TLI.getExceptionSelectorRegister(PersonalityFn) == 0)
02188     return;
02189 
02190   // If landingpad's return type is token type, we don't create DAG nodes
02191   // for its exception pointer and selector value. The extraction of exception
02192   // pointer or selector value from token type landingpads is not currently
02193   // supported.
02194   if (LP.getType()->isTokenTy())
02195     return;
02196 
02197   SmallVector<EVT, 2> ValueVTs;
02198   SDLoc dl = getCurSDLoc();
02199   ComputeValueVTs(TLI, DAG.getDataLayout(), LP.getType(), ValueVTs);
02200   assert(ValueVTs.size() == 2 && "Only two-valued landingpads are supported");
02201 
02202   // Get the two live-in registers as SDValues. The physregs have already been
02203   // copied into virtual registers.
02204   SDValue Ops[2];
02205   if (FuncInfo.ExceptionPointerVirtReg) {
02206     Ops[0] = DAG.getZExtOrTrunc(
02207         DAG.getCopyFromReg(DAG.getEntryNode(), dl,
02208                            FuncInfo.ExceptionPointerVirtReg,
02209                            TLI.getPointerTy(DAG.getDataLayout())),
02210         dl, ValueVTs[0]);
02211   } else {
02212     Ops[0] = DAG.getConstant(0, dl, TLI.getPointerTy(DAG.getDataLayout()));
02213   }
02214   Ops[1] = DAG.getZExtOrTrunc(
02215       DAG.getCopyFromReg(DAG.getEntryNode(), dl,
02216                          FuncInfo.ExceptionSelectorVirtReg,
02217                          TLI.getPointerTy(DAG.getDataLayout())),
02218       dl, ValueVTs[1]);
02219 
02220   // Merge into one.
02221   SDValue Res = DAG.getNode(ISD::MERGE_VALUES, dl,
02222                             DAG.getVTList(ValueVTs), Ops);
02223   setValue(&LP, Res);
02224 }
02225 
02226 void SelectionDAGBuilder::sortAndRangeify(CaseClusterVector &Clusters) {
02227 #ifndef NDEBUG
02228   for (const CaseCluster &CC : Clusters)
02229     assert(CC.Low == CC.High && "Input clusters must be single-case");
02230 #endif
02231 
02232   std::sort(Clusters.begin(), Clusters.end(),
02233             [](const CaseCluster &a, const CaseCluster &b) {
02234     return a.Low->getValue().slt(b.Low->getValue());
02235   });
02236 
02237   // Merge adjacent clusters with the same destination.
02238   const unsigned N = Clusters.size();
02239   unsigned DstIndex = 0;
02240   for (unsigned SrcIndex = 0; SrcIndex < N; ++SrcIndex) {
02241     CaseCluster &CC = Clusters[SrcIndex];
02242     const ConstantInt *CaseVal = CC.Low;
02243     MachineBasicBlock *Succ = CC.MBB;
02244 
02245     if (DstIndex != 0 && Clusters[DstIndex - 1].MBB == Succ &&
02246         (CaseVal->getValue() - Clusters[DstIndex - 1].High->getValue()) == 1) {
02247       // If this case has the same successor and is a neighbour, merge it into
02248       // the previous cluster.
02249       Clusters[DstIndex - 1].High = CaseVal;
02250       Clusters[DstIndex - 1].Prob += CC.Prob;
02251     } else {
02252       std::memmove(&Clusters[DstIndex++], &Clusters[SrcIndex],
02253                    sizeof(Clusters[SrcIndex]));
02254     }
02255   }
02256   Clusters.resize(DstIndex);
02257 }
02258 
02259 void SelectionDAGBuilder::UpdateSplitBlock(MachineBasicBlock *First,
02260                                            MachineBasicBlock *Last) {
02261   // Update JTCases.
02262   for (unsigned i = 0, e = JTCases.size(); i != e; ++i)
02263     if (JTCases[i].first.HeaderBB == First)
02264       JTCases[i].first.HeaderBB = Last;
02265 
02266   // Update BitTestCases.
02267   for (unsigned i = 0, e = BitTestCases.size(); i != e; ++i)
02268     if (BitTestCases[i].Parent == First)
02269       BitTestCases[i].Parent = Last;
02270 }
02271 
02272 void SelectionDAGBuilder::visitIndirectBr(const IndirectBrInst &I) {
02273   MachineBasicBlock *IndirectBrMBB = FuncInfo.MBB;
02274 
02275   // Update machine-CFG edges with unique successors.
02276   SmallSet<BasicBlock*, 32> Done;
02277   for (unsigned i = 0, e = I.getNumSuccessors(); i != e; ++i) {
02278     BasicBlock *BB = I.getSuccessor(i);
02279     bool Inserted = Done.insert(BB).second;
02280     if (!Inserted)
02281         continue;
02282 
02283     MachineBasicBlock *Succ = FuncInfo.MBBMap[BB];
02284     addSuccessorWithProb(IndirectBrMBB, Succ);
02285   }
02286   IndirectBrMBB->normalizeSuccProbs();
02287 
02288   DAG.setRoot(DAG.getNode(ISD::BRIND, getCurSDLoc(),
02289                           MVT::Other, getControlRoot(),
02290                           getValue(I.getAddress())));
02291 }
02292 
02293 void SelectionDAGBuilder::visitUnreachable(const UnreachableInst &I) {
02294   if (DAG.getTarget().Options.TrapUnreachable)
02295     DAG.setRoot(
02296         DAG.getNode(ISD::TRAP, getCurSDLoc(), MVT::Other, DAG.getRoot()));
02297 }
02298 
02299 void SelectionDAGBuilder::visitFSub(const User &I) {
02300   // -0.0 - X --> fneg
02301   Type *Ty = I.getType();
02302   if (isa<Constant>(I.getOperand(0)) &&
02303       I.getOperand(0) == ConstantFP::getZeroValueForNegation(Ty)) {
02304     SDValue Op2 = getValue(I.getOperand(1));
02305     setValue(&I, DAG.getNode(ISD::FNEG, getCurSDLoc(),
02306                              Op2.getValueType(), Op2));
02307     return;
02308   }
02309 
02310   visitBinary(I, ISD::FSUB);
02311 }
02312 
02313 void SelectionDAGBuilder::visitBinary(const User &I, unsigned OpCode) {
02314   SDValue Op1 = getValue(I.getOperand(0));
02315   SDValue Op2 = getValue(I.getOperand(1));
02316 
02317   bool nuw = false;
02318   bool nsw = false;
02319   bool exact = false;
02320   FastMathFlags FMF;
02321 
02322   if (const OverflowingBinaryOperator *OFBinOp =
02323           dyn_cast<const OverflowingBinaryOperator>(&I)) {
02324     nuw = OFBinOp->hasNoUnsignedWrap();
02325     nsw = OFBinOp->hasNoSignedWrap();
02326   }
02327   if (const PossiblyExactOperator *ExactOp =
02328           dyn_cast<const PossiblyExactOperator>(&I))
02329     exact = ExactOp->isExact();
02330   if (const FPMathOperator *FPOp = dyn_cast<const FPMathOperator>(&I))
02331     FMF = FPOp->getFastMathFlags();
02332 
02333   SDNodeFlags Flags;
02334   Flags.setExact(exact);
02335   Flags.setNoSignedWrap(nsw);
02336   Flags.setNoUnsignedWrap(nuw);
02337   if (EnableFMFInDAG) {
02338     Flags.setAllowReciprocal(FMF.allowReciprocal());
02339     Flags.setNoInfs(FMF.noInfs());
02340     Flags.setNoNaNs(FMF.noNaNs());
02341     Flags.setNoSignedZeros(FMF.noSignedZeros());
02342     Flags.setUnsafeAlgebra(FMF.unsafeAlgebra());
02343   }
02344   SDValue BinNodeValue = DAG.getNode(OpCode, getCurSDLoc(), Op1.getValueType(),
02345                                      Op1, Op2, &Flags);
02346   setValue(&I, BinNodeValue);
02347 }
02348 
02349 void SelectionDAGBuilder::visitShift(const User &I, unsigned Opcode) {
02350   SDValue Op1 = getValue(I.getOperand(0));
02351   SDValue Op2 = getValue(I.getOperand(1));
02352 
02353   EVT ShiftTy = DAG.getTargetLoweringInfo().getShiftAmountTy(
02354       Op2.getValueType(), DAG.getDataLayout());
02355 
02356   // Coerce the shift amount to the right type if we can.
02357   if (!I.getType()->isVectorTy() && Op2.getValueType() != ShiftTy) {
02358     unsigned ShiftSize = ShiftTy.getSizeInBits();
02359     unsigned Op2Size = Op2.getValueType().getSizeInBits();
02360     SDLoc DL = getCurSDLoc();
02361 
02362     // If the operand is smaller than the shift count type, promote it.
02363     if (ShiftSize > Op2Size)
02364       Op2 = DAG.getNode(ISD::ZERO_EXTEND, DL, ShiftTy, Op2);
02365 
02366     // If the operand is larger than the shift count type but the shift
02367     // count type has enough bits to represent any shift value, truncate
02368     // it now. This is a common case and it exposes the truncate to
02369     // optimization early.
02370     else if (ShiftSize >= Log2_32_Ceil(Op2.getValueType().getSizeInBits()))
02371       Op2 = DAG.getNode(ISD::TRUNCATE, DL, ShiftTy, Op2);
02372     // Otherwise we'll need to temporarily settle for some other convenient
02373     // type.  Type legalization will make adjustments once the shiftee is split.
02374     else
02375       Op2 = DAG.getZExtOrTrunc(Op2, DL, MVT::i32);
02376   }
02377 
02378   bool nuw = false;
02379   bool nsw = false;
02380   bool exact = false;
02381 
02382   if (Opcode == ISD::SRL || Opcode == ISD::SRA || Opcode == ISD::SHL) {
02383 
02384     if (const OverflowingBinaryOperator *OFBinOp =
02385             dyn_cast<const OverflowingBinaryOperator>(&I)) {
02386       nuw = OFBinOp->hasNoUnsignedWrap();
02387       nsw = OFBinOp->hasNoSignedWrap();
02388     }
02389     if (const PossiblyExactOperator *ExactOp =
02390             dyn_cast<const PossiblyExactOperator>(&I))
02391       exact = ExactOp->isExact();
02392   }
02393   SDNodeFlags Flags;
02394   Flags.setExact(exact);
02395   Flags.setNoSignedWrap(nsw);
02396   Flags.setNoUnsignedWrap(nuw);
02397   SDValue Res = DAG.getNode(Opcode, getCurSDLoc(), Op1.getValueType(), Op1, Op2,
02398                             &Flags);
02399   setValue(&I, Res);
02400 }
02401 
02402 void SelectionDAGBuilder::visitSDiv(const User &I) {
02403   SDValue Op1 = getValue(I.getOperand(0));
02404   SDValue Op2 = getValue(I.getOperand(1));
02405 
02406   SDNodeFlags Flags;
02407   Flags.setExact(isa<PossiblyExactOperator>(&I) &&
02408                  cast<PossiblyExactOperator>(&I)->isExact());
02409   setValue(&I, DAG.getNode(ISD::SDIV, getCurSDLoc(), Op1.getValueType(), Op1,
02410                            Op2, &Flags));
02411 }
02412 
02413 void SelectionDAGBuilder::visitICmp(const User &I) {
02414   ICmpInst::Predicate predicate = ICmpInst::BAD_ICMP_PREDICATE;
02415   if (const ICmpInst *IC = dyn_cast<ICmpInst>(&I))
02416     predicate = IC->getPredicate();
02417   else if (const ConstantExpr *IC = dyn_cast<ConstantExpr>(&I))
02418     predicate = ICmpInst::Predicate(IC->getPredicate());
02419   SDValue Op1 = getValue(I.getOperand(0));
02420   SDValue Op2 = getValue(I.getOperand(1));
02421   ISD::CondCode Opcode = getICmpCondCode(predicate);
02422 
02423   EVT DestVT = DAG.getTargetLoweringInfo().getValueType(DAG.getDataLayout(),
02424                                                         I.getType());
02425   setValue(&I, DAG.getSetCC(getCurSDLoc(), DestVT, Op1, Op2, Opcode));
02426 }
02427 
02428 void SelectionDAGBuilder::visitFCmp(const User &I) {
02429   FCmpInst::Predicate predicate = FCmpInst::BAD_FCMP_PREDICATE;
02430   if (const FCmpInst *FC = dyn_cast<FCmpInst>(&I))
02431     predicate = FC->getPredicate();
02432   else if (const ConstantExpr *FC = dyn_cast<ConstantExpr>(&I))
02433     predicate = FCmpInst::Predicate(FC->getPredicate());
02434   SDValue Op1 = getValue(I.getOperand(0));
02435   SDValue Op2 = getValue(I.getOperand(1));
02436   ISD::CondCode Condition = getFCmpCondCode(predicate);
02437 
02438   // FIXME: Fcmp instructions have fast-math-flags in IR, so we should use them.
02439   // FIXME: We should propagate the fast-math-flags to the DAG node itself for
02440   // further optimization, but currently FMF is only applicable to binary nodes.
02441   if (TM.Options.NoNaNsFPMath)
02442     Condition = getFCmpCodeWithoutNaN(Condition);
02443   EVT DestVT = DAG.getTargetLoweringInfo().getValueType(DAG.getDataLayout(),
02444                                                         I.getType());
02445   setValue(&I, DAG.getSetCC(getCurSDLoc(), DestVT, Op1, Op2, Condition));
02446 }
02447 
02448 void SelectionDAGBuilder::visitSelect(const User &I) {
02449   SmallVector<EVT, 4> ValueVTs;
02450   ComputeValueVTs(DAG.getTargetLoweringInfo(), DAG.getDataLayout(), I.getType(),
02451                   ValueVTs);
02452   unsigned NumValues = ValueVTs.size();
02453   if (NumValues == 0) return;
02454 
02455   SmallVector<SDValue, 4> Values(NumValues);
02456   SDValue Cond     = getValue(I.getOperand(0));
02457   SDValue LHSVal   = getValue(I.getOperand(1));
02458   SDValue RHSVal   = getValue(I.getOperand(2));
02459   auto BaseOps = {Cond};
02460   ISD::NodeType OpCode = Cond.getValueType().isVector() ?
02461     ISD::VSELECT : ISD::SELECT;
02462 
02463   // Min/max matching is only viable if all output VTs are the same.
02464   if (std::equal(ValueVTs.begin(), ValueVTs.end(), ValueVTs.begin())) {
02465     EVT VT = ValueVTs[0];
02466     LLVMContext &Ctx = *DAG.getContext();
02467     auto &TLI = DAG.getTargetLoweringInfo();
02468 
02469     // We care about the legality of the operation after it has been type
02470     // legalized.
02471     while (TLI.getTypeAction(Ctx, VT) != TargetLoweringBase::TypeLegal &&
02472            VT != TLI.getTypeToTransformTo(Ctx, VT))
02473       VT = TLI.getTypeToTransformTo(Ctx, VT);
02474 
02475     // If the vselect is legal, assume we want to leave this as a vector setcc +
02476     // vselect. Otherwise, if this is going to be scalarized, we want to see if
02477     // min/max is legal on the scalar type.
02478     bool UseScalarMinMax = VT.isVector() &&
02479       !TLI.isOperationLegalOrCustom(ISD::VSELECT, VT);
02480 
02481     Value *LHS, *RHS;
02482     auto SPR = matchSelectPattern(const_cast<User*>(&I), LHS, RHS);
02483     ISD::NodeType Opc = ISD::DELETED_NODE;
02484     switch (SPR.Flavor) {
02485     case SPF_UMAX:    Opc = ISD::UMAX; break;
02486     case SPF_UMIN:    Opc = ISD::UMIN; break;
02487     case SPF_SMAX:    Opc = ISD::SMAX; break;
02488     case SPF_SMIN:    Opc = ISD::SMIN; break;
02489     case SPF_FMINNUM:
02490       switch (SPR.NaNBehavior) {
02491       case SPNB_NA: llvm_unreachable("No NaN behavior for FP op?");
02492       case SPNB_RETURNS_NAN:   Opc = ISD::FMINNAN; break;
02493       case SPNB_RETURNS_OTHER: Opc = ISD::FMINNUM; break;
02494       case SPNB_RETURNS_ANY: {
02495         if (TLI.isOperationLegalOrCustom(ISD::FMINNUM, VT))
02496           Opc = ISD::FMINNUM;
02497         else if (TLI.isOperationLegalOrCustom(ISD::FMINNAN, VT))
02498           Opc = ISD::FMINNAN;
02499         else if (UseScalarMinMax)
02500           Opc = TLI.isOperationLegalOrCustom(ISD::FMINNUM, VT.getScalarType()) ?
02501             ISD::FMINNUM : ISD::FMINNAN;
02502         break;
02503       }
02504       }
02505       break;
02506     case SPF_FMAXNUM:
02507       switch (SPR.NaNBehavior) {
02508       case SPNB_NA: llvm_unreachable("No NaN behavior for FP op?");
02509       case SPNB_RETURNS_NAN:   Opc = ISD::FMAXNAN; break;
02510       case SPNB_RETURNS_OTHER: Opc = ISD::FMAXNUM; break;
02511       case SPNB_RETURNS_ANY:
02512 
02513         if (TLI.isOperationLegalOrCustom(ISD::FMAXNUM, VT))
02514           Opc = ISD::FMAXNUM;
02515         else if (TLI.isOperationLegalOrCustom(ISD::FMAXNAN, VT))
02516           Opc = ISD::FMAXNAN;
02517         else if (UseScalarMinMax)
02518           Opc = TLI.isOperationLegalOrCustom(ISD::FMAXNUM, VT.getScalarType()) ?
02519             ISD::FMAXNUM : ISD::FMAXNAN;
02520         break;
02521       }
02522       break;
02523     default: break;
02524     }
02525 
02526     if (Opc != ISD::DELETED_NODE &&
02527         (TLI.isOperationLegalOrCustom(Opc, VT) ||
02528          (UseScalarMinMax &&
02529           TLI.isOperationLegalOrCustom(Opc, VT.getScalarType()))) &&
02530         // If the underlying comparison instruction is used by any other
02531         // instruction, the consumed instructions won't be destroyed, so it is
02532         // not profitable to convert to a min/max.
02533         cast<SelectInst>(&I)->getCondition()->hasOneUse()) {
02534       OpCode = Opc;
02535       LHSVal = getValue(LHS);
02536       RHSVal = getValue(RHS);
02537       BaseOps = {};
02538     }
02539   }
02540 
02541   for (unsigned i = 0; i != NumValues; ++i) {
02542     SmallVector<SDValue, 3> Ops(BaseOps.begin(), BaseOps.end());
02543     Ops.push_back(SDValue(LHSVal.getNode(), LHSVal.getResNo() + i));
02544     Ops.push_back(SDValue(RHSVal.getNode(), RHSVal.getResNo() + i));
02545     Values[i] = DAG.getNode(OpCode, getCurSDLoc(),
02546                             LHSVal.getNode()->getValueType(LHSVal.getResNo()+i),
02547                             Ops);
02548   }
02549 
02550   setValue(&I, DAG.getNode(ISD::MERGE_VALUES, getCurSDLoc(),
02551                            DAG.getVTList(ValueVTs), Values));
02552 }
02553 
02554 void SelectionDAGBuilder::visitTrunc(const User &I) {
02555   // TruncInst cannot be a no-op cast because sizeof(src) > sizeof(dest).
02556   SDValue N = getValue(I.getOperand(0));
02557   EVT DestVT = DAG.getTargetLoweringInfo().getValueType(DAG.getDataLayout(),
02558                                                         I.getType());
02559   setValue(&I, DAG.getNode(ISD::TRUNCATE, getCurSDLoc(), DestVT, N));
02560 }
02561 
02562 void SelectionDAGBuilder::visitZExt(const User &I) {
02563   // ZExt cannot be a no-op cast because sizeof(src) < sizeof(dest).
02564   // ZExt also can't be a cast to bool for same reason. So, nothing much to do
02565   SDValue N = getValue(I.getOperand(0));
02566   EVT DestVT = DAG.getTargetLoweringInfo().getValueType(DAG.getDataLayout(),
02567                                                         I.getType());
02568   setValue(&I, DAG.getNode(ISD::ZERO_EXTEND, getCurSDLoc(), DestVT, N));
02569 }
02570 
02571 void SelectionDAGBuilder::visitSExt(const User &I) {
02572   // SExt cannot be a no-op cast because sizeof(src) < sizeof(dest).
02573   // SExt also can't be a cast to bool for same reason. So, nothing much to do
02574   SDValue N = getValue(I.getOperand(0));
02575   EVT DestVT = DAG.getTargetLoweringInfo().getValueType(DAG.getDataLayout(),
02576                                                         I.getType());
02577   setValue(&I, DAG.getNode(ISD::SIGN_EXTEND, getCurSDLoc(), DestVT, N));
02578 }
02579 
02580 void SelectionDAGBuilder::visitFPTrunc(const User &I) {
02581   // FPTrunc is never a no-op cast, no need to check
02582   SDValue N = getValue(I.getOperand(0));
02583   SDLoc dl = getCurSDLoc();
02584   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
02585   EVT DestVT = TLI.getValueType(DAG.getDataLayout(), I.getType());
02586   setValue(&I, DAG.getNode(ISD::FP_ROUND, dl, DestVT, N,
02587                            DAG.getTargetConstant(
02588                                0, dl, TLI.getPointerTy(DAG.getDataLayout()))));
02589 }
02590 
02591 void SelectionDAGBuilder::visitFPExt(const User &I) {
02592   // FPExt is never a no-op cast, no need to check
02593   SDValue N = getValue(I.getOperand(0));
02594   EVT DestVT = DAG.getTargetLoweringInfo().getValueType(DAG.getDataLayout(),
02595                                                         I.getType());
02596   setValue(&I, DAG.getNode(ISD::FP_EXTEND, getCurSDLoc(), DestVT, N));
02597 }
02598 
02599 void SelectionDAGBuilder::visitFPToUI(const User &I) {
02600   // FPToUI is never a no-op cast, no need to check
02601   SDValue N = getValue(I.getOperand(0));
02602   EVT DestVT = DAG.getTargetLoweringInfo().getValueType(DAG.getDataLayout(),
02603                                                         I.getType());
02604   setValue(&I, DAG.getNode(ISD::FP_TO_UINT, getCurSDLoc(), DestVT, N));
02605 }
02606 
02607 void SelectionDAGBuilder::visitFPToSI(const User &I) {
02608   // FPToSI is never a no-op cast, no need to check
02609   SDValue N = getValue(I.getOperand(0));
02610   EVT DestVT = DAG.getTargetLoweringInfo().getValueType(DAG.getDataLayout(),
02611                                                         I.getType());
02612   setValue(&I, DAG.getNode(ISD::FP_TO_SINT, getCurSDLoc(), DestVT, N));
02613 }
02614 
02615 void SelectionDAGBuilder::visitUIToFP(const User &I) {
02616   // UIToFP is never a no-op cast, no need to check
02617   SDValue N = getValue(I.getOperand(0));
02618   EVT DestVT = DAG.getTargetLoweringInfo().getValueType(DAG.getDataLayout(),
02619                                                         I.getType());
02620   setValue(&I, DAG.getNode(ISD::UINT_TO_FP, getCurSDLoc(), DestVT, N));
02621 }
02622 
02623 void SelectionDAGBuilder::visitSIToFP(const User &I) {
02624   // SIToFP is never a no-op cast, no need to check
02625   SDValue N = getValue(I.getOperand(0));
02626   EVT DestVT = DAG.getTargetLoweringInfo().getValueType(DAG.getDataLayout(),
02627                                                         I.getType());
02628   setValue(&I, DAG.getNode(ISD::SINT_TO_FP, getCurSDLoc(), DestVT, N));
02629 }
02630 
02631 void SelectionDAGBuilder::visitPtrToInt(const User &I) {
02632   // What to do depends on the size of the integer and the size of the pointer.
02633   // We can either truncate, zero extend, or no-op, accordingly.
02634   SDValue N = getValue(I.getOperand(0));
02635   EVT DestVT = DAG.getTargetLoweringInfo().getValueType(DAG.getDataLayout(),
02636                                                         I.getType());
02637   setValue(&I, DAG.getZExtOrTrunc(N, getCurSDLoc(), DestVT));
02638 }
02639 
02640 void SelectionDAGBuilder::visitIntToPtr(const User &I) {
02641   // What to do depends on the size of the integer and the size of the pointer.
02642   // We can either truncate, zero extend, or no-op, accordingly.
02643   SDValue N = getValue(I.getOperand(0));
02644   EVT DestVT = DAG.getTargetLoweringInfo().getValueType(DAG.getDataLayout(),
02645                                                         I.getType());
02646   setValue(&I, DAG.getZExtOrTrunc(N, getCurSDLoc(), DestVT));
02647 }
02648 
02649 void SelectionDAGBuilder::visitBitCast(const User &I) {
02650   SDValue N = getValue(I.getOperand(0));
02651   SDLoc dl = getCurSDLoc();
02652   EVT DestVT = DAG.getTargetLoweringInfo().getValueType(DAG.getDataLayout(),
02653                                                         I.getType());
02654 
02655   // BitCast assures us that source and destination are the same size so this is
02656   // either a BITCAST or a no-op.
02657   if (DestVT != N.getValueType())
02658     setValue(&I, DAG.getNode(ISD::BITCAST, dl,
02659                              DestVT, N)); // convert types.
02660   // Check if the original LLVM IR Operand was a ConstantInt, because getValue()
02661   // might fold any kind of constant expression to an integer constant and that
02662   // is not what we are looking for. Only regcognize a bitcast of a genuine
02663   // constant integer as an opaque constant.
02664   else if(ConstantInt *C = dyn_cast<ConstantInt>(I.getOperand(0)))
02665     setValue(&I, DAG.getConstant(C->getValue(), dl, DestVT, /*isTarget=*/false,
02666                                  /*isOpaque*/true));
02667   else
02668     setValue(&I, N);            // noop cast.
02669 }
02670 
02671 void SelectionDAGBuilder::visitAddrSpaceCast(const User &I) {
02672   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
02673   const Value *SV = I.getOperand(0);
02674   SDValue N = getValue(SV);
02675   EVT DestVT = TLI.getValueType(DAG.getDataLayout(), I.getType());
02676 
02677   unsigned SrcAS = SV->getType()->getPointerAddressSpace();
02678   unsigned DestAS = I.getType()->getPointerAddressSpace();
02679 
02680   if (!TLI.isNoopAddrSpaceCast(SrcAS, DestAS))
02681     N = DAG.getAddrSpaceCast(getCurSDLoc(), DestVT, N, SrcAS, DestAS);
02682 
02683   setValue(&I, N);
02684 }
02685 
02686 void SelectionDAGBuilder::visitInsertElement(const User &I) {
02687   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
02688   SDValue InVec = getValue(I.getOperand(0));
02689   SDValue InVal = getValue(I.getOperand(1));
02690   SDValue InIdx = DAG.getSExtOrTrunc(getValue(I.getOperand(2)), getCurSDLoc(),
02691                                      TLI.getVectorIdxTy(DAG.getDataLayout()));
02692   setValue(&I, DAG.getNode(ISD::INSERT_VECTOR_ELT, getCurSDLoc(),
02693                            TLI.getValueType(DAG.getDataLayout(), I.getType()),
02694                            InVec, InVal, InIdx));
02695 }
02696 
02697 void SelectionDAGBuilder::visitExtractElement(const User &I) {
02698   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
02699   SDValue InVec = getValue(I.getOperand(0));
02700   SDValue InIdx = DAG.getSExtOrTrunc(getValue(I.getOperand(1)), getCurSDLoc(),
02701                                      TLI.getVectorIdxTy(DAG.getDataLayout()));
02702   setValue(&I, DAG.getNode(ISD::EXTRACT_VECTOR_ELT, getCurSDLoc(),
02703                            TLI.getValueType(DAG.getDataLayout(), I.getType()),
02704                            InVec, InIdx));
02705 }
02706 
02707 // Utility for visitShuffleVector - Return true if every element in Mask,
02708 // beginning from position Pos and ending in Pos+Size, falls within the
02709 // specified sequential range [L, L+Pos). or is undef.
02710 static bool isSequentialInRange(const SmallVectorImpl<int> &Mask,
02711                                 unsigned Pos, unsigned Size, int Low) {
02712   for (unsigned i = Pos, e = Pos+Size; i != e; ++i, ++Low)
02713     if (Mask[i] >= 0 && Mask[i] != Low)
02714       return false;
02715   return true;
02716 }
02717 
02718 void SelectionDAGBuilder::visitShuffleVector(const User &I) {
02719   SDValue Src1 = getValue(I.getOperand(0));
02720   SDValue Src2 = getValue(I.getOperand(1));
02721 
02722   SmallVector<int, 8> Mask;
02723   ShuffleVectorInst::getShuffleMask(cast<Constant>(I.getOperand(2)), Mask);
02724   unsigned MaskNumElts = Mask.size();
02725 
02726   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
02727   EVT VT = TLI.getValueType(DAG.getDataLayout(), I.getType());
02728   EVT SrcVT = Src1.getValueType();
02729   unsigned SrcNumElts = SrcVT.getVectorNumElements();
02730 
02731   if (SrcNumElts == MaskNumElts) {
02732     setValue(&I, DAG.getVectorShuffle(VT, getCurSDLoc(), Src1, Src2,
02733                                       &Mask[0]));
02734     return;
02735   }
02736 
02737   // Normalize the shuffle vector since mask and vector length don't match.
02738   if (SrcNumElts < MaskNumElts && MaskNumElts % SrcNumElts == 0) {
02739     // Mask is longer than the source vectors and is a multiple of the source
02740     // vectors.  We can use concatenate vector to make the mask and vectors
02741     // lengths match.
02742     if (SrcNumElts*2 == MaskNumElts) {
02743       // First check for Src1 in low and Src2 in high
02744       if (isSequentialInRange(Mask, 0, SrcNumElts, 0) &&
02745           isSequentialInRange(Mask, SrcNumElts, SrcNumElts, SrcNumElts)) {
02746         // The shuffle is concatenating two vectors together.
02747         setValue(&I, DAG.getNode(ISD::CONCAT_VECTORS, getCurSDLoc(),
02748                                  VT, Src1, Src2));
02749         return;
02750       }
02751       // Then check for Src2 in low and Src1 in high
02752       if (isSequentialInRange(Mask, 0, SrcNumElts, SrcNumElts) &&
02753           isSequentialInRange(Mask, SrcNumElts, SrcNumElts, 0)) {
02754         // The shuffle is concatenating two vectors together.
02755         setValue(&I, DAG.getNode(ISD::CONCAT_VECTORS, getCurSDLoc(),
02756                                  VT, Src2, Src1));
02757         return;
02758       }
02759     }
02760 
02761     // Pad both vectors with undefs to make them the same length as the mask.
02762     unsigned NumConcat = MaskNumElts / SrcNumElts;
02763     bool Src1U = Src1.getOpcode() == ISD::UNDEF;
02764     bool Src2U = Src2.getOpcode() == ISD::UNDEF;
02765     SDValue UndefVal = DAG.getUNDEF(SrcVT);
02766 
02767     SmallVector<SDValue, 8> MOps1(NumConcat, UndefVal);
02768     SmallVector<SDValue, 8> MOps2(NumConcat, UndefVal);
02769     MOps1[0] = Src1;
02770     MOps2[0] = Src2;
02771 
02772     Src1 = Src1U ? DAG.getUNDEF(VT) : DAG.getNode(ISD::CONCAT_VECTORS,
02773                                                   getCurSDLoc(), VT, MOps1);
02774     Src2 = Src2U ? DAG.getUNDEF(VT) : DAG.getNode(ISD::CONCAT_VECTORS,
02775                                                   getCurSDLoc(), VT, MOps2);
02776 
02777     // Readjust mask for new input vector length.
02778     SmallVector<int, 8> MappedOps;
02779     for (unsigned i = 0; i != MaskNumElts; ++i) {
02780       int Idx = Mask[i];
02781       if (Idx >= (int)SrcNumElts)
02782         Idx -= SrcNumElts - MaskNumElts;
02783       MappedOps.push_back(Idx);
02784     }
02785 
02786     setValue(&I, DAG.getVectorShuffle(VT, getCurSDLoc(), Src1, Src2,
02787                                       &MappedOps[0]));
02788     return;
02789   }
02790 
02791   if (SrcNumElts > MaskNumElts) {
02792     // Analyze the access pattern of the vector to see if we can extract
02793     // two subvectors and do the shuffle. The analysis is done by calculating
02794     // the range of elements the mask access on both vectors.
02795     int MinRange[2] = { static_cast<int>(SrcNumElts),
02796                         static_cast<int>(SrcNumElts)};
02797     int MaxRange[2] = {-1, -1};
02798 
02799     for (unsigned i = 0; i != MaskNumElts; ++i) {
02800       int Idx = Mask[i];
02801       unsigned Input = 0;
02802       if (Idx < 0)
02803         continue;
02804 
02805       if (Idx >= (int)SrcNumElts) {
02806         Input = 1;
02807         Idx -= SrcNumElts;
02808       }
02809       if (Idx > MaxRange[Input])
02810         MaxRange[Input] = Idx;
02811       if (Idx < MinRange[Input])
02812         MinRange[Input] = Idx;
02813     }
02814 
02815     // Check if the access is smaller than the vector size and can we find
02816     // a reasonable extract index.
02817     int RangeUse[2] = { -1, -1 };  // 0 = Unused, 1 = Extract, -1 = Can not
02818                                    // Extract.
02819     int StartIdx[2];  // StartIdx to extract from
02820     for (unsigned Input = 0; Input < 2; ++Input) {
02821       if (MinRange[Input] >= (int)SrcNumElts && MaxRange[Input] < 0) {
02822         RangeUse[Input] = 0; // Unused
02823         StartIdx[Input] = 0;
02824         continue;
02825       }
02826 
02827       // Find a good start index that is a multiple of the mask length. Then
02828       // see if the rest of the elements are in range.
02829       StartIdx[Input] = (MinRange[Input]/MaskNumElts)*MaskNumElts;
02830       if (MaxRange[Input] - StartIdx[Input] < (int)MaskNumElts &&
02831           StartIdx[Input] + MaskNumElts <= SrcNumElts)
02832         RangeUse[Input] = 1; // Extract from a multiple of the mask length.
02833     }
02834 
02835     if (RangeUse[0] == 0 && RangeUse[1] == 0) {
02836       setValue(&I, DAG.getUNDEF(VT)); // Vectors are not used.
02837       return;
02838     }
02839     if (RangeUse[0] >= 0 && RangeUse[1] >= 0) {
02840       // Extract appropriate subvector and generate a vector shuffle
02841       for (unsigned Input = 0; Input < 2; ++Input) {
02842         SDValue &Src = Input == 0 ? Src1 : Src2;
02843         if (RangeUse[Input] == 0)
02844           Src = DAG.getUNDEF(VT);
02845         else {
02846           SDLoc dl = getCurSDLoc();
02847           Src = DAG.getNode(
02848               ISD::EXTRACT_SUBVECTOR, dl, VT, Src,
02849               DAG.getConstant(StartIdx[Input], dl,
02850                               TLI.getVectorIdxTy(DAG.getDataLayout())));
02851         }
02852       }
02853 
02854       // Calculate new mask.
02855       SmallVector<int, 8> MappedOps;
02856       for (unsigned i = 0; i != MaskNumElts; ++i) {
02857         int Idx = Mask[i];
02858         if (Idx >= 0) {
02859           if (Idx < (int)SrcNumElts)
02860             Idx -= StartIdx[0];
02861           else
02862             Idx -= SrcNumElts + StartIdx[1] - MaskNumElts;
02863         }
02864         MappedOps.push_back(Idx);
02865       }
02866 
02867       setValue(&I, DAG.getVectorShuffle(VT, getCurSDLoc(), Src1, Src2,
02868                                         &MappedOps[0]));
02869       return;
02870     }
02871   }
02872 
02873   // We can't use either concat vectors or extract subvectors so fall back to
02874   // replacing the shuffle with extract and build vector.
02875   // to insert and build vector.
02876   EVT EltVT = VT.getVectorElementType();
02877   EVT IdxVT = TLI.getVectorIdxTy(DAG.getDataLayout());
02878   SDLoc dl = getCurSDLoc();
02879   SmallVector<SDValue,8> Ops;
02880   for (unsigned i = 0; i != MaskNumElts; ++i) {
02881     int Idx = Mask[i];
02882     SDValue Res;
02883 
02884     if (Idx < 0) {
02885       Res = DAG.getUNDEF(EltVT);
02886     } else {
02887       SDValue &Src = Idx < (int)SrcNumElts ? Src1 : Src2;
02888       if (Idx >= (int)SrcNumElts) Idx -= SrcNumElts;
02889 
02890       Res = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl,
02891                         EltVT, Src, DAG.getConstant(Idx, dl, IdxVT));
02892     }
02893 
02894     Ops.push_back(Res);
02895   }
02896 
02897   setValue(&I, DAG.getNode(ISD::BUILD_VECTOR, dl, VT, Ops));
02898 }
02899 
02900 void SelectionDAGBuilder::visitInsertValue(const InsertValueInst &I) {
02901   const Value *Op0 = I.getOperand(0);
02902   const Value *Op1 = I.getOperand(1);
02903   Type *AggTy = I.getType();
02904   Type *ValTy = Op1->getType();
02905   bool IntoUndef = isa<UndefValue>(Op0);
02906   bool FromUndef = isa<UndefValue>(Op1);
02907 
02908   unsigned LinearIndex = ComputeLinearIndex(AggTy, I.getIndices());
02909 
02910   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
02911   SmallVector<EVT, 4> AggValueVTs;
02912   ComputeValueVTs(TLI, DAG.getDataLayout(), AggTy, AggValueVTs);
02913   SmallVector<EVT, 4> ValValueVTs;
02914   ComputeValueVTs(TLI, DAG.getDataLayout(), ValTy, ValValueVTs);
02915 
02916   unsigned NumAggValues = AggValueVTs.size();
02917   unsigned NumValValues = ValValueVTs.size();
02918   SmallVector<SDValue, 4> Values(NumAggValues);
02919 
02920   // Ignore an insertvalue that produces an empty object
02921   if (!NumAggValues) {
02922     setValue(&I, DAG.getUNDEF(MVT(MVT::Other)));
02923     return;
02924   }
02925 
02926   SDValue Agg = getValue(Op0);
02927   unsigned i = 0;
02928   // Copy the beginning value(s) from the original aggregate.
02929   for (; i != LinearIndex; ++i)
02930     Values[i] = IntoUndef ? DAG.getUNDEF(AggValueVTs[i]) :
02931                 SDValue(Agg.getNode(), Agg.getResNo() + i);
02932   // Copy values from the inserted value(s).
02933   if (NumValValues) {
02934     SDValue Val = getValue(Op1);
02935     for (; i != LinearIndex + NumValValues; ++i)
02936       Values[i] = FromUndef ? DAG.getUNDEF(AggValueVTs[i]) :
02937                   SDValue(Val.getNode(), Val.getResNo() + i - LinearIndex);
02938   }
02939   // Copy remaining value(s) from the original aggregate.
02940   for (; i != NumAggValues; ++i)
02941     Values[i] = IntoUndef ? DAG.getUNDEF(AggValueVTs[i]) :
02942                 SDValue(Agg.getNode(), Agg.getResNo() + i);
02943 
02944   setValue(&I, DAG.getNode(ISD::MERGE_VALUES, getCurSDLoc(),
02945                            DAG.getVTList(AggValueVTs), Values));
02946 }
02947 
02948 void SelectionDAGBuilder::visitExtractValue(const ExtractValueInst &I) {
02949   const Value *Op0 = I.getOperand(0);
02950   Type *AggTy = Op0->getType();
02951   Type *ValTy = I.getType();
02952   bool OutOfUndef = isa<UndefValue>(Op0);
02953 
02954   unsigned LinearIndex = ComputeLinearIndex(AggTy, I.getIndices());
02955 
02956   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
02957   SmallVector<EVT, 4> ValValueVTs;
02958   ComputeValueVTs(TLI, DAG.getDataLayout(), ValTy, ValValueVTs);
02959 
02960   unsigned NumValValues = ValValueVTs.size();
02961 
02962   // Ignore a extractvalue that produces an empty object
02963   if (!NumValValues) {
02964     setValue(&I, DAG.getUNDEF(MVT(MVT::Other)));
02965     return;
02966   }
02967 
02968   SmallVector<SDValue, 4> Values(NumValValues);
02969 
02970   SDValue Agg = getValue(Op0);
02971   // Copy out the selected value(s).
02972   for (unsigned i = LinearIndex; i != LinearIndex + NumValValues; ++i)
02973     Values[i - LinearIndex] =
02974       OutOfUndef ?
02975         DAG.getUNDEF(Agg.getNode()->getValueType(Agg.getResNo() + i)) :
02976         SDValue(Agg.getNode(), Agg.getResNo() + i);
02977 
02978   setValue(&I, DAG.getNode(ISD::MERGE_VALUES, getCurSDLoc(),
02979                            DAG.getVTList(ValValueVTs), Values));
02980 }
02981 
02982 void SelectionDAGBuilder::visitGetElementPtr(const User &I) {
02983   Value *Op0 = I.getOperand(0);
02984   // Note that the pointer operand may be a vector of pointers. Take the scalar
02985   // element which holds a pointer.
02986   unsigned AS = Op0->getType()->getScalarType()->getPointerAddressSpace();
02987   SDValue N = getValue(Op0);
02988   SDLoc dl = getCurSDLoc();
02989 
02990   // Normalize Vector GEP - all scalar operands should be converted to the
02991   // splat vector.
02992   unsigned VectorWidth = I.getType()->isVectorTy() ?
02993     cast<VectorType>(I.getType())->getVectorNumElements() : 0;
02994 
02995   if (VectorWidth && !N.getValueType().isVector()) {
02996     MVT VT = MVT::getVectorVT(N.getValueType().getSimpleVT(), VectorWidth);
02997     SmallVector<SDValue, 16> Ops(VectorWidth, N);
02998     N = DAG.getNode(ISD::BUILD_VECTOR, dl, VT, Ops);
02999   }
03000   for (gep_type_iterator GTI = gep_type_begin(&I), E = gep_type_end(&I);
03001        GTI != E; ++GTI) {
03002     const Value *Idx = GTI.getOperand();
03003     if (StructType *StTy = dyn_cast<StructType>(*GTI)) {
03004       unsigned Field = cast<Constant>(Idx)->getUniqueInteger().getZExtValue();
03005       if (Field) {
03006         // N = N + Offset
03007         uint64_t Offset = DL->getStructLayout(StTy)->getElementOffset(Field);
03008 
03009         // In an inbouds GEP with an offset that is nonnegative even when
03010         // interpreted as signed, assume there is no unsigned overflow.
03011         SDNodeFlags Flags;
03012         if (int64_t(Offset) >= 0 && cast<GEPOperator>(I).isInBounds())
03013           Flags.setNoUnsignedWrap(true);
03014 
03015         N = DAG.getNode(ISD::ADD, dl, N.getValueType(), N,
03016                         DAG.getConstant(Offset, dl, N.getValueType()), &Flags);
03017       }
03018     } else {
03019       MVT PtrTy =
03020           DAG.getTargetLoweringInfo().getPointerTy(DAG.getDataLayout(), AS);
03021       unsigned PtrSize = PtrTy.getSizeInBits();
03022       APInt ElementSize(PtrSize, DL->getTypeAllocSize(GTI.getIndexedType()));
03023 
03024       // If this is a scalar constant or a splat vector of constants,
03025       // handle it quickly.
03026       const auto *CI = dyn_cast<ConstantInt>(Idx);
03027       if (!CI && isa<ConstantDataVector>(Idx) &&
03028           cast<ConstantDataVector>(Idx)->getSplatValue())
03029         CI = cast<ConstantInt>(cast<ConstantDataVector>(Idx)->getSplatValue());
03030 
03031       if (CI) {
03032         if (CI->isZero())
03033           continue;
03034         APInt Offs = ElementSize * CI->getValue().sextOrTrunc(PtrSize);
03035         SDValue OffsVal = VectorWidth ?
03036           DAG.getConstant(Offs, dl, MVT::getVectorVT(PtrTy, VectorWidth)) :
03037           DAG.getConstant(Offs, dl, PtrTy);
03038 
03039         // In an inbouds GEP with an offset that is nonnegative even when
03040         // interpreted as signed, assume there is no unsigned overflow.
03041         SDNodeFlags Flags;
03042         if (Offs.isNonNegative() && cast<GEPOperator>(I).isInBounds())
03043           Flags.setNoUnsignedWrap(true);
03044 
03045         N = DAG.getNode(ISD::ADD, dl, N.getValueType(), N, OffsVal, &Flags);
03046         continue;
03047       }
03048 
03049       // N = N + Idx * ElementSize;
03050       SDValue IdxN = getValue(Idx);
03051 
03052       if (!IdxN.getValueType().isVector() && VectorWidth) {
03053         MVT VT = MVT::getVectorVT(IdxN.getValueType().getSimpleVT(), VectorWidth);
03054         SmallVector<SDValue, 16> Ops(VectorWidth, IdxN);
03055         IdxN = DAG.getNode(ISD::BUILD_VECTOR, dl, VT, Ops);
03056       }
03057       // If the index is smaller or larger than intptr_t, truncate or extend
03058       // it.
03059       IdxN = DAG.getSExtOrTrunc(IdxN, dl, N.getValueType());
03060 
03061       // If this is a multiply by a power of two, turn it into a shl
03062       // immediately.  This is a very common case.
03063       if (ElementSize != 1) {
03064         if (ElementSize.isPowerOf2()) {
03065           unsigned Amt = ElementSize.logBase2();
03066           IdxN = DAG.getNode(ISD::SHL, dl,
03067                              N.getValueType(), IdxN,
03068                              DAG.getConstant(Amt, dl, IdxN.getValueType()));
03069         } else {
03070           SDValue Scale = DAG.getConstant(ElementSize, dl, IdxN.getValueType());
03071           IdxN = DAG.getNode(ISD::MUL, dl,
03072                              N.getValueType(), IdxN, Scale);
03073         }
03074       }
03075 
03076       N = DAG.getNode(ISD::ADD, dl,
03077                       N.getValueType(), N, IdxN);
03078     }
03079   }
03080 
03081   setValue(&I, N);
03082 }
03083 
03084 void SelectionDAGBuilder::visitAlloca(const AllocaInst &I) {
03085   // If this is a fixed sized alloca in the entry block of the function,
03086   // allocate it statically on the stack.
03087   if (FuncInfo.StaticAllocaMap.count(&I))
03088     return;   // getValue will auto-populate this.
03089 
03090   SDLoc dl = getCurSDLoc();
03091   Type *Ty = I.getAllocatedType();
03092   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
03093   auto &DL = DAG.getDataLayout();
03094   uint64_t TySize = DL.getTypeAllocSize(Ty);
03095   unsigned Align =
03096       std::max((unsigned)DL.getPrefTypeAlignment(Ty), I.getAlignment());
03097 
03098   SDValue AllocSize = getValue(I.getArraySize());
03099 
03100   EVT IntPtr = TLI.getPointerTy(DAG.getDataLayout());
03101   if (AllocSize.getValueType() != IntPtr)
03102     AllocSize = DAG.getZExtOrTrunc(AllocSize, dl, IntPtr);
03103 
03104   AllocSize = DAG.getNode(ISD::MUL, dl, IntPtr,
03105                           AllocSize,
03106                           DAG.getConstant(TySize, dl, IntPtr));
03107 
03108   // Handle alignment.  If the requested alignment is less than or equal to
03109   // the stack alignment, ignore it.  If the size is greater than or equal to
03110   // the stack alignment, we note this in the DYNAMIC_STACKALLOC node.
03111   unsigned StackAlign =
03112       DAG.getSubtarget().getFrameLowering()->getStackAlignment();
03113   if (Align <= StackAlign)
03114     Align = 0;
03115 
03116   // Round the size of the allocation up to the stack alignment size
03117   // by add SA-1 to the size. This doesn't overflow because we're computing
03118   // an address inside an alloca.
03119   SDNodeFlags Flags;
03120   Flags.setNoUnsignedWrap(true);
03121   AllocSize = DAG.getNode(ISD::ADD, dl,
03122                           AllocSize.getValueType(), AllocSize,
03123                           DAG.getIntPtrConstant(StackAlign - 1, dl), &Flags);
03124 
03125   // Mask out the low bits for alignment purposes.
03126   AllocSize = DAG.getNode(ISD::AND, dl,
03127                           AllocSize.getValueType(), AllocSize,
03128                           DAG.getIntPtrConstant(~(uint64_t)(StackAlign - 1),
03129                                                 dl));
03130 
03131   SDValue Ops[] = { getRoot(), AllocSize, DAG.getIntPtrConstant(Align, dl) };
03132   SDVTList VTs = DAG.getVTList(AllocSize.getValueType(), MVT::Other);
03133   SDValue DSA = DAG.getNode(ISD::DYNAMIC_STACKALLOC, dl, VTs, Ops);
03134   setValue(&I, DSA);
03135   DAG.setRoot(DSA.getValue(1));
03136 
03137   assert(FuncInfo.MF->getFrameInfo()->hasVarSizedObjects());
03138 }
03139 
03140 void SelectionDAGBuilder::visitLoad(const LoadInst &I) {
03141   if (I.isAtomic())
03142     return visitAtomicLoad(I);
03143 
03144   const Value *SV = I.getOperand(0);
03145   SDValue Ptr = getValue(SV);
03146 
03147   Type *Ty = I.getType();
03148 
03149   bool isVolatile = I.isVolatile();
03150   bool isNonTemporal = I.getMetadata(LLVMContext::MD_nontemporal) != nullptr;
03151 
03152   // The IR notion of invariant_load only guarantees that all *non-faulting*
03153   // invariant loads result in the same value.  The MI notion of invariant load
03154   // guarantees that the load can be legally moved to any location within its
03155   // containing function.  The MI notion of invariant_load is stronger than the
03156   // IR notion of invariant_load -- an MI invariant_load is an IR invariant_load
03157   // with a guarantee that the location being loaded from is dereferenceable
03158   // throughout the function's lifetime.
03159 
03160   bool isInvariant = I.getMetadata(LLVMContext::MD_invariant_load) != nullptr &&
03161                      isDereferenceablePointer(SV, DAG.getDataLayout());
03162   unsigned Alignment = I.getAlignment();
03163 
03164   AAMDNodes AAInfo;
03165   I.getAAMetadata(AAInfo);
03166   const MDNode *Ranges = I.getMetadata(LLVMContext::MD_range);
03167 
03168   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
03169   SmallVector<EVT, 4> ValueVTs;
03170   SmallVector<uint64_t, 4> Offsets;
03171   ComputeValueVTs(TLI, DAG.getDataLayout(), Ty, ValueVTs, &Offsets);
03172   unsigned NumValues = ValueVTs.size();
03173   if (NumValues == 0)
03174     return;
03175 
03176   SDValue Root;
03177   bool ConstantMemory = false;
03178   if (isVolatile || NumValues > MaxParallelChains)
03179     // Serialize volatile loads with other side effects.
03180     Root = getRoot();
03181   else if (AA->pointsToConstantMemory(MemoryLocation(
03182                SV, DAG.getDataLayout().getTypeStoreSize(Ty), AAInfo))) {
03183     // Do not serialize (non-volatile) loads of constant memory with anything.
03184     Root = DAG.getEntryNode();
03185     ConstantMemory = true;
03186   } else {
03187     // Do not serialize non-volatile loads against each other.
03188     Root = DAG.getRoot();
03189   }
03190 
03191   SDLoc dl = getCurSDLoc();
03192 
03193   if (isVolatile)
03194     Root = TLI.prepareVolatileOrAtomicLoad(Root, dl, DAG);
03195 
03196   // An aggregate load cannot wrap around the address space, so offsets to its
03197   // parts don't wrap either.
03198   SDNodeFlags Flags;
03199   Flags.setNoUnsignedWrap(true);
03200 
03201   SmallVector<SDValue, 4> Values(NumValues);
03202   SmallVector<SDValue, 4> Chains(std::min(MaxParallelChains, NumValues));
03203   EVT PtrVT = Ptr.getValueType();
03204   unsigned ChainI = 0;
03205   for (unsigned i = 0; i != NumValues; ++i, ++ChainI) {
03206     // Serializing loads here may result in excessive register pressure, and
03207     // TokenFactor places arbitrary choke points on the scheduler. SD scheduling
03208     // could recover a bit by hoisting nodes upward in the chain by recognizing
03209     // they are side-effect free or do not alias. The optimizer should really
03210     // avoid this case by converting large object/array copies to llvm.memcpy
03211     // (MaxParallelChains should always remain as failsafe).
03212     if (ChainI == MaxParallelChains) {
03213       assert(PendingLoads.empty() && "PendingLoads must be serialized first");
03214       SDValue Chain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other,
03215                                   makeArrayRef(Chains.data(), ChainI));
03216       Root = Chain;
03217       ChainI = 0;
03218     }
03219     SDValue A = DAG.getNode(ISD::ADD, dl,
03220                             PtrVT, Ptr,
03221                             DAG.getConstant(Offsets[i], dl, PtrVT),
03222                             &Flags);
03223     SDValue L = DAG.getLoad(ValueVTs[i], dl, Root,
03224                             A, MachinePointerInfo(SV, Offsets[i]), isVolatile,
03225                             isNonTemporal, isInvariant, Alignment, AAInfo,
03226                             Ranges);
03227 
03228     Values[i] = L;
03229     Chains[ChainI] = L.getValue(1);
03230   }
03231 
03232   if (!ConstantMemory) {
03233     SDValue Chain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other,
03234                                 makeArrayRef(Chains.data(), ChainI));
03235     if (isVolatile)
03236       DAG.setRoot(Chain);
03237     else
03238       PendingLoads.push_back(Chain);
03239   }
03240 
03241   setValue(&I, DAG.getNode(ISD::MERGE_VALUES, dl,
03242                            DAG.getVTList(ValueVTs), Values));
03243 }
03244 
03245 void SelectionDAGBuilder::visitStore(const StoreInst &I) {
03246   if (I.isAtomic())
03247     return visitAtomicStore(I);
03248 
03249   const Value *SrcV = I.getOperand(0);
03250   const Value *PtrV = I.getOperand(1);
03251 
03252   SmallVector<EVT, 4> ValueVTs;
03253   SmallVector<uint64_t, 4> Offsets;
03254   ComputeValueVTs(DAG.getTargetLoweringInfo(), DAG.getDataLayout(),
03255                   SrcV->getType(), ValueVTs, &Offsets);
03256   unsigned NumValues = ValueVTs.size();
03257   if (NumValues == 0)
03258     return;
03259 
03260   // Get the lowered operands. Note that we do this after
03261   // checking if NumResults is zero, because with zero results
03262   // the operands won't have values in the map.
03263   SDValue Src = getValue(SrcV);
03264   SDValue Ptr = getValue(PtrV);
03265 
03266   SDValue Root = getRoot();
03267   SmallVector<SDValue, 4> Chains(std::min(MaxParallelChains, NumValues));
03268   EVT PtrVT = Ptr.getValueType();
03269   bool isVolatile = I.isVolatile();
03270   bool isNonTemporal = I.getMetadata(LLVMContext::MD_nontemporal) != nullptr;
03271   unsigned Alignment = I.getAlignment();
03272   SDLoc dl = getCurSDLoc();
03273 
03274   AAMDNodes AAInfo;
03275   I.getAAMetadata(AAInfo);
03276 
03277   // An aggregate load cannot wrap around the address space, so offsets to its
03278   // parts don't wrap either.
03279   SDNodeFlags Flags;
03280   Flags.setNoUnsignedWrap(true);
03281 
03282   unsigned ChainI = 0;
03283   for (unsigned i = 0; i != NumValues; ++i, ++ChainI) {
03284     // See visitLoad comments.
03285     if (ChainI == MaxParallelChains) {
03286       SDValue Chain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other,
03287                                   makeArrayRef(Chains.data(), ChainI));
03288       Root = Chain;
03289       ChainI = 0;
03290     }
03291     SDValue Add = DAG.getNode(ISD::ADD, dl, PtrVT, Ptr,
03292                               DAG.getConstant(Offsets[i], dl, PtrVT), &Flags);
03293     SDValue St = DAG.getStore(Root, dl,
03294                               SDValue(Src.getNode(), Src.getResNo() + i),
03295                               Add, MachinePointerInfo(PtrV, Offsets[i]),
03296                               isVolatile, isNonTemporal, Alignment, AAInfo);
03297     Chains[ChainI] = St;
03298   }
03299 
03300   SDValue StoreNode = DAG.getNode(ISD::TokenFactor, dl, MVT::Other,
03301                                   makeArrayRef(Chains.data(), ChainI));
03302   DAG.setRoot(StoreNode);
03303 }
03304 
03305 void SelectionDAGBuilder::visitMaskedStore(const CallInst &I) {
03306   SDLoc sdl = getCurSDLoc();
03307 
03308   // llvm.masked.store.*(Src0, Ptr, alignment, Mask)
03309   Value  *PtrOperand = I.getArgOperand(1);
03310   SDValue Ptr = getValue(PtrOperand);
03311   SDValue Src0 = getValue(I.getArgOperand(0));
03312   SDValue Mask = getValue(I.getArgOperand(3));
03313   EVT VT = Src0.getValueType();
03314   unsigned Alignment = (cast<ConstantInt>(I.getArgOperand(2)))->getZExtValue();
03315   if (!Alignment)
03316     Alignment = DAG.getEVTAlignment(VT);
03317 
03318   AAMDNodes AAInfo;
03319   I.getAAMetadata(AAInfo);
03320 
03321   MachineMemOperand *MMO =
03322     DAG.getMachineFunction().
03323     getMachineMemOperand(MachinePointerInfo(PtrOperand),
03324                           MachineMemOperand::MOStore,  VT.getStoreSize(),
03325                           Alignment, AAInfo);
03326   SDValue StoreNode = DAG.getMaskedStore(getRoot(), sdl, Src0, Ptr, Mask, VT,
03327                                          MMO, false);
03328   DAG.setRoot(StoreNode);
03329   setValue(&I, StoreNode);
03330 }
03331 
03332 // Get a uniform base for the Gather/Scatter intrinsic.
03333 // The first argument of the Gather/Scatter intrinsic is a vector of pointers.
03334 // We try to represent it as a base pointer + vector of indices.
03335 // Usually, the vector of pointers comes from a 'getelementptr' instruction.
03336 // The first operand of the GEP may be a single pointer or a vector of pointers
03337 // Example:
03338 //   %gep.ptr = getelementptr i32, <8 x i32*> %vptr, <8 x i32> %ind
03339 //  or
03340 //   %gep.ptr = getelementptr i32, i32* %ptr,        <8 x i32> %ind
03341 // %res = call <8 x i32> @llvm.masked.gather.v8i32(<8 x i32*> %gep.ptr, ..
03342 //
03343 // When the first GEP operand is a single pointer - it is the uniform base we
03344 // are looking for. If first operand of the GEP is a splat vector - we
03345 // extract the spalt value and use it as a uniform base.
03346 // In all other cases the function returns 'false'.
03347 //
03348 static bool getUniformBase(const Value *& Ptr, SDValue& Base, SDValue& Index,
03349                            SelectionDAGBuilder* SDB) {
03350 
03351   SelectionDAG& DAG = SDB->DAG;
03352   LLVMContext &Context = *DAG.getContext();
03353 
03354   assert(Ptr->getType()->isVectorTy() && "Uexpected pointer type");
03355   const GetElementPtrInst *GEP = dyn_cast<GetElementPtrInst>(Ptr);
03356   if (!GEP || GEP->getNumOperands() > 2)
03357     return false;
03358 
03359   const Value *GEPPtr = GEP->getPointerOperand();
03360   if (!GEPPtr->getType()->isVectorTy())
03361     Ptr = GEPPtr;
03362   else if (!(Ptr = getSplatValue(GEPPtr)))
03363     return false;
03364 
03365   Value *IndexVal = GEP->getOperand(1);
03366 
03367   // The operands of the GEP may be defined in another basic block.
03368   // In this case we'll not find nodes for the operands.
03369   if (!SDB->findValue(Ptr) || !SDB->findValue(IndexVal))
03370     return false;
03371 
03372   Base = SDB->getValue(Ptr);
03373   Index = SDB->getValue(IndexVal);
03374 
03375   // Suppress sign extension.
03376   if (SExtInst* Sext = dyn_cast<SExtInst>(IndexVal)) {
03377     if (SDB->findValue(Sext->getOperand(0))) {
03378       IndexVal = Sext->getOperand(0);
03379       Index = SDB->getValue(IndexVal);
03380     }
03381   }
03382   if (!Index.getValueType().isVector()) {
03383     unsigned GEPWidth = GEP->getType()->getVectorNumElements();
03384     EVT VT = EVT::getVectorVT(Context, Index.getValueType(), GEPWidth);
03385     SmallVector<SDValue, 16> Ops(GEPWidth, Index);
03386     Index = DAG.getNode(ISD::BUILD_VECTOR, SDLoc(Index), VT, Ops);
03387   }
03388   return true;
03389 }
03390 
03391 void SelectionDAGBuilder::visitMaskedScatter(const CallInst &I) {
03392   SDLoc sdl = getCurSDLoc();
03393 
03394   // llvm.masked.scatter.*(Src0, Ptrs, alignemt, Mask)
03395   const Value *Ptr = I.getArgOperand(1);
03396   SDValue Src0 = getValue(I.getArgOperand(0));
03397   SDValue Mask = getValue(I.getArgOperand(3));
03398   EVT VT = Src0.getValueType();
03399   unsigned Alignment = (cast<ConstantInt>(I.getArgOperand(2)))->getZExtValue();
03400   if (!Alignment)
03401     Alignment = DAG.getEVTAlignment(VT);
03402   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
03403 
03404   AAMDNodes AAInfo;
03405   I.getAAMetadata(AAInfo);
03406 
03407   SDValue Base;
03408   SDValue Index;
03409   const Value *BasePtr = Ptr;
03410   bool UniformBase = getUniformBase(BasePtr, Base, Index, this);
03411 
03412   const Value *MemOpBasePtr = UniformBase ? BasePtr : nullptr;
03413   MachineMemOperand *MMO = DAG.getMachineFunction().
03414     getMachineMemOperand(MachinePointerInfo(MemOpBasePtr),
03415                          MachineMemOperand::MOStore,  VT.getStoreSize(),
03416                          Alignment, AAInfo);
03417   if (!UniformBase) {
03418     Base = DAG.getTargetConstant(0, sdl, TLI.getPointerTy(DAG.getDataLayout()));
03419     Index = getValue(Ptr);
03420   }
03421   SDValue Ops[] = { getRoot(), Src0, Mask, Base, Index };
03422   SDValue Scatter = DAG.getMaskedScatter(DAG.getVTList(MVT::Other), VT, sdl,
03423                                          Ops, MMO);
03424   DAG.setRoot(Scatter);
03425   setValue(&I, Scatter);
03426 }
03427 
03428 void SelectionDAGBuilder::visitMaskedLoad(const CallInst &I) {
03429   SDLoc sdl = getCurSDLoc();
03430 
03431   // @llvm.masked.load.*(Ptr, alignment, Mask, Src0)
03432   Value  *PtrOperand = I.getArgOperand(0);
03433   SDValue Ptr = getValue(PtrOperand);
03434   SDValue Src0 = getValue(I.getArgOperand(3));
03435   SDValue Mask = getValue(I.getArgOperand(2));
03436 
03437   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
03438   EVT VT = TLI.getValueType(DAG.getDataLayout(), I.getType());
03439   unsigned Alignment = (cast<ConstantInt>(I.getArgOperand(1)))->getZExtValue();
03440   if (!Alignment)
03441     Alignment = DAG.getEVTAlignment(VT);
03442 
03443   AAMDNodes AAInfo;
03444   I.getAAMetadata(AAInfo);
03445   const MDNode *Ranges = I.getMetadata(LLVMContext::MD_range);
03446 
03447   SDValue InChain = DAG.getRoot();
03448   if (AA->pointsToConstantMemory(MemoryLocation(
03449           PtrOperand, DAG.getDataLayout().getTypeStoreSize(I.getType()),
03450           AAInfo))) {
03451     // Do not serialize (non-volatile) loads of constant memory with anything.
03452     InChain = DAG.getEntryNode();
03453   }
03454 
03455   MachineMemOperand *MMO =
03456     DAG.getMachineFunction().
03457     getMachineMemOperand(MachinePointerInfo(PtrOperand),
03458                           MachineMemOperand::MOLoad,  VT.getStoreSize(),
03459                           Alignment, AAInfo, Ranges);
03460 
03461   SDValue Load = DAG.getMaskedLoad(VT, sdl, InChain, Ptr, Mask, Src0, VT, MMO,
03462                                    ISD::NON_EXTLOAD);
03463   SDValue OutChain = Load.getValue(1);
03464   DAG.setRoot(OutChain);
03465   setValue(&I, Load);
03466 }
03467 
03468 void SelectionDAGBuilder::visitMaskedGather(const CallInst &I) {
03469   SDLoc sdl = getCurSDLoc();
03470 
03471   // @llvm.masked.gather.*(Ptrs, alignment, Mask, Src0)
03472   const Value *Ptr = I.getArgOperand(0);
03473   SDValue Src0 = getValue(I.getArgOperand(3));
03474   SDValue Mask = getValue(I.getArgOperand(2));
03475 
03476   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
03477   EVT VT = TLI.getValueType(DAG.getDataLayout(), I.getType());
03478   unsigned Alignment = (cast<ConstantInt>(I.getArgOperand(1)))->getZExtValue();
03479   if (!Alignment)
03480     Alignment = DAG.getEVTAlignment(VT);
03481 
03482   AAMDNodes AAInfo;
03483   I.getAAMetadata(AAInfo);
03484   const MDNode *Ranges = I.getMetadata(LLVMContext::MD_range);
03485 
03486   SDValue Root = DAG.getRoot();
03487   SDValue Base;
03488   SDValue Index;
03489   const Value *BasePtr = Ptr;
03490   bool UniformBase = getUniformBase(BasePtr, Base, Index, this);
03491   bool ConstantMemory = false;
03492   if (UniformBase &&
03493       AA->pointsToConstantMemory(MemoryLocation(
03494           BasePtr, DAG.getDataLayout().getTypeStoreSize(I.getType()),
03495           AAInfo))) {
03496     // Do not serialize (non-volatile) loads of constant memory with anything.
03497     Root = DAG.getEntryNode();
03498     ConstantMemory = true;
03499   }
03500 
03501   MachineMemOperand *MMO =
03502     DAG.getMachineFunction().
03503     getMachineMemOperand(MachinePointerInfo(UniformBase ? BasePtr : nullptr),
03504                          MachineMemOperand::MOLoad,  VT.getStoreSize(),
03505                          Alignment, AAInfo, Ranges);
03506 
03507   if (!UniformBase) {
03508     Base = DAG.getTargetConstant(0, sdl, TLI.getPointerTy(DAG.getDataLayout()));
03509     Index = getValue(Ptr);
03510   }
03511   SDValue Ops[] = { Root, Src0, Mask, Base, Index };
03512   SDValue Gather = DAG.getMaskedGather(DAG.getVTList(VT, MVT::Other), VT, sdl,
03513                                        Ops, MMO);
03514 
03515   SDValue OutChain = Gather.getValue(1);
03516   if (!ConstantMemory)
03517     PendingLoads.push_back(OutChain);
03518   setValue(&I, Gather);
03519 }
03520 
03521 void SelectionDAGBuilder::visitAtomicCmpXchg(const AtomicCmpXchgInst &I) {
03522   SDLoc dl = getCurSDLoc();
03523   AtomicOrdering SuccessOrder = I.getSuccessOrdering();
03524   AtomicOrdering FailureOrder = I.getFailureOrdering();
03525   SynchronizationScope Scope = I.getSynchScope();
03526 
03527   SDValue InChain = getRoot();
03528 
03529   MVT MemVT = getValue(I.getCompareOperand()).getSimpleValueType();
03530   SDVTList VTs = DAG.getVTList(MemVT, MVT::i1, MVT::Other);
03531   SDValue L = DAG.getAtomicCmpSwap(
03532       ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS, dl, MemVT, VTs, InChain,
03533       getValue(I.getPointerOperand()), getValue(I.getCompareOperand()),
03534       getValue(I.getNewValOperand()), MachinePointerInfo(I.getPointerOperand()),
03535       /*Alignment=*/ 0, SuccessOrder, FailureOrder, Scope);
03536 
03537   SDValue OutChain = L.getValue(2);
03538 
03539   setValue(&I, L);
03540   DAG.setRoot(OutChain);
03541 }
03542 
03543 void SelectionDAGBuilder::visitAtomicRMW(const AtomicRMWInst &I) {
03544   SDLoc dl = getCurSDLoc();
03545   ISD::NodeType NT;
03546   switch (I.getOperation()) {
03547   default: llvm_unreachable("Unknown atomicrmw operation");
03548   case AtomicRMWInst::Xchg: NT = ISD::ATOMIC_SWAP; break;
03549   case AtomicRMWInst::Add:  NT = ISD::ATOMIC_LOAD_ADD; break;
03550   case AtomicRMWInst::Sub:  NT = ISD::ATOMIC_LOAD_SUB; break;
03551   case AtomicRMWInst::And:  NT = ISD::ATOMIC_LOAD_AND; break;
03552   case AtomicRMWInst::Nand: NT = ISD::ATOMIC_LOAD_NAND; break;
03553   case AtomicRMWInst::Or:   NT = ISD::ATOMIC_LOAD_OR; break;
03554   case AtomicRMWInst::Xor:  NT = ISD::ATOMIC_LOAD_XOR; break;
03555   case AtomicRMWInst::Max:  NT = ISD::ATOMIC_LOAD_MAX; break;
03556   case AtomicRMWInst::Min:  NT = ISD::ATOMIC_LOAD_MIN; break;
03557   case AtomicRMWInst::UMax: NT = ISD::ATOMIC_LOAD_UMAX; break;
03558   case AtomicRMWInst::UMin: NT = ISD::ATOMIC_LOAD_UMIN; break;
03559   }
03560   AtomicOrdering Order = I.getOrdering();
03561   SynchronizationScope Scope = I.getSynchScope();
03562 
03563   SDValue InChain = getRoot();
03564 
03565   SDValue L =
03566     DAG.getAtomic(NT, dl,
03567                   getValue(I.getValOperand()).getSimpleValueType(),
03568                   InChain,
03569                   getValue(I.getPointerOperand()),
03570                   getValue(I.getValOperand()),
03571                   I.getPointerOperand(),
03572                   /* Alignment=*/ 0, Order, Scope);
03573 
03574   SDValue OutChain = L.getValue(1);
03575 
03576   setValue(&I, L);
03577   DAG.setRoot(OutChain);
03578 }
03579 
03580 void SelectionDAGBuilder::visitFence(const FenceInst &I) {
03581   SDLoc dl = getCurSDLoc();
03582   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
03583   SDValue Ops[3];
03584   Ops[0] = getRoot();
03585   Ops[1] = DAG.getConstant(I.getOrdering(), dl,
03586                            TLI.getPointerTy(DAG.getDataLayout()));
03587   Ops[2] = DAG.getConstant(I.getSynchScope(), dl,
03588                            TLI.getPointerTy(DAG.getDataLayout()));
03589   DAG.setRoot(DAG.getNode(ISD::ATOMIC_FENCE, dl, MVT::Other, Ops));
03590 }
03591 
03592 void SelectionDAGBuilder::visitAtomicLoad(const LoadInst &I) {
03593   SDLoc dl = getCurSDLoc();
03594   AtomicOrdering Order = I.getOrdering();
03595   SynchronizationScope Scope = I.getSynchScope();
03596 
03597   SDValue InChain = getRoot();
03598 
03599   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
03600   EVT VT = TLI.getValueType(DAG.getDataLayout(), I.getType());
03601 
03602   if (I.getAlignment() < VT.getSizeInBits() / 8)
03603     report_fatal_error("Cannot generate unaligned atomic load");
03604 
03605   MachineMemOperand *MMO =
03606       DAG.getMachineFunction().
03607       getMachineMemOperand(MachinePointerInfo(I.getPointerOperand()),
03608                            MachineMemOperand::MOVolatile |
03609                            MachineMemOperand::MOLoad,
03610                            VT.getStoreSize(),
03611                            I.getAlignment() ? I.getAlignment() :
03612                                               DAG.getEVTAlignment(VT));
03613 
03614   InChain = TLI.prepareVolatileOrAtomicLoad(InChain, dl, DAG);
03615   SDValue L =
03616       DAG.getAtomic(ISD::ATOMIC_LOAD, dl, VT, VT, InChain,
03617                     getValue(I.getPointerOperand()), MMO,
03618                     Order, Scope);
03619 
03620   SDValue OutChain = L.getValue(1);
03621 
03622   setValue(&I, L);
03623   DAG.setRoot(OutChain);
03624 }
03625 
03626 void SelectionDAGBuilder::visitAtomicStore(const StoreInst &I) {
03627   SDLoc dl = getCurSDLoc();
03628 
03629   AtomicOrdering Order = I.getOrdering();
03630   SynchronizationScope Scope = I.getSynchScope();
03631 
03632   SDValue InChain = getRoot();
03633 
03634   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
03635   EVT VT =
03636       TLI.getValueType(DAG.getDataLayout(), I.getValueOperand()->getType());
03637 
03638   if (I.getAlignment() < VT.getSizeInBits() / 8)
03639     report_fatal_error("Cannot generate unaligned atomic store");
03640 
03641   SDValue OutChain =
03642     DAG.getAtomic(ISD::ATOMIC_STORE, dl, VT,
03643                   InChain,
03644                   getValue(I.getPointerOperand()),
03645                   getValue(I.getValueOperand()),
03646                   I.getPointerOperand(), I.getAlignment(),
03647                   Order, Scope);
03648 
03649   DAG.setRoot(OutChain);
03650 }
03651 
03652 /// visitTargetIntrinsic - Lower a call of a target intrinsic to an INTRINSIC
03653 /// node.
03654 void SelectionDAGBuilder::visitTargetIntrinsic(const CallInst &I,
03655                                                unsigned Intrinsic) {
03656   bool HasChain = !I.doesNotAccessMemory();
03657   bool OnlyLoad = HasChain && I.onlyReadsMemory();
03658 
03659   // Build the operand list.
03660   SmallVector<SDValue, 8> Ops;
03661   if (HasChain) {  // If this intrinsic has side-effects, chainify it.
03662     if (OnlyLoad) {
03663       // We don't need to serialize loads against other loads.
03664       Ops.push_back(DAG.getRoot());
03665     } else {
03666       Ops.push_back(getRoot());
03667     }
03668   }
03669 
03670   // Info is set by getTgtMemInstrinsic
03671   TargetLowering::IntrinsicInfo Info;
03672   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
03673   bool IsTgtIntrinsic = TLI.getTgtMemIntrinsic(Info, I, Intrinsic);
03674 
03675   // Add the intrinsic ID as an integer operand if it's not a target intrinsic.
03676   if (!IsTgtIntrinsic || Info.opc == ISD::INTRINSIC_VOID ||
03677       Info.opc == ISD::INTRINSIC_W_CHAIN)
03678     Ops.push_back(DAG.getTargetConstant(Intrinsic, getCurSDLoc(),
03679                                         TLI.getPointerTy(DAG.getDataLayout())));
03680 
03681   // Add all operands of the call to the operand list.
03682   for (unsigned i = 0, e = I.getNumArgOperands(); i != e; ++i) {
03683     SDValue Op = getValue(I.getArgOperand(i));
03684     Ops.push_back(Op);
03685   }
03686 
03687   SmallVector<EVT, 4> ValueVTs;
03688   ComputeValueVTs(TLI, DAG.getDataLayout(), I.getType(), ValueVTs);
03689 
03690   if (HasChain)
03691     ValueVTs.push_back(MVT::Other);
03692 
03693   SDVTList VTs = DAG.getVTList(ValueVTs);
03694 
03695   // Create the node.
03696   SDValue Result;
03697   if (IsTgtIntrinsic) {
03698     // This is target intrinsic that touches memory
03699     Result = DAG.getMemIntrinsicNode(Info.opc, getCurSDLoc(),
03700                                      VTs, Ops, Info.memVT,
03701                                    MachinePointerInfo(Info.ptrVal, Info.offset),
03702                                      Info.align, Info.vol,
03703                                      Info.readMem, Info.writeMem, Info.size);
03704   } else if (!HasChain) {
03705     Result = DAG.getNode(ISD::INTRINSIC_WO_CHAIN, getCurSDLoc(), VTs, Ops);
03706   } else if (!I.getType()->isVoidTy()) {
03707     Result = DAG.getNode(ISD::INTRINSIC_W_CHAIN, getCurSDLoc(), VTs, Ops);
03708   } else {
03709     Result = DAG.getNode(ISD::INTRINSIC_VOID, getCurSDLoc(), VTs, Ops);
03710   }
03711 
03712   if (HasChain) {
03713     SDValue Chain = Result.getValue(Result.getNode()->getNumValues()-1);
03714     if (OnlyLoad)
03715       PendingLoads.push_back(Chain);
03716     else
03717       DAG.setRoot(Chain);
03718   }
03719 
03720   if (!I.getType()->isVoidTy()) {
03721     if (VectorType *PTy = dyn_cast<VectorType>(I.getType())) {
03722       EVT VT = TLI.getValueType(DAG.getDataLayout(), PTy);
03723       Result = DAG.getNode(ISD::BITCAST, getCurSDLoc(), VT, Result);
03724     }
03725 
03726     setValue(&I, Result);
03727   }
03728 }
03729 
03730 /// GetSignificand - Get the significand and build it into a floating-point
03731 /// number with exponent of 1:
03732 ///
03733 ///   Op = (Op & 0x007fffff) | 0x3f800000;
03734 ///
03735 /// where Op is the hexadecimal representation of floating point value.
03736 static SDValue
03737 GetSignificand(SelectionDAG &DAG, SDValue Op, SDLoc dl) {
03738   SDValue t1 = DAG.getNode(ISD::AND, dl, MVT::i32, Op,
03739                            DAG.getConstant(0x007fffff, dl, MVT::i32));
03740   SDValue t2 = DAG.getNode(ISD::OR, dl, MVT::i32, t1,
03741                            DAG.getConstant(0x3f800000, dl, MVT::i32));
03742   return DAG.getNode(ISD::BITCAST, dl, MVT::f32, t2);
03743 }
03744 
03745 /// GetExponent - Get the exponent:
03746 ///
03747 ///   (float)(int)(((Op & 0x7f800000) >> 23) - 127);
03748 ///
03749 /// where Op is the hexadecimal representation of floating point value.
03750 static SDValue
03751 GetExponent(SelectionDAG &DAG, SDValue Op, const TargetLowering &TLI,
03752             SDLoc dl) {
03753   SDValue t0 = DAG.getNode(ISD::AND, dl, MVT::i32, Op,
03754                            DAG.getConstant(0x7f800000, dl, MVT::i32));
03755   SDValue t1 = DAG.getNode(
03756       ISD::SRL, dl, MVT::i32, t0,
03757       DAG.getConstant(23, dl, TLI.getPointerTy(DAG.getDataLayout())));
03758   SDValue t2 = DAG.getNode(ISD::SUB, dl, MVT::i32, t1,
03759                            DAG.getConstant(127, dl, MVT::i32));
03760   return DAG.getNode(ISD::SINT_TO_FP, dl, MVT::f32, t2);
03761 }
03762 
03763 /// getF32Constant - Get 32-bit floating point constant.
03764 static SDValue
03765 getF32Constant(SelectionDAG &DAG, unsigned Flt, SDLoc dl) {
03766   return DAG.getConstantFP(APFloat(APFloat::IEEEsingle, APInt(32, Flt)), dl,
03767                            MVT::f32);
03768 }
03769 
03770 static SDValue getLimitedPrecisionExp2(SDValue t0, SDLoc dl,
03771                                        SelectionDAG &DAG) {
03772   // TODO: What fast-math-flags should be set on the floating-point nodes?
03773 
03774   //   IntegerPartOfX = ((int32_t)(t0);
03775   SDValue IntegerPartOfX = DAG.getNode(ISD::FP_TO_SINT, dl, MVT::i32, t0);
03776 
03777   //   FractionalPartOfX = t0 - (float)IntegerPartOfX;
03778   SDValue t1 = DAG.getNode(ISD::SINT_TO_FP, dl, MVT::f32, IntegerPartOfX);
03779   SDValue X = DAG.getNode(ISD::FSUB, dl, MVT::f32, t0, t1);
03780 
03781   //   IntegerPartOfX <<= 23;
03782   IntegerPartOfX = DAG.getNode(
03783       ISD::SHL, dl, MVT::i32, IntegerPartOfX,
03784       DAG.getConstant(23, dl, DAG.getTargetLoweringInfo().getPointerTy(
03785                                   DAG.getDataLayout())));
03786 
03787   SDValue TwoToFractionalPartOfX;
03788   if (LimitFloatPrecision <= 6) {
03789     // For floating-point precision of 6:
03790     //
03791     //   TwoToFractionalPartOfX =
03792     //     0.997535578f +
03793     //       (0.735607626f + 0.252464424f * x) * x;
03794     //
03795     // error 0.0144103317, which is 6 bits
03796     SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X,
03797                              getF32Constant(DAG, 0x3e814304, dl));
03798     SDValue t3 = DAG.getNode(ISD::FADD, dl, MVT::f32, t2,
03799                              getF32Constant(DAG, 0x3f3c50c8, dl));
03800     SDValue t4 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t3, X);
03801     TwoToFractionalPartOfX = DAG.getNode(ISD::FADD, dl, MVT::f32, t4,
03802                                          getF32Constant(DAG, 0x3f7f5e7e, dl));
03803   } else if (LimitFloatPrecision <= 12) {
03804     // For floating-point precision of 12:
03805     //
03806     //   TwoToFractionalPartOfX =
03807     //     0.999892986f +
03808     //       (0.696457318f +
03809     //         (0.224338339f + 0.792043434e-1f * x) * x) * x;
03810     //
03811     // error 0.000107046256, which is 13 to 14 bits
03812     SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X,
03813                              getF32Constant(DAG, 0x3da235e3, dl));
03814     SDValue t3 = DAG.getNode(ISD::FADD, dl, MVT::f32, t2,
03815                              getF32Constant(DAG, 0x3e65b8f3, dl));
03816     SDValue t4 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t3, X);
03817     SDValue t5 = DAG.getNode(ISD::FADD, dl, MVT::f32, t4,
03818                              getF32Constant(DAG, 0x3f324b07, dl));
03819     SDValue t6 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t5, X);
03820     TwoToFractionalPartOfX = DAG.getNode(ISD::FADD, dl, MVT::f32, t6,
03821                                          getF32Constant(DAG, 0x3f7ff8fd, dl));
03822   } else { // LimitFloatPrecision <= 18
03823     // For floating-point precision of 18:
03824     //
03825     //   TwoToFractionalPartOfX =
03826     //     0.999999982f +
03827     //       (0.693148872f +
03828     //         (0.240227044f +
03829     //           (0.554906021e-1f +
03830     //             (0.961591928e-2f +
03831     //               (0.136028312e-2f + 0.157059148e-3f *x)*x)*x)*x)*x)*x;
03832     // error 2.47208000*10^(-7), which is better than 18 bits
03833     SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X,
03834                              getF32Constant(DAG, 0x3924b03e, dl));
03835     SDValue t3 = DAG.getNode(ISD::FADD, dl, MVT::f32, t2,
03836                              getF32Constant(DAG, 0x3ab24b87, dl));
03837     SDValue t4 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t3, X);
03838     SDValue t5 = DAG.getNode(ISD::FADD, dl, MVT::f32, t4,
03839                              getF32Constant(DAG, 0x3c1d8c17, dl));
03840     SDValue t6 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t5, X);
03841     SDValue t7 = DAG.getNode(ISD::FADD, dl, MVT::f32, t6,
03842                              getF32Constant(DAG, 0x3d634a1d, dl));
03843     SDValue t8 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t7, X);
03844     SDValue t9 = DAG.getNode(ISD::FADD, dl, MVT::f32, t8,
03845                              getF32Constant(DAG, 0x3e75fe14, dl));
03846     SDValue t10 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t9, X);
03847     SDValue t11 = DAG.getNode(ISD::FADD, dl, MVT::f32, t10,
03848                               getF32Constant(DAG, 0x3f317234, dl));
03849     SDValue t12 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t11, X);
03850     TwoToFractionalPartOfX = DAG.getNode(ISD::FADD, dl, MVT::f32, t12,
03851                                          getF32Constant(DAG, 0x3f800000, dl));
03852   }
03853 
03854   // Add the exponent into the result in integer domain.
03855   SDValue t13 = DAG.getNode(ISD::BITCAST, dl, MVT::i32, TwoToFractionalPartOfX);
03856   return DAG.getNode(ISD::BITCAST, dl, MVT::f32,
03857                      DAG.getNode(ISD::ADD, dl, MVT::i32, t13, IntegerPartOfX));
03858 }
03859 
03860 /// expandExp - Lower an exp intrinsic. Handles the special sequences for
03861 /// limited-precision mode.
03862 static SDValue expandExp(SDLoc dl, SDValue Op, SelectionDAG &DAG,
03863                          const TargetLowering &TLI) {
03864   if (Op.getValueType() == MVT::f32 &&
03865       LimitFloatPrecision > 0 && LimitFloatPrecision <= 18) {
03866 
03867     // Put the exponent in the right bit position for later addition to the
03868     // final result:
03869     //
03870     //   #define LOG2OFe 1.4426950f
03871     //   t0 = Op * LOG2OFe
03872 
03873     // TODO: What fast-math-flags should be set here?
03874     SDValue t0 = DAG.getNode(ISD::FMUL, dl, MVT::f32, Op,
03875                              getF32Constant(DAG, 0x3fb8aa3b, dl));
03876     return getLimitedPrecisionExp2(t0, dl, DAG);
03877   }
03878 
03879   // No special expansion.
03880   return DAG.getNode(ISD::FEXP, dl, Op.getValueType(), Op);
03881 }
03882 
03883 /// expandLog - Lower a log intrinsic. Handles the special sequences for
03884 /// limited-precision mode.
03885 static SDValue expandLog(SDLoc dl, SDValue Op, SelectionDAG &DAG,
03886                          const TargetLowering &TLI) {
03887 
03888   // TODO: What fast-math-flags should be set on the floating-point nodes?
03889 
03890   if (Op.getValueType() == MVT::f32 &&
03891       LimitFloatPrecision > 0 && LimitFloatPrecision <= 18) {
03892     SDValue Op1 = DAG.getNode(ISD::BITCAST, dl, MVT::i32, Op);
03893 
03894     // Scale the exponent by log(2) [0.69314718f].
03895     SDValue Exp = GetExponent(DAG, Op1, TLI, dl);
03896     SDValue LogOfExponent = DAG.getNode(ISD::FMUL, dl, MVT::f32, Exp,
03897                                         getF32Constant(DAG, 0x3f317218, dl));
03898 
03899     // Get the significand and build it into a floating-point number with
03900     // exponent of 1.
03901     SDValue X = GetSignificand(DAG, Op1, dl);
03902 
03903     SDValue LogOfMantissa;
03904     if (LimitFloatPrecision <= 6) {
03905       // For floating-point precision of 6:
03906       //
03907       //   LogofMantissa =
03908       //     -1.1609546f +
03909       //       (1.4034025f - 0.23903021f * x) * x;
03910       //
03911       // error 0.0034276066, which is better than 8 bits
03912       SDValue t0 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X,
03913                                getF32Constant(DAG, 0xbe74c456, dl));
03914       SDValue t1 = DAG.getNode(ISD::FADD, dl, MVT::f32, t0,
03915                                getF32Constant(DAG, 0x3fb3a2b1, dl));
03916       SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t1, X);
03917       LogOfMantissa = DAG.getNode(ISD::FSUB, dl, MVT::f32, t2,
03918                                   getF32Constant(DAG, 0x3f949a29, dl));
03919     } else if (LimitFloatPrecision <= 12) {
03920       // For floating-point precision of 12:
03921       //
03922       //   LogOfMantissa =
03923       //     -1.7417939f +
03924       //       (2.8212026f +
03925       //         (-1.4699568f +
03926       //           (0.44717955f - 0.56570851e-1f * x) * x) * x) * x;
03927       //
03928       // error 0.000061011436, which is 14 bits
03929       SDValue t0 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X,
03930                                getF32Constant(DAG, 0xbd67b6d6, dl));
03931       SDValue t1 = DAG.getNode(ISD::FADD, dl, MVT::f32, t0,
03932                                getF32Constant(DAG, 0x3ee4f4b8, dl));
03933       SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t1, X);
03934       SDValue t3 = DAG.getNode(ISD::FSUB, dl, MVT::f32, t2,
03935                                getF32Constant(DAG, 0x3fbc278b, dl));
03936       SDValue t4 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t3, X);
03937       SDValue t5 = DAG.getNode(ISD::FADD, dl, MVT::f32, t4,
03938                                getF32Constant(DAG, 0x40348e95, dl));
03939       SDValue t6 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t5, X);
03940       LogOfMantissa = DAG.getNode(ISD::FSUB, dl, MVT::f32, t6,
03941                                   getF32Constant(DAG, 0x3fdef31a, dl));
03942     } else { // LimitFloatPrecision <= 18
03943       // For floating-point precision of 18:
03944       //
03945       //   LogOfMantissa =
03946       //     -2.1072184f +
03947       //       (4.2372794f +
03948       //         (-3.7029485f +
03949       //           (2.2781945f +
03950       //             (-0.87823314f +
03951       //               (0.19073739f - 0.17809712e-1f * x) * x) * x) * x) * x)*x;
03952       //
03953       // error 0.0000023660568, which is better than 18 bits
03954       SDValue t0 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X,
03955                                getF32Constant(DAG, 0xbc91e5ac, dl));
03956       SDValue t1 = DAG.getNode(ISD::FADD, dl, MVT::f32, t0,
03957                                getF32Constant(DAG, 0x3e4350aa, dl));
03958       SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t1, X);
03959       SDValue t3 = DAG.getNode(ISD::FSUB, dl, MVT::f32, t2,
03960                                getF32Constant(DAG, 0x3f60d3e3, dl));
03961       SDValue t4 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t3, X);
03962       SDValue t5 = DAG.getNode(ISD::FADD, dl, MVT::f32, t4,
03963                                getF32Constant(DAG, 0x4011cdf0, dl));
03964       SDValue t6 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t5, X);
03965       SDValue t7 = DAG.getNode(ISD::FSUB, dl, MVT::f32, t6,
03966                                getF32Constant(DAG, 0x406cfd1c, dl));
03967       SDValue t8 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t7, X);
03968       SDValue t9 = DAG.getNode(ISD::FADD, dl, MVT::f32, t8,
03969                                getF32Constant(DAG, 0x408797cb, dl));
03970       SDValue t10 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t9, X);
03971       LogOfMantissa = DAG.getNode(ISD::FSUB, dl, MVT::f32, t10,
03972                                   getF32Constant(DAG, 0x4006dcab, dl));
03973     }
03974 
03975     return DAG.getNode(ISD::FADD, dl, MVT::f32, LogOfExponent, LogOfMantissa);
03976   }
03977 
03978   // No special expansion.
03979   return DAG.getNode(ISD::FLOG, dl, Op.getValueType(), Op);
03980 }
03981 
03982 /// expandLog2 - Lower a log2 intrinsic. Handles the special sequences for
03983 /// limited-precision mode.
03984 static SDValue expandLog2(SDLoc dl, SDValue Op, SelectionDAG &DAG,
03985                           const TargetLowering &TLI) {
03986 
03987   // TODO: What fast-math-flags should be set on the floating-point nodes?
03988 
03989   if (Op.getValueType() == MVT::f32 &&
03990       LimitFloatPrecision > 0 && LimitFloatPrecision <= 18) {
03991     SDValue Op1 = DAG.getNode(ISD::BITCAST, dl, MVT::i32, Op);
03992 
03993     // Get the exponent.
03994     SDValue LogOfExponent = GetExponent(DAG, Op1, TLI, dl);
03995 
03996     // Get the significand and build it into a floating-point number with
03997     // exponent of 1.
03998     SDValue X = GetSignificand(DAG, Op1, dl);
03999 
04000     // Different possible minimax approximations of significand in
04001     // floating-point for various degrees of accuracy over [1,2].
04002     SDValue Log2ofMantissa;
04003     if (LimitFloatPrecision <= 6) {
04004       // For floating-point precision of 6:
04005       //
04006       //   Log2ofMantissa = -1.6749035f + (2.0246817f - .34484768f * x) * x;
04007       //
04008       // error 0.0049451742, which is more than 7 bits
04009       SDValue t0 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X,
04010                                getF32Constant(DAG, 0xbeb08fe0, dl));
04011       SDValue t1 = DAG.getNode(ISD::FADD, dl, MVT::f32, t0,
04012                                getF32Constant(DAG, 0x40019463, dl));
04013       SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t1, X);
04014       Log2ofMantissa = DAG.getNode(ISD::FSUB, dl, MVT::f32, t2,
04015                                    getF32Constant(DAG, 0x3fd6633d, dl));
04016     } else if (LimitFloatPrecision <= 12) {
04017       // For floating-point precision of 12:
04018       //
04019       //   Log2ofMantissa =
04020       //     -2.51285454f +
04021       //       (4.07009056f +
04022       //         (-2.12067489f +
04023       //           (.645142248f - 0.816157886e-1f * x) * x) * x) * x;
04024       //
04025       // error 0.0000876136000, which is better than 13 bits
04026       SDValue t0 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X,
04027                                getF32Constant(DAG, 0xbda7262e, dl));
04028       SDValue t1 = DAG.getNode(ISD::FADD, dl, MVT::f32, t0,
04029                                getF32Constant(DAG, 0x3f25280b, dl));
04030       SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t1, X);
04031       SDValue t3 = DAG.getNode(ISD::FSUB, dl, MVT::f32, t2,
04032                                getF32Constant(DAG, 0x4007b923, dl));
04033       SDValue t4 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t3, X);
04034       SDValue t5 = DAG.getNode(ISD::FADD, dl, MVT::f32, t4,
04035                                getF32Constant(DAG, 0x40823e2f, dl));
04036       SDValue t6 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t5, X);
04037       Log2ofMantissa = DAG.getNode(ISD::FSUB, dl, MVT::f32, t6,
04038                                    getF32Constant(DAG, 0x4020d29c, dl));
04039     } else { // LimitFloatPrecision <= 18
04040       // For floating-point precision of 18:
04041       //
04042       //   Log2ofMantissa =
04043       //     -3.0400495f +
04044       //       (6.1129976f +
04045       //         (-5.3420409f +
04046       //           (3.2865683f +
04047       //             (-1.2669343f +
04048       //               (0.27515199f -
04049       //                 0.25691327e-1f * x) * x) * x) * x) * x) * x;
04050       //
04051       // error 0.0000018516, which is better than 18 bits
04052       SDValue t0 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X,
04053                                getF32Constant(DAG, 0xbcd2769e, dl));
04054       SDValue t1 = DAG.getNode(ISD::FADD, dl, MVT::f32, t0,
04055                                getF32Constant(DAG, 0x3e8ce0b9, dl));
04056       SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t1, X);
04057       SDValue t3 = DAG.getNode(ISD::FSUB, dl, MVT::f32, t2,
04058                                getF32Constant(DAG, 0x3fa22ae7, dl));
04059       SDValue t4 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t3, X);
04060       SDValue t5 = DAG.getNode(ISD::FADD, dl, MVT::f32, t4,
04061                                getF32Constant(DAG, 0x40525723, dl));
04062       SDValue t6 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t5, X);
04063       SDValue t7 = DAG.getNode(ISD::FSUB, dl, MVT::f32, t6,
04064                                getF32Constant(DAG, 0x40aaf200, dl));
04065       SDValue t8 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t7, X);
04066       SDValue t9 = DAG.getNode(ISD::FADD, dl, MVT::f32, t8,
04067                                getF32Constant(DAG, 0x40c39dad, dl));
04068       SDValue t10 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t9, X);
04069       Log2ofMantissa = DAG.getNode(ISD::FSUB, dl, MVT::f32, t10,
04070                                    getF32Constant(DAG, 0x4042902c, dl));
04071     }
04072 
04073     return DAG.getNode(ISD::FADD, dl, MVT::f32, LogOfExponent, Log2ofMantissa);
04074   }
04075 
04076   // No special expansion.
04077   return DAG.getNode(ISD::FLOG2, dl, Op.getValueType(), Op);
04078 }
04079 
04080 /// expandLog10 - Lower a log10 intrinsic. Handles the special sequences for
04081 /// limited-precision mode.
04082 static SDValue expandLog10(SDLoc dl, SDValue Op, SelectionDAG &DAG,
04083                            const TargetLowering &TLI) {
04084 
04085   // TODO: What fast-math-flags should be set on the floating-point nodes?
04086 
04087   if (Op.getValueType() == MVT::f32 &&
04088       LimitFloatPrecision > 0 && LimitFloatPrecision <= 18) {
04089     SDValue Op1 = DAG.getNode(ISD::BITCAST, dl, MVT::i32, Op);
04090 
04091     // Scale the exponent by log10(2) [0.30102999f].
04092     SDValue Exp = GetExponent(DAG, Op1, TLI, dl);
04093     SDValue LogOfExponent = DAG.getNode(ISD::FMUL, dl, MVT::f32, Exp,
04094                                         getF32Constant(DAG, 0x3e9a209a, dl));
04095 
04096     // Get the significand and build it into a floating-point number with
04097     // exponent of 1.
04098     SDValue X = GetSignificand(DAG, Op1, dl);
04099 
04100     SDValue Log10ofMantissa;
04101     if (LimitFloatPrecision <= 6) {
04102       // For floating-point precision of 6:
04103       //
04104       //   Log10ofMantissa =
04105       //     -0.50419619f +
04106       //       (0.60948995f - 0.10380950f * x) * x;
04107       //
04108       // error 0.0014886165, which is 6 bits
04109       SDValue t0 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X,
04110                                getF32Constant(DAG, 0xbdd49a13, dl));
04111       SDValue t1 = DAG.getNode(ISD::FADD, dl, MVT::f32, t0,
04112                                getF32Constant(DAG, 0x3f1c0789, dl));
04113       SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t1, X);
04114       Log10ofMantissa = DAG.getNode(ISD::FSUB, dl, MVT::f32, t2,
04115                                     getF32Constant(DAG, 0x3f011300, dl));
04116     } else if (LimitFloatPrecision <= 12) {
04117       // For floating-point precision of 12:
04118       //
04119       //   Log10ofMantissa =
04120       //     -0.64831180f +
04121       //       (0.91751397f +
04122       //         (-0.31664806f + 0.47637168e-1f * x) * x) * x;
04123       //
04124       // error 0.00019228036, which is better than 12 bits
04125       SDValue t0 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X,
04126                                getF32Constant(DAG, 0x3d431f31, dl));
04127       SDValue t1 = DAG.getNode(ISD::FSUB, dl, MVT::f32, t0,
04128                                getF32Constant(DAG, 0x3ea21fb2, dl));
04129       SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t1, X);
04130       SDValue t3 = DAG.getNode(ISD::FADD, dl, MVT::f32, t2,
04131                                getF32Constant(DAG, 0x3f6ae232, dl));
04132       SDValue t4 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t3, X);
04133       Log10ofMantissa = DAG.getNode(ISD::FSUB, dl, MVT::f32, t4,
04134                                     getF32Constant(DAG, 0x3f25f7c3, dl));
04135     } else { // LimitFloatPrecision <= 18
04136       // For floating-point precision of 18:
04137       //
04138       //   Log10ofMantissa =
04139       //     -0.84299375f +
04140       //       (1.5327582f +
04141       //         (-1.0688956f +
04142       //           (0.49102474f +
04143       //             (-0.12539807f + 0.13508273e-1f * x) * x) * x) * x) * x;
04144       //
04145       // error 0.0000037995730, which is better than 18 bits
04146       SDValue t0 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X,
04147                                getF32Constant(DAG, 0x3c5d51ce, dl));
04148       SDValue t1 = DAG.getNode(ISD::FSUB, dl, MVT::f32, t0,
04149                                getF32Constant(DAG, 0x3e00685a, dl));
04150       SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t1, X);
04151       SDValue t3 = DAG.getNode(ISD::FADD, dl, MVT::f32, t2,
04152                                getF32Constant(DAG, 0x3efb6798, dl));
04153       SDValue t4 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t3, X);
04154       SDValue t5 = DAG.getNode(ISD::FSUB, dl, MVT::f32, t4,
04155                                getF32Constant(DAG, 0x3f88d192, dl));
04156       SDValue t6 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t5, X);
04157       SDValue t7 = DAG.getNode(ISD::FADD, dl, MVT::f32, t6,
04158                                getF32Constant(DAG, 0x3fc4316c, dl));
04159       SDValue t8 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t7, X);
04160       Log10ofMantissa = DAG.getNode(ISD::FSUB, dl, MVT::f32, t8,
04161                                     getF32Constant(DAG, 0x3f57ce70, dl));
04162     }
04163 
04164     return DAG.getNode(ISD::FADD, dl, MVT::f32, LogOfExponent, Log10ofMantissa);
04165   }
04166 
04167   // No special expansion.
04168   return DAG.getNode(ISD::FLOG10, dl, Op.getValueType(), Op);
04169 }
04170 
04171 /// expandExp2 - Lower an exp2 intrinsic. Handles the special sequences for
04172 /// limited-precision mode.
04173 static SDValue expandExp2(SDLoc dl, SDValue Op, SelectionDAG &DAG,
04174                           const TargetLowering &TLI) {
04175   if (Op.getValueType() == MVT::f32 &&
04176       LimitFloatPrecision > 0 && LimitFloatPrecision <= 18)
04177     return getLimitedPrecisionExp2(Op, dl, DAG);
04178 
04179   // No special expansion.
04180   return DAG.getNode(ISD::FEXP2, dl, Op.getValueType(), Op);
04181 }
04182 
04183 /// visitPow - Lower a pow intrinsic. Handles the special sequences for
04184 /// limited-precision mode with x == 10.0f.
04185 static SDValue expandPow(SDLoc dl, SDValue LHS, SDValue RHS,
04186                          SelectionDAG &DAG, const TargetLowering &TLI) {
04187   bool IsExp10 = false;
04188   if (LHS.getValueType() == MVT::f32 && RHS.getValueType() == MVT::f32 &&
04189       LimitFloatPrecision > 0 && LimitFloatPrecision <= 18) {
04190     if (ConstantFPSDNode *LHSC = dyn_cast<ConstantFPSDNode>(LHS)) {
04191       APFloat Ten(10.0f);
04192       IsExp10 = LHSC->isExactlyValue(Ten);
04193     }
04194   }
04195 
04196   // TODO: What fast-math-flags should be set on the FMUL node?
04197   if (IsExp10) {
04198     // Put the exponent in the right bit position for later addition to the
04199     // final result:
04200     //
04201     //   #define LOG2OF10 3.3219281f
04202     //   t0 = Op * LOG2OF10;
04203     SDValue t0 = DAG.getNode(ISD::FMUL, dl, MVT::f32, RHS,
04204                              getF32Constant(DAG, 0x40549a78, dl));
04205     return getLimitedPrecisionExp2(t0, dl, DAG);
04206   }
04207 
04208   // No special expansion.
04209   return DAG.getNode(ISD::FPOW, dl, LHS.getValueType(), LHS, RHS);
04210 }
04211 
04212 
04213 /// ExpandPowI - Expand a llvm.powi intrinsic.
04214 static SDValue ExpandPowI(SDLoc DL, SDValue LHS, SDValue RHS,
04215                           SelectionDAG &DAG) {
04216   // If RHS is a constant, we can expand this out to a multiplication tree,
04217   // otherwise we end up lowering to a call to __powidf2 (for example).  When
04218   // optimizing for size, we only want to do this if the expansion would produce
04219   // a small number of multiplies, otherwise we do the full expansion.
04220   if (ConstantSDNode *RHSC = dyn_cast<ConstantSDNode>(RHS)) {
04221     // Get the exponent as a positive value.
04222     unsigned Val = RHSC->getSExtValue();
04223     if ((int)Val < 0) Val = -Val;
04224 
04225     // powi(x, 0) -> 1.0
04226     if (Val == 0)
04227       return DAG.getConstantFP(1.0, DL, LHS.getValueType());
04228 
04229     const Function *F = DAG.getMachineFunction().getFunction();
04230     if (!F->optForSize() ||
04231         // If optimizing for size, don't insert too many multiplies.
04232         // This inserts up to 5 multiplies.
04233         countPopulation(Val) + Log2_32(Val) < 7) {
04234       // We use the simple binary decomposition method to generate the multiply
04235       // sequence.  There are more optimal ways to do this (for example,
04236       // powi(x,15) generates one more multiply than it should), but this has
04237       // the benefit of being both really simple and much better than a libcall.
04238       SDValue Res;  // Logically starts equal to 1.0
04239       SDValue CurSquare = LHS;
04240       // TODO: Intrinsics should have fast-math-flags that propagate to these
04241       // nodes.
04242       while (Val) {
04243         if (Val & 1) {
04244           if (Res.getNode())
04245             Res = DAG.getNode(ISD::FMUL, DL,Res.getValueType(), Res, CurSquare);
04246           else
04247             Res = CurSquare;  // 1.0*CurSquare.
04248         }
04249 
04250         CurSquare = DAG.getNode(ISD::FMUL, DL, CurSquare.getValueType(),
04251                                 CurSquare, CurSquare);
04252         Val >>= 1;
04253       }
04254 
04255       // If the original was negative, invert the result, producing 1/(x*x*x).
04256       if (RHSC->getSExtValue() < 0)
04257         Res = DAG.getNode(ISD::FDIV, DL, LHS.getValueType(),
04258                           DAG.getConstantFP(1.0, DL, LHS.getValueType()), Res);
04259       return Res;
04260     }
04261   }
04262 
04263   // Otherwise, expand to a libcall.
04264   return DAG.getNode(ISD::FPOWI, DL, LHS.getValueType(), LHS, RHS);
04265 }
04266 
04267 // getUnderlyingArgReg - Find underlying register used for a truncated or
04268 // bitcasted argument.
04269 static unsigned getUnderlyingArgReg(const SDValue &N) {
04270   switch (N.getOpcode()) {
04271   case ISD::CopyFromReg:
04272     return cast<RegisterSDNode>(N.getOperand(1))->getReg();
04273   case ISD::BITCAST:
04274   case ISD::AssertZext:
04275   case ISD::AssertSext:
04276   case ISD::TRUNCATE:
04277     return getUnderlyingArgReg(N.getOperand(0));
04278   default:
04279     return 0;
04280   }
04281 }
04282 
04283 /// EmitFuncArgumentDbgValue - If the DbgValueInst is a dbg_value of a function
04284 /// argument, create the corresponding DBG_VALUE machine instruction for it now.
04285 /// At the end of instruction selection, they will be inserted to the entry BB.
04286 bool SelectionDAGBuilder::EmitFuncArgumentDbgValue(
04287     const Value *V, DILocalVariable *Variable, DIExpression *Expr,
04288     DILocation *DL, int64_t Offset, bool IsIndirect, const SDValue &N) {
04289   const Argument *Arg = dyn_cast<Argument>(V);
04290   if (!Arg)
04291     return false;
04292 
04293   MachineFunction &MF = DAG.getMachineFunction();
04294   const TargetInstrInfo *TII = DAG.getSubtarget().getInstrInfo();
04295 
04296   // Ignore inlined function arguments here.
04297   //
04298   // FIXME: Should we be checking DL->inlinedAt() to determine this?
04299   if (!Variable->getScope()->getSubprogram()->describes(MF.getFunction()))
04300     return false;
04301 
04302   Optional<MachineOperand> Op;
04303   // Some arguments' frame index is recorded during argument lowering.
04304   if (int FI = FuncInfo.getArgumentFrameIndex(Arg))
04305     Op = MachineOperand::CreateFI(FI);
04306 
04307   if (!Op && N.getNode()) {
04308     unsigned Reg = getUnderlyingArgReg(N);
04309     if (Reg && TargetRegisterInfo::isVirtualRegister(Reg)) {
04310       MachineRegisterInfo &RegInfo = MF.getRegInfo();
04311       unsigned PR = RegInfo.getLiveInPhysReg(Reg);
04312       if (PR)
04313         Reg = PR;
04314     }
04315     if (Reg)
04316       Op = MachineOperand::CreateReg(Reg, false);
04317   }
04318 
04319   if (!Op) {
04320     // Check if ValueMap has reg number.
04321     DenseMap<const Value *, unsigned>::iterator VMI = FuncInfo.ValueMap.find(V);
04322     if (VMI != FuncInfo.ValueMap.end())
04323       Op = MachineOperand::CreateReg(VMI->second, false);
04324   }
04325 
04326   if (!Op && N.getNode())
04327     // Check if frame index is available.
04328     if (LoadSDNode *LNode = dyn_cast<LoadSDNode>(N.getNode()))
04329       if (FrameIndexSDNode *FINode =
04330           dyn_cast<FrameIndexSDNode>(LNode->getBasePtr().getNode()))
04331         Op = MachineOperand::CreateFI(FINode->getIndex());
04332 
04333   if (!Op)
04334     return false;
04335 
04336   assert(Variable->isValidLocationForIntrinsic(DL) &&
04337          "Expected inlined-at fields to agree");
04338   if (Op->isReg())
04339     FuncInfo.ArgDbgValues.push_back(
04340         BuildMI(MF, DL, TII->get(TargetOpcode::DBG_VALUE), IsIndirect,
04341                 Op->getReg(), Offset, Variable, Expr));
04342   else
04343     FuncInfo.ArgDbgValues.push_back(
04344         BuildMI(MF, DL, TII->get(TargetOpcode::DBG_VALUE))
04345             .addOperand(*Op)
04346             .addImm(Offset)
04347             .addMetadata(Variable)
04348             .addMetadata(Expr));
04349 
04350   return true;
04351 }
04352 
04353 // VisualStudio defines setjmp as _setjmp
04354 #if defined(_MSC_VER) && defined(setjmp) && \
04355                          !defined(setjmp_undefined_for_msvc)
04356 #  pragma push_macro("setjmp")
04357 #  undef setjmp
04358 #  define setjmp_undefined_for_msvc
04359 #endif
04360 
04361 /// visitIntrinsicCall - Lower the call to the specified intrinsic function.  If
04362 /// we want to emit this as a call to a named external function, return the name
04363 /// otherwise lower it and return null.
04364 const char *
04365 SelectionDAGBuilder::visitIntrinsicCall(const CallInst &I, unsigned Intrinsic) {
04366   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
04367   SDLoc sdl = getCurSDLoc();
04368   DebugLoc dl = getCurDebugLoc();
04369   SDValue Res;
04370 
04371   switch (Intrinsic) {
04372   default:
04373     // By default, turn this into a target intrinsic node.
04374     visitTargetIntrinsic(I, Intrinsic);
04375     return nullptr;
04376   case Intrinsic::vastart:  visitVAStart(I); return nullptr;
04377   case Intrinsic::vaend:    visitVAEnd(I); return nullptr;
04378   case Intrinsic::vacopy:   visitVACopy(I); return nullptr;
04379   case Intrinsic::returnaddress:
04380     setValue(&I, DAG.getNode(ISD::RETURNADDR, sdl,
04381                              TLI.getPointerTy(DAG.getDataLayout()),
04382                              getValue(I.getArgOperand(0))));
04383     return nullptr;
04384   case Intrinsic::frameaddress:
04385     setValue(&I, DAG.getNode(ISD::FRAMEADDR, sdl,
04386                              TLI.getPointerTy(DAG.getDataLayout()),
04387                              getValue(I.getArgOperand(0))));
04388     return nullptr;
04389   case Intrinsic::read_register: {
04390     Value *Reg = I.getArgOperand(0);
04391     SDValue Chain = getRoot();
04392     SDValue RegName =
04393         DAG.getMDNode(cast<MDNode>(cast<MetadataAsValue>(Reg)->getMetadata()));
04394     EVT VT = TLI.getValueType(DAG.getDataLayout(), I.getType());
04395     Res = DAG.getNode(ISD::READ_REGISTER, sdl,
04396       DAG.getVTList(VT, MVT::Other), Chain, RegName);
04397     setValue(&I, Res);
04398     DAG.setRoot(Res.getValue(1));
04399     return nullptr;
04400   }
04401   case Intrinsic::write_register: {
04402     Value *Reg = I.getArgOperand(0);
04403     Value *RegValue = I.getArgOperand(1);
04404     SDValue Chain = getRoot();
04405     SDValue RegName =
04406         DAG.getMDNode(cast<MDNode>(cast<MetadataAsValue>(Reg)->getMetadata()));
04407     DAG.setRoot(DAG.getNode(ISD::WRITE_REGISTER, sdl, MVT::Other, Chain,
04408                             RegName, getValue(RegValue)));
04409     return nullptr;
04410   }
04411   case Intrinsic::setjmp:
04412     return &"_setjmp"[!TLI.usesUnderscoreSetJmp()];
04413   case Intrinsic::longjmp:
04414     return &"_longjmp"[!TLI.usesUnderscoreLongJmp()];
04415   case Intrinsic::memcpy: {
04416     SDValue Op1 = getValue(I.getArgOperand(0));
04417     SDValue Op2 = getValue(I.getArgOperand(1));
04418     SDValue Op3 = getValue(I.getArgOperand(2));
04419     unsigned Align = cast<ConstantInt>(I.getArgOperand(3))->getZExtValue();
04420     if (!Align)
04421       Align = 1; // @llvm.memcpy defines 0 and 1 to both mean no alignment.
04422     bool isVol = cast<ConstantInt>(I.getArgOperand(4))->getZExtValue();
04423     bool isTC = I.isTailCall() && isInTailCallPosition(&I, DAG.getTarget());
04424     SDValue MC = DAG.getMemcpy(getRoot(), sdl, Op1, Op2, Op3, Align, isVol,
04425                                false, isTC,
04426                                MachinePointerInfo(I.getArgOperand(0)),
04427                                MachinePointerInfo(I.getArgOperand(1)));
04428     updateDAGForMaybeTailCall(MC);
04429     return nullptr;
04430   }
04431   case Intrinsic::memset: {
04432     SDValue Op1 = getValue(I.getArgOperand(0));
04433     SDValue Op2 = getValue(I.getArgOperand(1));
04434     SDValue Op3 = getValue(I.getArgOperand(2));
04435     unsigned Align = cast<ConstantInt>(I.getArgOperand(3))->getZExtValue();
04436     if (!Align)
04437       Align = 1; // @llvm.memset defines 0 and 1 to both mean no alignment.
04438     bool isVol = cast<ConstantInt>(I.getArgOperand(4))->getZExtValue();
04439     bool isTC = I.isTailCall() && isInTailCallPosition(&I, DAG.getTarget());
04440     SDValue MS = DAG.getMemset(getRoot(), sdl, Op1, Op2, Op3, Align, isVol,
04441                                isTC, MachinePointerInfo(I.getArgOperand(0)));
04442     updateDAGForMaybeTailCall(MS);
04443     return nullptr;
04444   }
04445   case Intrinsic::memmove: {
04446     SDValue Op1 = getValue(I.getArgOperand(0));
04447     SDValue Op2 = getValue(I.getArgOperand(1));
04448     SDValue Op3 = getValue(I.getArgOperand(2));
04449     unsigned Align = cast<ConstantInt>(I.getArgOperand(3))->getZExtValue();
04450     if (!Align)
04451       Align = 1; // @llvm.memmove defines 0 and 1 to both mean no alignment.
04452     bool isVol = cast<ConstantInt>(I.getArgOperand(4))->getZExtValue();
04453     bool isTC = I.isTailCall() && isInTailCallPosition(&I, DAG.getTarget());
04454     SDValue MM = DAG.getMemmove(getRoot(), sdl, Op1, Op2, Op3, Align, isVol,
04455                                 isTC, MachinePointerInfo(I.getArgOperand(0)),
04456                                 MachinePointerInfo(I.getArgOperand(1)));
04457     updateDAGForMaybeTailCall(MM);
04458     return nullptr;
04459   }
04460   case Intrinsic::dbg_declare: {
04461     const DbgDeclareInst &DI = cast<DbgDeclareInst>(I);
04462     DILocalVariable *Variable = DI.getVariable();
04463     DIExpression *Expression = DI.getExpression();
04464     const Value *Address = DI.getAddress();
04465     assert(Variable && "Missing variable");
04466     if (!Address) {
04467       DEBUG(dbgs() << "Dropping debug info for " << DI << "\n");
04468       return nullptr;
04469     }
04470 
04471     // Check if address has undef value.
04472     if (isa<UndefValue>(Address) ||
04473         (Address->use_empty() && !isa<Argument>(Address))) {
04474       DEBUG(dbgs() << "Dropping debug info for " << DI << "\n");
04475       return nullptr;
04476     }
04477 
04478     SDValue &N = NodeMap[Address];
04479     if (!N.getNode() && isa<Argument>(Address))
04480       // Check unused arguments map.
04481       N = UnusedArgNodeMap[Address];
04482     SDDbgValue *SDV;
04483     if (N.getNode()) {
04484       if (const BitCastInst *BCI = dyn_cast<BitCastInst>(Address))
04485         Address = BCI->getOperand(0);
04486       // Parameters are handled specially.
04487       bool isParameter = Variable->isParameter() || isa<Argument>(Address);
04488       auto FINode = dyn_cast<FrameIndexSDNode>(N.getNode());
04489       if (isParameter && FINode) {
04490         // Byval parameter. We have a frame index at this point.
04491         SDV = DAG.getFrameIndexDbgValue(Variable, Expression,
04492                                         FINode->getIndex(), 0, dl, SDNodeOrder);
04493       } else if (isa<Argument>(Address)) {
04494         // Address is an argument, so try to emit its dbg value using
04495         // virtual register info from the FuncInfo.ValueMap.
04496         EmitFuncArgumentDbgValue(Address, Variable, Expression, dl, 0, false,
04497                                  N);
04498         return nullptr;
04499       } else {
04500         SDV = DAG.getDbgValue(Variable, Expression, N.getNode(), N.getResNo(),
04501                               true, 0, dl, SDNodeOrder);
04502       }
04503       DAG.AddDbgValue(SDV, N.getNode(), isParameter);
04504     } else {
04505       // If Address is an argument then try to emit its dbg value using
04506       // virtual register info from the FuncInfo.ValueMap.
04507       if (!EmitFuncArgumentDbgValue(Address, Variable, Expression, dl, 0, false,
04508                                     N)) {
04509         // If variable is pinned by a alloca in dominating bb then
04510         // use StaticAllocaMap.
04511         if (const AllocaInst *AI = dyn_cast<AllocaInst>(Address)) {
04512           if (AI->getParent() != DI.getParent()) {
04513             DenseMap<const AllocaInst*, int>::iterator SI =
04514               FuncInfo.StaticAllocaMap.find(AI);
04515             if (SI != FuncInfo.StaticAllocaMap.end()) {
04516               SDV = DAG.getFrameIndexDbgValue(Variable, Expression, SI->second,
04517                                               0, dl, SDNodeOrder);
04518               DAG.AddDbgValue(SDV, nullptr, false);
04519               return nullptr;
04520             }
04521           }
04522         }
04523         DEBUG(dbgs() << "Dropping debug info for " << DI << "\n");
04524       }
04525     }
04526     return nullptr;
04527   }
04528   case Intrinsic::dbg_value: {
04529     const DbgValueInst &DI = cast<DbgValueInst>(I);
04530     assert(DI.getVariable() && "Missing variable");
04531 
04532     DILocalVariable *Variable = DI.getVariable();
04533     DIExpression *Expression = DI.getExpression();
04534     uint64_t Offset = DI.getOffset();
04535     const Value *V = DI.getValue();
04536     if (!V)
04537       return nullptr;
04538 
04539     SDDbgValue *SDV;
04540     if (isa<ConstantInt>(V) || isa<ConstantFP>(V) || isa<UndefValue>(V)) {
04541       SDV = DAG.getConstantDbgValue(Variable, Expression, V, Offset, dl,
04542                                     SDNodeOrder);
04543       DAG.AddDbgValue(SDV, nullptr, false);
04544     } else {
04545       // Do not use getValue() in here; we don't want to generate code at
04546       // this point if it hasn't been done yet.
04547       SDValue N = NodeMap[V];
04548       if (!N.getNode() && isa<Argument>(V))
04549         // Check unused arguments map.
04550         N = UnusedArgNodeMap[V];
04551       if (N.getNode()) {
04552         if (!EmitFuncArgumentDbgValue(V, Variable, Expression, dl, Offset,
04553                                       false, N)) {
04554           SDV = DAG.getDbgValue(Variable, Expression, N.getNode(), N.getResNo(),
04555                                 false, Offset, dl, SDNodeOrder);
04556           DAG.AddDbgValue(SDV, N.getNode(), false);
04557         }
04558       } else if (!V->use_empty() ) {
04559         // Do not call getValue(V) yet, as we don't want to generate code.
04560         // Remember it for later.
04561         DanglingDebugInfo DDI(&DI, dl, SDNodeOrder);
04562         DanglingDebugInfoMap[V] = DDI;
04563       } else {
04564         // We may expand this to cover more cases.  One case where we have no
04565         // data available is an unreferenced parameter.
04566         DEBUG(dbgs() << "Dropping debug info for " << DI << "\n");
04567       }
04568     }
04569 
04570     // Build a debug info table entry.
04571     if (const BitCastInst *BCI = dyn_cast<BitCastInst>(V))
04572       V = BCI->getOperand(0);
04573     const AllocaInst *AI = dyn_cast<AllocaInst>(V);
04574     // Don't handle byval struct arguments or VLAs, for example.
04575     if (!AI) {
04576       DEBUG(dbgs() << "Dropping debug location info for:\n  " << DI << "\n");
04577       DEBUG(dbgs() << "  Last seen at:\n    " << *V << "\n");
04578       return nullptr;
04579     }
04580     DenseMap<const AllocaInst*, int>::iterator SI =
04581       FuncInfo.StaticAllocaMap.find(AI);
04582     if (SI == FuncInfo.StaticAllocaMap.end())
04583       return nullptr; // VLAs.
04584     return nullptr;
04585   }
04586 
04587   case Intrinsic::eh_typeid_for: {
04588     // Find the type id for the given typeinfo.
04589     GlobalValue *GV = ExtractTypeInfo(I.getArgOperand(0));
04590     unsigned TypeID = DAG.getMachineFunction().getMMI().getTypeIDFor(GV);
04591     Res = DAG.getConstant(TypeID, sdl, MVT::i32);
04592     setValue(&I, Res);
04593     return nullptr;
04594   }
04595 
04596   case Intrinsic::eh_return_i32:
04597   case Intrinsic::eh_return_i64:
04598     DAG.getMachineFunction().getMMI().setCallsEHReturn(true);
04599     DAG.setRoot(DAG.getNode(ISD::EH_RETURN, sdl,
04600                             MVT::Other,
04601                             getControlRoot(),
04602                             getValue(I.getArgOperand(0)),
04603                             getValue(I.getArgOperand(1))));
04604     return nullptr;
04605   case Intrinsic::eh_unwind_init:
04606     DAG.getMachineFunction().getMMI().setCallsUnwindInit(true);
04607     return nullptr;
04608   case Intrinsic::eh_dwarf_cfa: {
04609     SDValue CfaArg = DAG.getSExtOrTrunc(getValue(I.getArgOperand(0)), sdl,
04610                                         TLI.getPointerTy(DAG.getDataLayout()));
04611     SDValue Offset = DAG.getNode(ISD::ADD, sdl,
04612                                  CfaArg.getValueType(),
04613                                  DAG.getNode(ISD::FRAME_TO_ARGS_OFFSET, sdl,
04614                                              CfaArg.getValueType()),
04615                                  CfaArg);
04616     SDValue FA = DAG.getNode(
04617         ISD::FRAMEADDR, sdl, TLI.getPointerTy(DAG.getDataLayout()),
04618         DAG.getConstant(0, sdl, TLI.getPointerTy(DAG.getDataLayout())));
04619     setValue(&I, DAG.getNode(ISD::ADD, sdl, FA.getValueType(),
04620                              FA, Offset));
04621     return nullptr;
04622   }
04623   case Intrinsic::eh_sjlj_callsite: {
04624     MachineModuleInfo &MMI = DAG.getMachineFunction().getMMI();
04625     ConstantInt *CI = dyn_cast<ConstantInt>(I.getArgOperand(0));
04626     assert(CI && "Non-constant call site value in eh.sjlj.callsite!");
04627     assert(MMI.getCurrentCallSite() == 0 && "Overlapping call sites!");
04628 
04629     MMI.setCurrentCallSite(CI->getZExtValue());
04630     return nullptr;
04631   }
04632   case Intrinsic::eh_sjlj_functioncontext: {
04633     // Get and store the index of the function context.
04634     MachineFrameInfo *MFI = DAG.getMachineFunction().getFrameInfo();
04635     AllocaInst *FnCtx =
04636       cast<AllocaInst>(I.getArgOperand(0)->stripPointerCasts());
04637     int FI = FuncInfo.StaticAllocaMap[FnCtx];
04638     MFI->setFunctionContextIndex(FI);
04639     return nullptr;
04640   }
04641   case Intrinsic::eh_sjlj_setjmp: {
04642     SDValue Ops[2];
04643     Ops[0] = getRoot();
04644     Ops[1] = getValue(I.getArgOperand(0));
04645     SDValue Op = DAG.getNode(ISD::EH_SJLJ_SETJMP, sdl,
04646                              DAG.getVTList(MVT::i32, MVT::Other), Ops);
04647     setValue(&I, Op.getValue(0));
04648     DAG.setRoot(Op.getValue(1));
04649     return nullptr;
04650   }
04651   case Intrinsic::eh_sjlj_longjmp: {
04652     DAG.setRoot(DAG.getNode(ISD::EH_SJLJ_LONGJMP, sdl, MVT::Other,
04653                             getRoot(), getValue(I.getArgOperand(0))));
04654     return nullptr;
04655   }
04656   case Intrinsic::eh_sjlj_setup_dispatch: {
04657     DAG.setRoot(DAG.getNode(ISD::EH_SJLJ_SETUP_DISPATCH, sdl, MVT::Other,
04658                             getRoot()));
04659     return nullptr;
04660   }
04661 
04662   case Intrinsic::masked_gather:
04663     visitMaskedGather(I);
04664     return nullptr;
04665   case Intrinsic::masked_load:
04666     visitMaskedLoad(I);
04667     return nullptr;
04668   case Intrinsic::masked_scatter:
04669     visitMaskedScatter(I);
04670     return nullptr;
04671   case Intrinsic::masked_store:
04672     visitMaskedStore(I);
04673     return nullptr;
04674   case Intrinsic::x86_mmx_pslli_w:
04675   case Intrinsic::x86_mmx_pslli_d:
04676   case Intrinsic::x86_mmx_pslli_q:
04677   case Intrinsic::x86_mmx_psrli_w:
04678   case Intrinsic::x86_mmx_psrli_d:
04679   case Intrinsic::x86_mmx_psrli_q:
04680   case Intrinsic::x86_mmx_psrai_w:
04681   case Intrinsic::x86_mmx_psrai_d: {
04682     SDValue ShAmt = getValue(I.getArgOperand(1));
04683     if (isa<ConstantSDNode>(ShAmt)) {
04684       visitTargetIntrinsic(I, Intrinsic);
04685       return nullptr;
04686     }
04687     unsigned NewIntrinsic = 0;
04688     EVT ShAmtVT = MVT::v2i32;
04689     switch (Intrinsic) {
04690     case Intrinsic::x86_mmx_pslli_w:
04691       NewIntrinsic = Intrinsic::x86_mmx_psll_w;
04692       break;
04693     case Intrinsic::x86_mmx_pslli_d:
04694       NewIntrinsic = Intrinsic::x86_mmx_psll_d;
04695       break;
04696     case Intrinsic::x86_mmx_pslli_q:
04697       NewIntrinsic = Intrinsic::x86_mmx_psll_q;
04698       break;
04699     case Intrinsic::x86_mmx_psrli_w:
04700       NewIntrinsic = Intrinsic::x86_mmx_psrl_w;
04701       break;
04702     case Intrinsic::x86_mmx_psrli_d:
04703       NewIntrinsic = Intrinsic::x86_mmx_psrl_d;
04704       break;
04705     case Intrinsic::x86_mmx_psrli_q:
04706       NewIntrinsic = Intrinsic::x86_mmx_psrl_q;
04707       break;
04708     case Intrinsic::x86_mmx_psrai_w:
04709       NewIntrinsic = Intrinsic::x86_mmx_psra_w;
04710       break;
04711     case Intrinsic::x86_mmx_psrai_d:
04712       NewIntrinsic = Intrinsic::x86_mmx_psra_d;
04713       break;
04714     default: llvm_unreachable("Impossible intrinsic");  // Can't reach here.
04715     }
04716 
04717     // The vector shift intrinsics with scalars uses 32b shift amounts but
04718     // the sse2/mmx shift instructions reads 64 bits. Set the upper 32 bits
04719     // to be zero.
04720     // We must do this early because v2i32 is not a legal type.
04721     SDValue ShOps[2];
04722     ShOps[0] = ShAmt;
04723     ShOps[1] = DAG.getConstant(0, sdl, MVT::i32);
04724     ShAmt =  DAG.getNode(ISD::BUILD_VECTOR, sdl, ShAmtVT, ShOps);
04725     EVT DestVT = TLI.getValueType(DAG.getDataLayout(), I.getType());
04726     ShAmt = DAG.getNode(ISD::BITCAST, sdl, DestVT, ShAmt);
04727     Res = DAG.getNode(ISD::INTRINSIC_WO_CHAIN, sdl, DestVT,
04728                        DAG.getConstant(NewIntrinsic, sdl, MVT::i32),
04729                        getValue(I.getArgOperand(0)), ShAmt);
04730     setValue(&I, Res);
04731     return nullptr;
04732   }
04733   case Intrinsic::convertff:
04734   case Intrinsic::convertfsi:
04735   case Intrinsic::convertfui:
04736   case Intrinsic::convertsif:
04737   case Intrinsic::convertuif:
04738   case Intrinsic::convertss:
04739   case Intrinsic::convertsu:
04740   case Intrinsic::convertus:
04741   case Intrinsic::convertuu: {
04742     ISD::CvtCode Code = ISD::CVT_INVALID;
04743     switch (Intrinsic) {
04744     default: llvm_unreachable("Impossible intrinsic");  // Can't reach here.
04745     case Intrinsic::convertff:  Code = ISD::CVT_FF; break;
04746     case Intrinsic::convertfsi: Code = ISD::CVT_FS; break;
04747     case Intrinsic::convertfui: Code = ISD::CVT_FU; break;
04748     case Intrinsic::convertsif: Code = ISD::CVT_SF; break;
04749     case Intrinsic::convertuif: Code = ISD::CVT_UF; break;
04750     case Intrinsic::convertss:  Code = ISD::CVT_SS; break;
04751     case Intrinsic::convertsu:  Code = ISD::CVT_SU; break;
04752     case Intrinsic::convertus:  Code = ISD::CVT_US; break;
04753     case Intrinsic::convertuu:  Code = ISD::CVT_UU; break;
04754     }
04755     EVT DestVT = TLI.getValueType(DAG.getDataLayout(), I.getType());
04756     const Value *Op1 = I.getArgOperand(0);
04757     Res = DAG.getConvertRndSat(DestVT, sdl, getValue(Op1),
04758                                DAG.