LLVM API Documentation

SelectionDAGBuilder.cpp
Go to the documentation of this file.
00001 //===-- SelectionDAGBuilder.cpp - Selection-DAG building ------------------===//
00002 //
00003 //                     The LLVM Compiler Infrastructure
00004 //
00005 // This file is distributed under the University of Illinois Open Source
00006 // License. See LICENSE.TXT for details.
00007 //
00008 //===----------------------------------------------------------------------===//
00009 //
00010 // This implements routines for translating from LLVM IR into SelectionDAG IR.
00011 //
00012 //===----------------------------------------------------------------------===//
00013 
00014 #include "SelectionDAGBuilder.h"
00015 #include "SDNodeDbgValue.h"
00016 #include "llvm/ADT/BitVector.h"
00017 #include "llvm/ADT/Optional.h"
00018 #include "llvm/ADT/SmallSet.h"
00019 #include "llvm/ADT/Statistic.h"
00020 #include "llvm/Analysis/AliasAnalysis.h"
00021 #include "llvm/Analysis/BranchProbabilityInfo.h"
00022 #include "llvm/Analysis/ConstantFolding.h"
00023 #include "llvm/Analysis/TargetLibraryInfo.h"
00024 #include "llvm/Analysis/ValueTracking.h"
00025 #include "llvm/CodeGen/Analysis.h"
00026 #include "llvm/CodeGen/FastISel.h"
00027 #include "llvm/CodeGen/FunctionLoweringInfo.h"
00028 #include "llvm/CodeGen/GCMetadata.h"
00029 #include "llvm/CodeGen/GCStrategy.h"
00030 #include "llvm/CodeGen/MachineFrameInfo.h"
00031 #include "llvm/CodeGen/MachineFunction.h"
00032 #include "llvm/CodeGen/MachineInstrBuilder.h"
00033 #include "llvm/CodeGen/MachineJumpTableInfo.h"
00034 #include "llvm/CodeGen/MachineModuleInfo.h"
00035 #include "llvm/CodeGen/MachineRegisterInfo.h"
00036 #include "llvm/CodeGen/SelectionDAG.h"
00037 #include "llvm/CodeGen/StackMaps.h"
00038 #include "llvm/IR/CallingConv.h"
00039 #include "llvm/IR/Constants.h"
00040 #include "llvm/IR/DataLayout.h"
00041 #include "llvm/IR/DebugInfo.h"
00042 #include "llvm/IR/DerivedTypes.h"
00043 #include "llvm/IR/Function.h"
00044 #include "llvm/IR/GlobalVariable.h"
00045 #include "llvm/IR/InlineAsm.h"
00046 #include "llvm/IR/Instructions.h"
00047 #include "llvm/IR/IntrinsicInst.h"
00048 #include "llvm/IR/Intrinsics.h"
00049 #include "llvm/IR/LLVMContext.h"
00050 #include "llvm/IR/Module.h"
00051 #include "llvm/IR/Statepoint.h"
00052 #include "llvm/MC/MCSymbol.h"
00053 #include "llvm/Support/CommandLine.h"
00054 #include "llvm/Support/Debug.h"
00055 #include "llvm/Support/ErrorHandling.h"
00056 #include "llvm/Support/MathExtras.h"
00057 #include "llvm/Support/raw_ostream.h"
00058 #include "llvm/Target/TargetFrameLowering.h"
00059 #include "llvm/Target/TargetInstrInfo.h"
00060 #include "llvm/Target/TargetIntrinsicInfo.h"
00061 #include "llvm/Target/TargetLowering.h"
00062 #include "llvm/Target/TargetOptions.h"
00063 #include "llvm/Target/TargetSelectionDAGInfo.h"
00064 #include "llvm/Target/TargetSubtargetInfo.h"
00065 #include <algorithm>
00066 using namespace llvm;
00067 
00068 #define DEBUG_TYPE "isel"
00069 
00070 /// LimitFloatPrecision - Generate low-precision inline sequences for
00071 /// some float libcalls (6, 8 or 12 bits).
00072 static unsigned LimitFloatPrecision;
00073 
00074 static cl::opt<unsigned, true>
00075 LimitFPPrecision("limit-float-precision",
00076                  cl::desc("Generate low-precision inline sequences "
00077                           "for some float libcalls"),
00078                  cl::location(LimitFloatPrecision),
00079                  cl::init(0));
00080 
00081 // Limit the width of DAG chains. This is important in general to prevent
00082 // prevent DAG-based analysis from blowing up. For example, alias analysis and
00083 // load clustering may not complete in reasonable time. It is difficult to
00084 // recognize and avoid this situation within each individual analysis, and
00085 // future analyses are likely to have the same behavior. Limiting DAG width is
00086 // the safe approach, and will be especially important with global DAGs.
00087 //
00088 // MaxParallelChains default is arbitrarily high to avoid affecting
00089 // optimization, but could be lowered to improve compile time. Any ld-ld-st-st
00090 // sequence over this should have been converted to llvm.memcpy by the
00091 // frontend. It easy to induce this behavior with .ll code such as:
00092 // %buffer = alloca [4096 x i8]
00093 // %data = load [4096 x i8]* %argPtr
00094 // store [4096 x i8] %data, [4096 x i8]* %buffer
00095 static const unsigned MaxParallelChains = 64;
00096 
00097 static SDValue getCopyFromPartsVector(SelectionDAG &DAG, SDLoc DL,
00098                                       const SDValue *Parts, unsigned NumParts,
00099                                       MVT PartVT, EVT ValueVT, const Value *V);
00100 
00101 /// getCopyFromParts - Create a value that contains the specified legal parts
00102 /// combined into the value they represent.  If the parts combine to a type
00103 /// larger then ValueVT then AssertOp can be used to specify whether the extra
00104 /// bits are known to be zero (ISD::AssertZext) or sign extended from ValueVT
00105 /// (ISD::AssertSext).
00106 static SDValue getCopyFromParts(SelectionDAG &DAG, SDLoc DL,
00107                                 const SDValue *Parts,
00108                                 unsigned NumParts, MVT PartVT, EVT ValueVT,
00109                                 const Value *V,
00110                                 ISD::NodeType AssertOp = ISD::DELETED_NODE) {
00111   if (ValueVT.isVector())
00112     return getCopyFromPartsVector(DAG, DL, Parts, NumParts,
00113                                   PartVT, ValueVT, V);
00114 
00115   assert(NumParts > 0 && "No parts to assemble!");
00116   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
00117   SDValue Val = Parts[0];
00118 
00119   if (NumParts > 1) {
00120     // Assemble the value from multiple parts.
00121     if (ValueVT.isInteger()) {
00122       unsigned PartBits = PartVT.getSizeInBits();
00123       unsigned ValueBits = ValueVT.getSizeInBits();
00124 
00125       // Assemble the power of 2 part.
00126       unsigned RoundParts = NumParts & (NumParts - 1) ?
00127         1 << Log2_32(NumParts) : NumParts;
00128       unsigned RoundBits = PartBits * RoundParts;
00129       EVT RoundVT = RoundBits == ValueBits ?
00130         ValueVT : EVT::getIntegerVT(*DAG.getContext(), RoundBits);
00131       SDValue Lo, Hi;
00132 
00133       EVT HalfVT = EVT::getIntegerVT(*DAG.getContext(), RoundBits/2);
00134 
00135       if (RoundParts > 2) {
00136         Lo = getCopyFromParts(DAG, DL, Parts, RoundParts / 2,
00137                               PartVT, HalfVT, V);
00138         Hi = getCopyFromParts(DAG, DL, Parts + RoundParts / 2,
00139                               RoundParts / 2, PartVT, HalfVT, V);
00140       } else {
00141         Lo = DAG.getNode(ISD::BITCAST, DL, HalfVT, Parts[0]);
00142         Hi = DAG.getNode(ISD::BITCAST, DL, HalfVT, Parts[1]);
00143       }
00144 
00145       if (TLI.isBigEndian())
00146         std::swap(Lo, Hi);
00147 
00148       Val = DAG.getNode(ISD::BUILD_PAIR, DL, RoundVT, Lo, Hi);
00149 
00150       if (RoundParts < NumParts) {
00151         // Assemble the trailing non-power-of-2 part.
00152         unsigned OddParts = NumParts - RoundParts;
00153         EVT OddVT = EVT::getIntegerVT(*DAG.getContext(), OddParts * PartBits);
00154         Hi = getCopyFromParts(DAG, DL,
00155                               Parts + RoundParts, OddParts, PartVT, OddVT, V);
00156 
00157         // Combine the round and odd parts.
00158         Lo = Val;
00159         if (TLI.isBigEndian())
00160           std::swap(Lo, Hi);
00161         EVT TotalVT = EVT::getIntegerVT(*DAG.getContext(), NumParts * PartBits);
00162         Hi = DAG.getNode(ISD::ANY_EXTEND, DL, TotalVT, Hi);
00163         Hi = DAG.getNode(ISD::SHL, DL, TotalVT, Hi,
00164                          DAG.getConstant(Lo.getValueType().getSizeInBits(),
00165                                          TLI.getPointerTy()));
00166         Lo = DAG.getNode(ISD::ZERO_EXTEND, DL, TotalVT, Lo);
00167         Val = DAG.getNode(ISD::OR, DL, TotalVT, Lo, Hi);
00168       }
00169     } else if (PartVT.isFloatingPoint()) {
00170       // FP split into multiple FP parts (for ppcf128)
00171       assert(ValueVT == EVT(MVT::ppcf128) && PartVT == MVT::f64 &&
00172              "Unexpected split");
00173       SDValue Lo, Hi;
00174       Lo = DAG.getNode(ISD::BITCAST, DL, EVT(MVT::f64), Parts[0]);
00175       Hi = DAG.getNode(ISD::BITCAST, DL, EVT(MVT::f64), Parts[1]);
00176       if (TLI.hasBigEndianPartOrdering(ValueVT))
00177         std::swap(Lo, Hi);
00178       Val = DAG.getNode(ISD::BUILD_PAIR, DL, ValueVT, Lo, Hi);
00179     } else {
00180       // FP split into integer parts (soft fp)
00181       assert(ValueVT.isFloatingPoint() && PartVT.isInteger() &&
00182              !PartVT.isVector() && "Unexpected split");
00183       EVT IntVT = EVT::getIntegerVT(*DAG.getContext(), ValueVT.getSizeInBits());
00184       Val = getCopyFromParts(DAG, DL, Parts, NumParts, PartVT, IntVT, V);
00185     }
00186   }
00187 
00188   // There is now one part, held in Val.  Correct it to match ValueVT.
00189   EVT PartEVT = Val.getValueType();
00190 
00191   if (PartEVT == ValueVT)
00192     return Val;
00193 
00194   if (PartEVT.isInteger() && ValueVT.isInteger()) {
00195     if (ValueVT.bitsLT(PartEVT)) {
00196       // For a truncate, see if we have any information to
00197       // indicate whether the truncated bits will always be
00198       // zero or sign-extension.
00199       if (AssertOp != ISD::DELETED_NODE)
00200         Val = DAG.getNode(AssertOp, DL, PartEVT, Val,
00201                           DAG.getValueType(ValueVT));
00202       return DAG.getNode(ISD::TRUNCATE, DL, ValueVT, Val);
00203     }
00204     return DAG.getNode(ISD::ANY_EXTEND, DL, ValueVT, Val);
00205   }
00206 
00207   if (PartEVT.isFloatingPoint() && ValueVT.isFloatingPoint()) {
00208     // FP_ROUND's are always exact here.
00209     if (ValueVT.bitsLT(Val.getValueType()))
00210       return DAG.getNode(ISD::FP_ROUND, DL, ValueVT, Val,
00211                          DAG.getTargetConstant(1, TLI.getPointerTy()));
00212 
00213     return DAG.getNode(ISD::FP_EXTEND, DL, ValueVT, Val);
00214   }
00215 
00216   if (PartEVT.getSizeInBits() == ValueVT.getSizeInBits())
00217     return DAG.getNode(ISD::BITCAST, DL, ValueVT, Val);
00218 
00219   llvm_unreachable("Unknown mismatch!");
00220 }
00221 
00222 static void diagnosePossiblyInvalidConstraint(LLVMContext &Ctx, const Value *V,
00223                                               const Twine &ErrMsg) {
00224   const Instruction *I = dyn_cast_or_null<Instruction>(V);
00225   if (!V)
00226     return Ctx.emitError(ErrMsg);
00227 
00228   const char *AsmError = ", possible invalid constraint for vector type";
00229   if (const CallInst *CI = dyn_cast<CallInst>(I))
00230     if (isa<InlineAsm>(CI->getCalledValue()))
00231       return Ctx.emitError(I, ErrMsg + AsmError);
00232 
00233   return Ctx.emitError(I, ErrMsg);
00234 }
00235 
00236 /// getCopyFromPartsVector - Create a value that contains the specified legal
00237 /// parts combined into the value they represent.  If the parts combine to a
00238 /// type larger then ValueVT then AssertOp can be used to specify whether the
00239 /// extra bits are known to be zero (ISD::AssertZext) or sign extended from
00240 /// ValueVT (ISD::AssertSext).
00241 static SDValue getCopyFromPartsVector(SelectionDAG &DAG, SDLoc DL,
00242                                       const SDValue *Parts, unsigned NumParts,
00243                                       MVT PartVT, EVT ValueVT, const Value *V) {
00244   assert(ValueVT.isVector() && "Not a vector value");
00245   assert(NumParts > 0 && "No parts to assemble!");
00246   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
00247   SDValue Val = Parts[0];
00248 
00249   // Handle a multi-element vector.
00250   if (NumParts > 1) {
00251     EVT IntermediateVT;
00252     MVT RegisterVT;
00253     unsigned NumIntermediates;
00254     unsigned NumRegs =
00255     TLI.getVectorTypeBreakdown(*DAG.getContext(), ValueVT, IntermediateVT,
00256                                NumIntermediates, RegisterVT);
00257     assert(NumRegs == NumParts && "Part count doesn't match vector breakdown!");
00258     NumParts = NumRegs; // Silence a compiler warning.
00259     assert(RegisterVT == PartVT && "Part type doesn't match vector breakdown!");
00260     assert(RegisterVT == Parts[0].getSimpleValueType() &&
00261            "Part type doesn't match part!");
00262 
00263     // Assemble the parts into intermediate operands.
00264     SmallVector<SDValue, 8> Ops(NumIntermediates);
00265     if (NumIntermediates == NumParts) {
00266       // If the register was not expanded, truncate or copy the value,
00267       // as appropriate.
00268       for (unsigned i = 0; i != NumParts; ++i)
00269         Ops[i] = getCopyFromParts(DAG, DL, &Parts[i], 1,
00270                                   PartVT, IntermediateVT, V);
00271     } else if (NumParts > 0) {
00272       // If the intermediate type was expanded, build the intermediate
00273       // operands from the parts.
00274       assert(NumParts % NumIntermediates == 0 &&
00275              "Must expand into a divisible number of parts!");
00276       unsigned Factor = NumParts / NumIntermediates;
00277       for (unsigned i = 0; i != NumIntermediates; ++i)
00278         Ops[i] = getCopyFromParts(DAG, DL, &Parts[i * Factor], Factor,
00279                                   PartVT, IntermediateVT, V);
00280     }
00281 
00282     // Build a vector with BUILD_VECTOR or CONCAT_VECTORS from the
00283     // intermediate operands.
00284     Val = DAG.getNode(IntermediateVT.isVector() ? ISD::CONCAT_VECTORS
00285                                                 : ISD::BUILD_VECTOR,
00286                       DL, ValueVT, Ops);
00287   }
00288 
00289   // There is now one part, held in Val.  Correct it to match ValueVT.
00290   EVT PartEVT = Val.getValueType();
00291 
00292   if (PartEVT == ValueVT)
00293     return Val;
00294 
00295   if (PartEVT.isVector()) {
00296     // If the element type of the source/dest vectors are the same, but the
00297     // parts vector has more elements than the value vector, then we have a
00298     // vector widening case (e.g. <2 x float> -> <4 x float>).  Extract the
00299     // elements we want.
00300     if (PartEVT.getVectorElementType() == ValueVT.getVectorElementType()) {
00301       assert(PartEVT.getVectorNumElements() > ValueVT.getVectorNumElements() &&
00302              "Cannot narrow, it would be a lossy transformation");
00303       return DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, ValueVT, Val,
00304                          DAG.getConstant(0, TLI.getVectorIdxTy()));
00305     }
00306 
00307     // Vector/Vector bitcast.
00308     if (ValueVT.getSizeInBits() == PartEVT.getSizeInBits())
00309       return DAG.getNode(ISD::BITCAST, DL, ValueVT, Val);
00310 
00311     assert(PartEVT.getVectorNumElements() == ValueVT.getVectorNumElements() &&
00312       "Cannot handle this kind of promotion");
00313     // Promoted vector extract
00314     bool Smaller = ValueVT.bitsLE(PartEVT);
00315     return DAG.getNode((Smaller ? ISD::TRUNCATE : ISD::ANY_EXTEND),
00316                        DL, ValueVT, Val);
00317 
00318   }
00319 
00320   // Trivial bitcast if the types are the same size and the destination
00321   // vector type is legal.
00322   if (PartEVT.getSizeInBits() == ValueVT.getSizeInBits() &&
00323       TLI.isTypeLegal(ValueVT))
00324     return DAG.getNode(ISD::BITCAST, DL, ValueVT, Val);
00325 
00326   // Handle cases such as i8 -> <1 x i1>
00327   if (ValueVT.getVectorNumElements() != 1) {
00328     diagnosePossiblyInvalidConstraint(*DAG.getContext(), V,
00329                                       "non-trivial scalar-to-vector conversion");
00330     return DAG.getUNDEF(ValueVT);
00331   }
00332 
00333   if (ValueVT.getVectorNumElements() == 1 &&
00334       ValueVT.getVectorElementType() != PartEVT) {
00335     bool Smaller = ValueVT.bitsLE(PartEVT);
00336     Val = DAG.getNode((Smaller ? ISD::TRUNCATE : ISD::ANY_EXTEND),
00337                        DL, ValueVT.getScalarType(), Val);
00338   }
00339 
00340   return DAG.getNode(ISD::BUILD_VECTOR, DL, ValueVT, Val);
00341 }
00342 
00343 static void getCopyToPartsVector(SelectionDAG &DAG, SDLoc dl,
00344                                  SDValue Val, SDValue *Parts, unsigned NumParts,
00345                                  MVT PartVT, const Value *V);
00346 
00347 /// getCopyToParts - Create a series of nodes that contain the specified value
00348 /// split into legal parts.  If the parts contain more bits than Val, then, for
00349 /// integers, ExtendKind can be used to specify how to generate the extra bits.
00350 static void getCopyToParts(SelectionDAG &DAG, SDLoc DL,
00351                            SDValue Val, SDValue *Parts, unsigned NumParts,
00352                            MVT PartVT, const Value *V,
00353                            ISD::NodeType ExtendKind = ISD::ANY_EXTEND) {
00354   EVT ValueVT = Val.getValueType();
00355 
00356   // Handle the vector case separately.
00357   if (ValueVT.isVector())
00358     return getCopyToPartsVector(DAG, DL, Val, Parts, NumParts, PartVT, V);
00359 
00360   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
00361   unsigned PartBits = PartVT.getSizeInBits();
00362   unsigned OrigNumParts = NumParts;
00363   assert(TLI.isTypeLegal(PartVT) && "Copying to an illegal type!");
00364 
00365   if (NumParts == 0)
00366     return;
00367 
00368   assert(!ValueVT.isVector() && "Vector case handled elsewhere");
00369   EVT PartEVT = PartVT;
00370   if (PartEVT == ValueVT) {
00371     assert(NumParts == 1 && "No-op copy with multiple parts!");
00372     Parts[0] = Val;
00373     return;
00374   }
00375 
00376   if (NumParts * PartBits > ValueVT.getSizeInBits()) {
00377     // If the parts cover more bits than the value has, promote the value.
00378     if (PartVT.isFloatingPoint() && ValueVT.isFloatingPoint()) {
00379       assert(NumParts == 1 && "Do not know what to promote to!");
00380       Val = DAG.getNode(ISD::FP_EXTEND, DL, PartVT, Val);
00381     } else {
00382       assert((PartVT.isInteger() || PartVT == MVT::x86mmx) &&
00383              ValueVT.isInteger() &&
00384              "Unknown mismatch!");
00385       ValueVT = EVT::getIntegerVT(*DAG.getContext(), NumParts * PartBits);
00386       Val = DAG.getNode(ExtendKind, DL, ValueVT, Val);
00387       if (PartVT == MVT::x86mmx)
00388         Val = DAG.getNode(ISD::BITCAST, DL, PartVT, Val);
00389     }
00390   } else if (PartBits == ValueVT.getSizeInBits()) {
00391     // Different types of the same size.
00392     assert(NumParts == 1 && PartEVT != ValueVT);
00393     Val = DAG.getNode(ISD::BITCAST, DL, PartVT, Val);
00394   } else if (NumParts * PartBits < ValueVT.getSizeInBits()) {
00395     // If the parts cover less bits than value has, truncate the value.
00396     assert((PartVT.isInteger() || PartVT == MVT::x86mmx) &&
00397            ValueVT.isInteger() &&
00398            "Unknown mismatch!");
00399     ValueVT = EVT::getIntegerVT(*DAG.getContext(), NumParts * PartBits);
00400     Val = DAG.getNode(ISD::TRUNCATE, DL, ValueVT, Val);
00401     if (PartVT == MVT::x86mmx)
00402       Val = DAG.getNode(ISD::BITCAST, DL, PartVT, Val);
00403   }
00404 
00405   // The value may have changed - recompute ValueVT.
00406   ValueVT = Val.getValueType();
00407   assert(NumParts * PartBits == ValueVT.getSizeInBits() &&
00408          "Failed to tile the value with PartVT!");
00409 
00410   if (NumParts == 1) {
00411     if (PartEVT != ValueVT)
00412       diagnosePossiblyInvalidConstraint(*DAG.getContext(), V,
00413                                         "scalar-to-vector conversion failed");
00414 
00415     Parts[0] = Val;
00416     return;
00417   }
00418 
00419   // Expand the value into multiple parts.
00420   if (NumParts & (NumParts - 1)) {
00421     // The number of parts is not a power of 2.  Split off and copy the tail.
00422     assert(PartVT.isInteger() && ValueVT.isInteger() &&
00423            "Do not know what to expand to!");
00424     unsigned RoundParts = 1 << Log2_32(NumParts);
00425     unsigned RoundBits = RoundParts * PartBits;
00426     unsigned OddParts = NumParts - RoundParts;
00427     SDValue OddVal = DAG.getNode(ISD::SRL, DL, ValueVT, Val,
00428                                  DAG.getIntPtrConstant(RoundBits));
00429     getCopyToParts(DAG, DL, OddVal, Parts + RoundParts, OddParts, PartVT, V);
00430 
00431     if (TLI.isBigEndian())
00432       // The odd parts were reversed by getCopyToParts - unreverse them.
00433       std::reverse(Parts + RoundParts, Parts + NumParts);
00434 
00435     NumParts = RoundParts;
00436     ValueVT = EVT::getIntegerVT(*DAG.getContext(), NumParts * PartBits);
00437     Val = DAG.getNode(ISD::TRUNCATE, DL, ValueVT, Val);
00438   }
00439 
00440   // The number of parts is a power of 2.  Repeatedly bisect the value using
00441   // EXTRACT_ELEMENT.
00442   Parts[0] = DAG.getNode(ISD::BITCAST, DL,
00443                          EVT::getIntegerVT(*DAG.getContext(),
00444                                            ValueVT.getSizeInBits()),
00445                          Val);
00446 
00447   for (unsigned StepSize = NumParts; StepSize > 1; StepSize /= 2) {
00448     for (unsigned i = 0; i < NumParts; i += StepSize) {
00449       unsigned ThisBits = StepSize * PartBits / 2;
00450       EVT ThisVT = EVT::getIntegerVT(*DAG.getContext(), ThisBits);
00451       SDValue &Part0 = Parts[i];
00452       SDValue &Part1 = Parts[i+StepSize/2];
00453 
00454       Part1 = DAG.getNode(ISD::EXTRACT_ELEMENT, DL,
00455                           ThisVT, Part0, DAG.getIntPtrConstant(1));
00456       Part0 = DAG.getNode(ISD::EXTRACT_ELEMENT, DL,
00457                           ThisVT, Part0, DAG.getIntPtrConstant(0));
00458 
00459       if (ThisBits == PartBits && ThisVT != PartVT) {
00460         Part0 = DAG.getNode(ISD::BITCAST, DL, PartVT, Part0);
00461         Part1 = DAG.getNode(ISD::BITCAST, DL, PartVT, Part1);
00462       }
00463     }
00464   }
00465 
00466   if (TLI.isBigEndian())
00467     std::reverse(Parts, Parts + OrigNumParts);
00468 }
00469 
00470 
00471 /// getCopyToPartsVector - Create a series of nodes that contain the specified
00472 /// value split into legal parts.
00473 static void getCopyToPartsVector(SelectionDAG &DAG, SDLoc DL,
00474                                  SDValue Val, SDValue *Parts, unsigned NumParts,
00475                                  MVT PartVT, const Value *V) {
00476   EVT ValueVT = Val.getValueType();
00477   assert(ValueVT.isVector() && "Not a vector");
00478   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
00479 
00480   if (NumParts == 1) {
00481     EVT PartEVT = PartVT;
00482     if (PartEVT == ValueVT) {
00483       // Nothing to do.
00484     } else if (PartVT.getSizeInBits() == ValueVT.getSizeInBits()) {
00485       // Bitconvert vector->vector case.
00486       Val = DAG.getNode(ISD::BITCAST, DL, PartVT, Val);
00487     } else if (PartVT.isVector() &&
00488                PartEVT.getVectorElementType() == ValueVT.getVectorElementType() &&
00489                PartEVT.getVectorNumElements() > ValueVT.getVectorNumElements()) {
00490       EVT ElementVT = PartVT.getVectorElementType();
00491       // Vector widening case, e.g. <2 x float> -> <4 x float>.  Shuffle in
00492       // undef elements.
00493       SmallVector<SDValue, 16> Ops;
00494       for (unsigned i = 0, e = ValueVT.getVectorNumElements(); i != e; ++i)
00495         Ops.push_back(DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL,
00496                                   ElementVT, Val, DAG.getConstant(i,
00497                                                   TLI.getVectorIdxTy())));
00498 
00499       for (unsigned i = ValueVT.getVectorNumElements(),
00500            e = PartVT.getVectorNumElements(); i != e; ++i)
00501         Ops.push_back(DAG.getUNDEF(ElementVT));
00502 
00503       Val = DAG.getNode(ISD::BUILD_VECTOR, DL, PartVT, Ops);
00504 
00505       // FIXME: Use CONCAT for 2x -> 4x.
00506 
00507       //SDValue UndefElts = DAG.getUNDEF(VectorTy);
00508       //Val = DAG.getNode(ISD::CONCAT_VECTORS, DL, PartVT, Val, UndefElts);
00509     } else if (PartVT.isVector() &&
00510                PartEVT.getVectorElementType().bitsGE(
00511                  ValueVT.getVectorElementType()) &&
00512                PartEVT.getVectorNumElements() == ValueVT.getVectorNumElements()) {
00513 
00514       // Promoted vector extract
00515       bool Smaller = PartEVT.bitsLE(ValueVT);
00516       Val = DAG.getNode((Smaller ? ISD::TRUNCATE : ISD::ANY_EXTEND),
00517                         DL, PartVT, Val);
00518     } else{
00519       // Vector -> scalar conversion.
00520       assert(ValueVT.getVectorNumElements() == 1 &&
00521              "Only trivial vector-to-scalar conversions should get here!");
00522       Val = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL,
00523                         PartVT, Val, DAG.getConstant(0, TLI.getVectorIdxTy()));
00524 
00525       bool Smaller = ValueVT.bitsLE(PartVT);
00526       Val = DAG.getNode((Smaller ? ISD::TRUNCATE : ISD::ANY_EXTEND),
00527                          DL, PartVT, Val);
00528     }
00529 
00530     Parts[0] = Val;
00531     return;
00532   }
00533 
00534   // Handle a multi-element vector.
00535   EVT IntermediateVT;
00536   MVT RegisterVT;
00537   unsigned NumIntermediates;
00538   unsigned NumRegs = TLI.getVectorTypeBreakdown(*DAG.getContext(), ValueVT,
00539                                                 IntermediateVT,
00540                                                 NumIntermediates, RegisterVT);
00541   unsigned NumElements = ValueVT.getVectorNumElements();
00542 
00543   assert(NumRegs == NumParts && "Part count doesn't match vector breakdown!");
00544   NumParts = NumRegs; // Silence a compiler warning.
00545   assert(RegisterVT == PartVT && "Part type doesn't match vector breakdown!");
00546 
00547   // Split the vector into intermediate operands.
00548   SmallVector<SDValue, 8> Ops(NumIntermediates);
00549   for (unsigned i = 0; i != NumIntermediates; ++i) {
00550     if (IntermediateVT.isVector())
00551       Ops[i] = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL,
00552                            IntermediateVT, Val,
00553                    DAG.getConstant(i * (NumElements / NumIntermediates),
00554                                    TLI.getVectorIdxTy()));
00555     else
00556       Ops[i] = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL,
00557                            IntermediateVT, Val,
00558                            DAG.getConstant(i, TLI.getVectorIdxTy()));
00559   }
00560 
00561   // Split the intermediate operands into legal parts.
00562   if (NumParts == NumIntermediates) {
00563     // If the register was not expanded, promote or copy the value,
00564     // as appropriate.
00565     for (unsigned i = 0; i != NumParts; ++i)
00566       getCopyToParts(DAG, DL, Ops[i], &Parts[i], 1, PartVT, V);
00567   } else if (NumParts > 0) {
00568     // If the intermediate type was expanded, split each the value into
00569     // legal parts.
00570     assert(NumIntermediates != 0 && "division by zero");
00571     assert(NumParts % NumIntermediates == 0 &&
00572            "Must expand into a divisible number of parts!");
00573     unsigned Factor = NumParts / NumIntermediates;
00574     for (unsigned i = 0; i != NumIntermediates; ++i)
00575       getCopyToParts(DAG, DL, Ops[i], &Parts[i*Factor], Factor, PartVT, V);
00576   }
00577 }
00578 
00579 namespace {
00580   /// RegsForValue - This struct represents the registers (physical or virtual)
00581   /// that a particular set of values is assigned, and the type information
00582   /// about the value. The most common situation is to represent one value at a
00583   /// time, but struct or array values are handled element-wise as multiple
00584   /// values.  The splitting of aggregates is performed recursively, so that we
00585   /// never have aggregate-typed registers. The values at this point do not
00586   /// necessarily have legal types, so each value may require one or more
00587   /// registers of some legal type.
00588   ///
00589   struct RegsForValue {
00590     /// ValueVTs - The value types of the values, which may not be legal, and
00591     /// may need be promoted or synthesized from one or more registers.
00592     ///
00593     SmallVector<EVT, 4> ValueVTs;
00594 
00595     /// RegVTs - The value types of the registers. This is the same size as
00596     /// ValueVTs and it records, for each value, what the type of the assigned
00597     /// register or registers are. (Individual values are never synthesized
00598     /// from more than one type of register.)
00599     ///
00600     /// With virtual registers, the contents of RegVTs is redundant with TLI's
00601     /// getRegisterType member function, however when with physical registers
00602     /// it is necessary to have a separate record of the types.
00603     ///
00604     SmallVector<MVT, 4> RegVTs;
00605 
00606     /// Regs - This list holds the registers assigned to the values.
00607     /// Each legal or promoted value requires one register, and each
00608     /// expanded value requires multiple registers.
00609     ///
00610     SmallVector<unsigned, 4> Regs;
00611 
00612     RegsForValue() {}
00613 
00614     RegsForValue(const SmallVector<unsigned, 4> &regs,
00615                  MVT regvt, EVT valuevt)
00616       : ValueVTs(1, valuevt), RegVTs(1, regvt), Regs(regs) {}
00617 
00618     RegsForValue(LLVMContext &Context, const TargetLowering &tli,
00619                  unsigned Reg, Type *Ty) {
00620       ComputeValueVTs(tli, Ty, ValueVTs);
00621 
00622       for (unsigned Value = 0, e = ValueVTs.size(); Value != e; ++Value) {
00623         EVT ValueVT = ValueVTs[Value];
00624         unsigned NumRegs = tli.getNumRegisters(Context, ValueVT);
00625         MVT RegisterVT = tli.getRegisterType(Context, ValueVT);
00626         for (unsigned i = 0; i != NumRegs; ++i)
00627           Regs.push_back(Reg + i);
00628         RegVTs.push_back(RegisterVT);
00629         Reg += NumRegs;
00630       }
00631     }
00632 
00633     /// append - Add the specified values to this one.
00634     void append(const RegsForValue &RHS) {
00635       ValueVTs.append(RHS.ValueVTs.begin(), RHS.ValueVTs.end());
00636       RegVTs.append(RHS.RegVTs.begin(), RHS.RegVTs.end());
00637       Regs.append(RHS.Regs.begin(), RHS.Regs.end());
00638     }
00639 
00640     /// getCopyFromRegs - Emit a series of CopyFromReg nodes that copies from
00641     /// this value and returns the result as a ValueVTs value.  This uses
00642     /// Chain/Flag as the input and updates them for the output Chain/Flag.
00643     /// If the Flag pointer is NULL, no flag is used.
00644     SDValue getCopyFromRegs(SelectionDAG &DAG, FunctionLoweringInfo &FuncInfo,
00645                             SDLoc dl,
00646                             SDValue &Chain, SDValue *Flag,
00647                             const Value *V = nullptr) const;
00648 
00649     /// getCopyToRegs - Emit a series of CopyToReg nodes that copies the
00650     /// specified value into the registers specified by this object.  This uses
00651     /// Chain/Flag as the input and updates them for the output Chain/Flag.
00652     /// If the Flag pointer is NULL, no flag is used.
00653     void
00654     getCopyToRegs(SDValue Val, SelectionDAG &DAG, SDLoc dl, SDValue &Chain,
00655                   SDValue *Flag, const Value *V,
00656                   ISD::NodeType PreferredExtendType = ISD::ANY_EXTEND) const;
00657 
00658     /// AddInlineAsmOperands - Add this value to the specified inlineasm node
00659     /// operand list.  This adds the code marker, matching input operand index
00660     /// (if applicable), and includes the number of values added into it.
00661     void AddInlineAsmOperands(unsigned Kind,
00662                               bool HasMatching, unsigned MatchingIdx,
00663                               SelectionDAG &DAG,
00664                               std::vector<SDValue> &Ops) const;
00665   };
00666 }
00667 
00668 /// getCopyFromRegs - Emit a series of CopyFromReg nodes that copies from
00669 /// this value and returns the result as a ValueVT value.  This uses
00670 /// Chain/Flag as the input and updates them for the output Chain/Flag.
00671 /// If the Flag pointer is NULL, no flag is used.
00672 SDValue RegsForValue::getCopyFromRegs(SelectionDAG &DAG,
00673                                       FunctionLoweringInfo &FuncInfo,
00674                                       SDLoc dl,
00675                                       SDValue &Chain, SDValue *Flag,
00676                                       const Value *V) const {
00677   // A Value with type {} or [0 x %t] needs no registers.
00678   if (ValueVTs.empty())
00679     return SDValue();
00680 
00681   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
00682 
00683   // Assemble the legal parts into the final values.
00684   SmallVector<SDValue, 4> Values(ValueVTs.size());
00685   SmallVector<SDValue, 8> Parts;
00686   for (unsigned Value = 0, Part = 0, e = ValueVTs.size(); Value != e; ++Value) {
00687     // Copy the legal parts from the registers.
00688     EVT ValueVT = ValueVTs[Value];
00689     unsigned NumRegs = TLI.getNumRegisters(*DAG.getContext(), ValueVT);
00690     MVT RegisterVT = RegVTs[Value];
00691 
00692     Parts.resize(NumRegs);
00693     for (unsigned i = 0; i != NumRegs; ++i) {
00694       SDValue P;
00695       if (!Flag) {
00696         P = DAG.getCopyFromReg(Chain, dl, Regs[Part+i], RegisterVT);
00697       } else {
00698         P = DAG.getCopyFromReg(Chain, dl, Regs[Part+i], RegisterVT, *Flag);
00699         *Flag = P.getValue(2);
00700       }
00701 
00702       Chain = P.getValue(1);
00703       Parts[i] = P;
00704 
00705       // If the source register was virtual and if we know something about it,
00706       // add an assert node.
00707       if (!TargetRegisterInfo::isVirtualRegister(Regs[Part+i]) ||
00708           !RegisterVT.isInteger() || RegisterVT.isVector())
00709         continue;
00710 
00711       const FunctionLoweringInfo::LiveOutInfo *LOI =
00712         FuncInfo.GetLiveOutRegInfo(Regs[Part+i]);
00713       if (!LOI)
00714         continue;
00715 
00716       unsigned RegSize = RegisterVT.getSizeInBits();
00717       unsigned NumSignBits = LOI->NumSignBits;
00718       unsigned NumZeroBits = LOI->KnownZero.countLeadingOnes();
00719 
00720       if (NumZeroBits == RegSize) {
00721         // The current value is a zero.
00722         // Explicitly express that as it would be easier for
00723         // optimizations to kick in.
00724         Parts[i] = DAG.getConstant(0, RegisterVT);
00725         continue;
00726       }
00727 
00728       // FIXME: We capture more information than the dag can represent.  For
00729       // now, just use the tightest assertzext/assertsext possible.
00730       bool isSExt = true;
00731       EVT FromVT(MVT::Other);
00732       if (NumSignBits == RegSize)
00733         isSExt = true, FromVT = MVT::i1;   // ASSERT SEXT 1
00734       else if (NumZeroBits >= RegSize-1)
00735         isSExt = false, FromVT = MVT::i1;  // ASSERT ZEXT 1
00736       else if (NumSignBits > RegSize-8)
00737         isSExt = true, FromVT = MVT::i8;   // ASSERT SEXT 8
00738       else if (NumZeroBits >= RegSize-8)
00739         isSExt = false, FromVT = MVT::i8;  // ASSERT ZEXT 8
00740       else if (NumSignBits > RegSize-16)
00741         isSExt = true, FromVT = MVT::i16;  // ASSERT SEXT 16
00742       else if (NumZeroBits >= RegSize-16)
00743         isSExt = false, FromVT = MVT::i16; // ASSERT ZEXT 16
00744       else if (NumSignBits > RegSize-32)
00745         isSExt = true, FromVT = MVT::i32;  // ASSERT SEXT 32
00746       else if (NumZeroBits >= RegSize-32)
00747         isSExt = false, FromVT = MVT::i32; // ASSERT ZEXT 32
00748       else
00749         continue;
00750 
00751       // Add an assertion node.
00752       assert(FromVT != MVT::Other);
00753       Parts[i] = DAG.getNode(isSExt ? ISD::AssertSext : ISD::AssertZext, dl,
00754                              RegisterVT, P, DAG.getValueType(FromVT));
00755     }
00756 
00757     Values[Value] = getCopyFromParts(DAG, dl, Parts.begin(),
00758                                      NumRegs, RegisterVT, ValueVT, V);
00759     Part += NumRegs;
00760     Parts.clear();
00761   }
00762 
00763   return DAG.getNode(ISD::MERGE_VALUES, dl, DAG.getVTList(ValueVTs), Values);
00764 }
00765 
00766 /// getCopyToRegs - Emit a series of CopyToReg nodes that copies the
00767 /// specified value into the registers specified by this object.  This uses
00768 /// Chain/Flag as the input and updates them for the output Chain/Flag.
00769 /// If the Flag pointer is NULL, no flag is used.
00770 void RegsForValue::getCopyToRegs(SDValue Val, SelectionDAG &DAG, SDLoc dl,
00771                                  SDValue &Chain, SDValue *Flag, const Value *V,
00772                                  ISD::NodeType PreferredExtendType) const {
00773   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
00774   ISD::NodeType ExtendKind = PreferredExtendType;
00775 
00776   // Get the list of the values's legal parts.
00777   unsigned NumRegs = Regs.size();
00778   SmallVector<SDValue, 8> Parts(NumRegs);
00779   for (unsigned Value = 0, Part = 0, e = ValueVTs.size(); Value != e; ++Value) {
00780     EVT ValueVT = ValueVTs[Value];
00781     unsigned NumParts = TLI.getNumRegisters(*DAG.getContext(), ValueVT);
00782     MVT RegisterVT = RegVTs[Value];
00783 
00784     if (ExtendKind == ISD::ANY_EXTEND && TLI.isZExtFree(Val, RegisterVT))
00785       ExtendKind = ISD::ZERO_EXTEND;
00786 
00787     getCopyToParts(DAG, dl, Val.getValue(Val.getResNo() + Value),
00788                    &Parts[Part], NumParts, RegisterVT, V, ExtendKind);
00789     Part += NumParts;
00790   }
00791 
00792   // Copy the parts into the registers.
00793   SmallVector<SDValue, 8> Chains(NumRegs);
00794   for (unsigned i = 0; i != NumRegs; ++i) {
00795     SDValue Part;
00796     if (!Flag) {
00797       Part = DAG.getCopyToReg(Chain, dl, Regs[i], Parts[i]);
00798     } else {
00799       Part = DAG.getCopyToReg(Chain, dl, Regs[i], Parts[i], *Flag);
00800       *Flag = Part.getValue(1);
00801     }
00802 
00803     Chains[i] = Part.getValue(0);
00804   }
00805 
00806   if (NumRegs == 1 || Flag)
00807     // If NumRegs > 1 && Flag is used then the use of the last CopyToReg is
00808     // flagged to it. That is the CopyToReg nodes and the user are considered
00809     // a single scheduling unit. If we create a TokenFactor and return it as
00810     // chain, then the TokenFactor is both a predecessor (operand) of the
00811     // user as well as a successor (the TF operands are flagged to the user).
00812     // c1, f1 = CopyToReg
00813     // c2, f2 = CopyToReg
00814     // c3     = TokenFactor c1, c2
00815     // ...
00816     //        = op c3, ..., f2
00817     Chain = Chains[NumRegs-1];
00818   else
00819     Chain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Chains);
00820 }
00821 
00822 /// AddInlineAsmOperands - Add this value to the specified inlineasm node
00823 /// operand list.  This adds the code marker and includes the number of
00824 /// values added into it.
00825 void RegsForValue::AddInlineAsmOperands(unsigned Code, bool HasMatching,
00826                                         unsigned MatchingIdx,
00827                                         SelectionDAG &DAG,
00828                                         std::vector<SDValue> &Ops) const {
00829   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
00830 
00831   unsigned Flag = InlineAsm::getFlagWord(Code, Regs.size());
00832   if (HasMatching)
00833     Flag = InlineAsm::getFlagWordForMatchingOp(Flag, MatchingIdx);
00834   else if (!Regs.empty() &&
00835            TargetRegisterInfo::isVirtualRegister(Regs.front())) {
00836     // Put the register class of the virtual registers in the flag word.  That
00837     // way, later passes can recompute register class constraints for inline
00838     // assembly as well as normal instructions.
00839     // Don't do this for tied operands that can use the regclass information
00840     // from the def.
00841     const MachineRegisterInfo &MRI = DAG.getMachineFunction().getRegInfo();
00842     const TargetRegisterClass *RC = MRI.getRegClass(Regs.front());
00843     Flag = InlineAsm::getFlagWordForRegClass(Flag, RC->getID());
00844   }
00845 
00846   SDValue Res = DAG.getTargetConstant(Flag, MVT::i32);
00847   Ops.push_back(Res);
00848 
00849   unsigned SP = TLI.getStackPointerRegisterToSaveRestore();
00850   for (unsigned Value = 0, Reg = 0, e = ValueVTs.size(); Value != e; ++Value) {
00851     unsigned NumRegs = TLI.getNumRegisters(*DAG.getContext(), ValueVTs[Value]);
00852     MVT RegisterVT = RegVTs[Value];
00853     for (unsigned i = 0; i != NumRegs; ++i) {
00854       assert(Reg < Regs.size() && "Mismatch in # registers expected");
00855       unsigned TheReg = Regs[Reg++];
00856       Ops.push_back(DAG.getRegister(TheReg, RegisterVT));
00857 
00858       if (TheReg == SP && Code == InlineAsm::Kind_Clobber) {
00859         // If we clobbered the stack pointer, MFI should know about it.
00860         assert(DAG.getMachineFunction().getFrameInfo()->
00861             hasInlineAsmWithSPAdjust());
00862       }
00863     }
00864   }
00865 }
00866 
00867 void SelectionDAGBuilder::init(GCFunctionInfo *gfi, AliasAnalysis &aa,
00868                                const TargetLibraryInfo *li) {
00869   AA = &aa;
00870   GFI = gfi;
00871   LibInfo = li;
00872   DL = DAG.getTarget().getDataLayout();
00873   Context = DAG.getContext();
00874   LPadToCallSiteMap.clear();
00875 }
00876 
00877 /// clear - Clear out the current SelectionDAG and the associated
00878 /// state and prepare this SelectionDAGBuilder object to be used
00879 /// for a new block. This doesn't clear out information about
00880 /// additional blocks that are needed to complete switch lowering
00881 /// or PHI node updating; that information is cleared out as it is
00882 /// consumed.
00883 void SelectionDAGBuilder::clear() {
00884   NodeMap.clear();
00885   UnusedArgNodeMap.clear();
00886   PendingLoads.clear();
00887   PendingExports.clear();
00888   CurInst = nullptr;
00889   HasTailCall = false;
00890   SDNodeOrder = LowestSDNodeOrder;
00891   StatepointLowering.clear();
00892 }
00893 
00894 /// clearDanglingDebugInfo - Clear the dangling debug information
00895 /// map. This function is separated from the clear so that debug
00896 /// information that is dangling in a basic block can be properly
00897 /// resolved in a different basic block. This allows the
00898 /// SelectionDAG to resolve dangling debug information attached
00899 /// to PHI nodes.
00900 void SelectionDAGBuilder::clearDanglingDebugInfo() {
00901   DanglingDebugInfoMap.clear();
00902 }
00903 
00904 /// getRoot - Return the current virtual root of the Selection DAG,
00905 /// flushing any PendingLoad items. This must be done before emitting
00906 /// a store or any other node that may need to be ordered after any
00907 /// prior load instructions.
00908 ///
00909 SDValue SelectionDAGBuilder::getRoot() {
00910   if (PendingLoads.empty())
00911     return DAG.getRoot();
00912 
00913   if (PendingLoads.size() == 1) {
00914     SDValue Root = PendingLoads[0];
00915     DAG.setRoot(Root);
00916     PendingLoads.clear();
00917     return Root;
00918   }
00919 
00920   // Otherwise, we have to make a token factor node.
00921   SDValue Root = DAG.getNode(ISD::TokenFactor, getCurSDLoc(), MVT::Other,
00922                              PendingLoads);
00923   PendingLoads.clear();
00924   DAG.setRoot(Root);
00925   return Root;
00926 }
00927 
00928 /// getControlRoot - Similar to getRoot, but instead of flushing all the
00929 /// PendingLoad items, flush all the PendingExports items. It is necessary
00930 /// to do this before emitting a terminator instruction.
00931 ///
00932 SDValue SelectionDAGBuilder::getControlRoot() {
00933   SDValue Root = DAG.getRoot();
00934 
00935   if (PendingExports.empty())
00936     return Root;
00937 
00938   // Turn all of the CopyToReg chains into one factored node.
00939   if (Root.getOpcode() != ISD::EntryToken) {
00940     unsigned i = 0, e = PendingExports.size();
00941     for (; i != e; ++i) {
00942       assert(PendingExports[i].getNode()->getNumOperands() > 1);
00943       if (PendingExports[i].getNode()->getOperand(0) == Root)
00944         break;  // Don't add the root if we already indirectly depend on it.
00945     }
00946 
00947     if (i == e)
00948       PendingExports.push_back(Root);
00949   }
00950 
00951   Root = DAG.getNode(ISD::TokenFactor, getCurSDLoc(), MVT::Other,
00952                      PendingExports);
00953   PendingExports.clear();
00954   DAG.setRoot(Root);
00955   return Root;
00956 }
00957 
00958 void SelectionDAGBuilder::visit(const Instruction &I) {
00959   // Set up outgoing PHI node register values before emitting the terminator.
00960   if (isa<TerminatorInst>(&I))
00961     HandlePHINodesInSuccessorBlocks(I.getParent());
00962 
00963   ++SDNodeOrder;
00964 
00965   CurInst = &I;
00966 
00967   visit(I.getOpcode(), I);
00968 
00969   if (!isa<TerminatorInst>(&I) && !HasTailCall)
00970     CopyToExportRegsIfNeeded(&I);
00971 
00972   CurInst = nullptr;
00973 }
00974 
00975 void SelectionDAGBuilder::visitPHI(const PHINode &) {
00976   llvm_unreachable("SelectionDAGBuilder shouldn't visit PHI nodes!");
00977 }
00978 
00979 void SelectionDAGBuilder::visit(unsigned Opcode, const User &I) {
00980   // Note: this doesn't use InstVisitor, because it has to work with
00981   // ConstantExpr's in addition to instructions.
00982   switch (Opcode) {
00983   default: llvm_unreachable("Unknown instruction type encountered!");
00984     // Build the switch statement using the Instruction.def file.
00985 #define HANDLE_INST(NUM, OPCODE, CLASS) \
00986     case Instruction::OPCODE: visit##OPCODE((const CLASS&)I); break;
00987 #include "llvm/IR/Instruction.def"
00988   }
00989 }
00990 
00991 // resolveDanglingDebugInfo - if we saw an earlier dbg_value referring to V,
00992 // generate the debug data structures now that we've seen its definition.
00993 void SelectionDAGBuilder::resolveDanglingDebugInfo(const Value *V,
00994                                                    SDValue Val) {
00995   DanglingDebugInfo &DDI = DanglingDebugInfoMap[V];
00996   if (DDI.getDI()) {
00997     const DbgValueInst *DI = DDI.getDI();
00998     DebugLoc dl = DDI.getdl();
00999     unsigned DbgSDNodeOrder = DDI.getSDNodeOrder();
01000     MDNode *Variable = DI->getVariable();
01001     MDNode *Expr = DI->getExpression();
01002     uint64_t Offset = DI->getOffset();
01003     // A dbg.value for an alloca is always indirect.
01004     bool IsIndirect = isa<AllocaInst>(V) || Offset != 0;
01005     SDDbgValue *SDV;
01006     if (Val.getNode()) {
01007       if (!EmitFuncArgumentDbgValue(V, Variable, Expr, Offset, IsIndirect,
01008                                     Val)) {
01009         SDV = DAG.getDbgValue(Variable, Expr, Val.getNode(), Val.getResNo(),
01010                               IsIndirect, Offset, dl, DbgSDNodeOrder);
01011         DAG.AddDbgValue(SDV, Val.getNode(), false);
01012       }
01013     } else
01014       DEBUG(dbgs() << "Dropping debug info for " << *DI << "\n");
01015     DanglingDebugInfoMap[V] = DanglingDebugInfo();
01016   }
01017 }
01018 
01019 /// getValue - Return an SDValue for the given Value.
01020 SDValue SelectionDAGBuilder::getValue(const Value *V) {
01021   // If we already have an SDValue for this value, use it. It's important
01022   // to do this first, so that we don't create a CopyFromReg if we already
01023   // have a regular SDValue.
01024   SDValue &N = NodeMap[V];
01025   if (N.getNode()) return N;
01026 
01027   // If there's a virtual register allocated and initialized for this
01028   // value, use it.
01029   DenseMap<const Value *, unsigned>::iterator It = FuncInfo.ValueMap.find(V);
01030   if (It != FuncInfo.ValueMap.end()) {
01031     unsigned InReg = It->second;
01032     RegsForValue RFV(*DAG.getContext(), DAG.getTargetLoweringInfo(), InReg,
01033                      V->getType());
01034     SDValue Chain = DAG.getEntryNode();
01035     N = RFV.getCopyFromRegs(DAG, FuncInfo, getCurSDLoc(), Chain, nullptr, V);
01036     resolveDanglingDebugInfo(V, N);
01037     return N;
01038   }
01039 
01040   // Otherwise create a new SDValue and remember it.
01041   SDValue Val = getValueImpl(V);
01042   NodeMap[V] = Val;
01043   resolveDanglingDebugInfo(V, Val);
01044   return Val;
01045 }
01046 
01047 /// getNonRegisterValue - Return an SDValue for the given Value, but
01048 /// don't look in FuncInfo.ValueMap for a virtual register.
01049 SDValue SelectionDAGBuilder::getNonRegisterValue(const Value *V) {
01050   // If we already have an SDValue for this value, use it.
01051   SDValue &N = NodeMap[V];
01052   if (N.getNode()) return N;
01053 
01054   // Otherwise create a new SDValue and remember it.
01055   SDValue Val = getValueImpl(V);
01056   NodeMap[V] = Val;
01057   resolveDanglingDebugInfo(V, Val);
01058   return Val;
01059 }
01060 
01061 /// getValueImpl - Helper function for getValue and getNonRegisterValue.
01062 /// Create an SDValue for the given value.
01063 SDValue SelectionDAGBuilder::getValueImpl(const Value *V) {
01064   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
01065 
01066   if (const Constant *C = dyn_cast<Constant>(V)) {
01067     EVT VT = TLI.getValueType(V->getType(), true);
01068 
01069     if (const ConstantInt *CI = dyn_cast<ConstantInt>(C))
01070       return DAG.getConstant(*CI, VT);
01071 
01072     if (const GlobalValue *GV = dyn_cast<GlobalValue>(C))
01073       return DAG.getGlobalAddress(GV, getCurSDLoc(), VT);
01074 
01075     if (isa<ConstantPointerNull>(C)) {
01076       unsigned AS = V->getType()->getPointerAddressSpace();
01077       return DAG.getConstant(0, TLI.getPointerTy(AS));
01078     }
01079 
01080     if (const ConstantFP *CFP = dyn_cast<ConstantFP>(C))
01081       return DAG.getConstantFP(*CFP, VT);
01082 
01083     if (isa<UndefValue>(C) && !V->getType()->isAggregateType())
01084       return DAG.getUNDEF(VT);
01085 
01086     if (const ConstantExpr *CE = dyn_cast<ConstantExpr>(C)) {
01087       visit(CE->getOpcode(), *CE);
01088       SDValue N1 = NodeMap[V];
01089       assert(N1.getNode() && "visit didn't populate the NodeMap!");
01090       return N1;
01091     }
01092 
01093     if (isa<ConstantStruct>(C) || isa<ConstantArray>(C)) {
01094       SmallVector<SDValue, 4> Constants;
01095       for (User::const_op_iterator OI = C->op_begin(), OE = C->op_end();
01096            OI != OE; ++OI) {
01097         SDNode *Val = getValue(*OI).getNode();
01098         // If the operand is an empty aggregate, there are no values.
01099         if (!Val) continue;
01100         // Add each leaf value from the operand to the Constants list
01101         // to form a flattened list of all the values.
01102         for (unsigned i = 0, e = Val->getNumValues(); i != e; ++i)
01103           Constants.push_back(SDValue(Val, i));
01104       }
01105 
01106       return DAG.getMergeValues(Constants, getCurSDLoc());
01107     }
01108 
01109     if (const ConstantDataSequential *CDS =
01110           dyn_cast<ConstantDataSequential>(C)) {
01111       SmallVector<SDValue, 4> Ops;
01112       for (unsigned i = 0, e = CDS->getNumElements(); i != e; ++i) {
01113         SDNode *Val = getValue(CDS->getElementAsConstant(i)).getNode();
01114         // Add each leaf value from the operand to the Constants list
01115         // to form a flattened list of all the values.
01116         for (unsigned i = 0, e = Val->getNumValues(); i != e; ++i)
01117           Ops.push_back(SDValue(Val, i));
01118       }
01119 
01120       if (isa<ArrayType>(CDS->getType()))
01121         return DAG.getMergeValues(Ops, getCurSDLoc());
01122       return NodeMap[V] = DAG.getNode(ISD::BUILD_VECTOR, getCurSDLoc(),
01123                                       VT, Ops);
01124     }
01125 
01126     if (C->getType()->isStructTy() || C->getType()->isArrayTy()) {
01127       assert((isa<ConstantAggregateZero>(C) || isa<UndefValue>(C)) &&
01128              "Unknown struct or array constant!");
01129 
01130       SmallVector<EVT, 4> ValueVTs;
01131       ComputeValueVTs(TLI, C->getType(), ValueVTs);
01132       unsigned NumElts = ValueVTs.size();
01133       if (NumElts == 0)
01134         return SDValue(); // empty struct
01135       SmallVector<SDValue, 4> Constants(NumElts);
01136       for (unsigned i = 0; i != NumElts; ++i) {
01137         EVT EltVT = ValueVTs[i];
01138         if (isa<UndefValue>(C))
01139           Constants[i] = DAG.getUNDEF(EltVT);
01140         else if (EltVT.isFloatingPoint())
01141           Constants[i] = DAG.getConstantFP(0, EltVT);
01142         else
01143           Constants[i] = DAG.getConstant(0, EltVT);
01144       }
01145 
01146       return DAG.getMergeValues(Constants, getCurSDLoc());
01147     }
01148 
01149     if (const BlockAddress *BA = dyn_cast<BlockAddress>(C))
01150       return DAG.getBlockAddress(BA, VT);
01151 
01152     VectorType *VecTy = cast<VectorType>(V->getType());
01153     unsigned NumElements = VecTy->getNumElements();
01154 
01155     // Now that we know the number and type of the elements, get that number of
01156     // elements into the Ops array based on what kind of constant it is.
01157     SmallVector<SDValue, 16> Ops;
01158     if (const ConstantVector *CV = dyn_cast<ConstantVector>(C)) {
01159       for (unsigned i = 0; i != NumElements; ++i)
01160         Ops.push_back(getValue(CV->getOperand(i)));
01161     } else {
01162       assert(isa<ConstantAggregateZero>(C) && "Unknown vector constant!");
01163       EVT EltVT = TLI.getValueType(VecTy->getElementType());
01164 
01165       SDValue Op;
01166       if (EltVT.isFloatingPoint())
01167         Op = DAG.getConstantFP(0, EltVT);
01168       else
01169         Op = DAG.getConstant(0, EltVT);
01170       Ops.assign(NumElements, Op);
01171     }
01172 
01173     // Create a BUILD_VECTOR node.
01174     return NodeMap[V] = DAG.getNode(ISD::BUILD_VECTOR, getCurSDLoc(), VT, Ops);
01175   }
01176 
01177   // If this is a static alloca, generate it as the frameindex instead of
01178   // computation.
01179   if (const AllocaInst *AI = dyn_cast<AllocaInst>(V)) {
01180     DenseMap<const AllocaInst*, int>::iterator SI =
01181       FuncInfo.StaticAllocaMap.find(AI);
01182     if (SI != FuncInfo.StaticAllocaMap.end())
01183       return DAG.getFrameIndex(SI->second, TLI.getPointerTy());
01184   }
01185 
01186   // If this is an instruction which fast-isel has deferred, select it now.
01187   if (const Instruction *Inst = dyn_cast<Instruction>(V)) {
01188     unsigned InReg = FuncInfo.InitializeRegForValue(Inst);
01189     RegsForValue RFV(*DAG.getContext(), TLI, InReg, Inst->getType());
01190     SDValue Chain = DAG.getEntryNode();
01191     return RFV.getCopyFromRegs(DAG, FuncInfo, getCurSDLoc(), Chain, nullptr, V);
01192   }
01193 
01194   llvm_unreachable("Can't get register for value!");
01195 }
01196 
01197 void SelectionDAGBuilder::visitRet(const ReturnInst &I) {
01198   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
01199   SDValue Chain = getControlRoot();
01200   SmallVector<ISD::OutputArg, 8> Outs;
01201   SmallVector<SDValue, 8> OutVals;
01202 
01203   if (!FuncInfo.CanLowerReturn) {
01204     unsigned DemoteReg = FuncInfo.DemoteRegister;
01205     const Function *F = I.getParent()->getParent();
01206 
01207     // Emit a store of the return value through the virtual register.
01208     // Leave Outs empty so that LowerReturn won't try to load return
01209     // registers the usual way.
01210     SmallVector<EVT, 1> PtrValueVTs;
01211     ComputeValueVTs(TLI, PointerType::getUnqual(F->getReturnType()),
01212                     PtrValueVTs);
01213 
01214     SDValue RetPtr = DAG.getRegister(DemoteReg, PtrValueVTs[0]);
01215     SDValue RetOp = getValue(I.getOperand(0));
01216 
01217     SmallVector<EVT, 4> ValueVTs;
01218     SmallVector<uint64_t, 4> Offsets;
01219     ComputeValueVTs(TLI, I.getOperand(0)->getType(), ValueVTs, &Offsets);
01220     unsigned NumValues = ValueVTs.size();
01221 
01222     SmallVector<SDValue, 4> Chains(NumValues);
01223     for (unsigned i = 0; i != NumValues; ++i) {
01224       SDValue Add = DAG.getNode(ISD::ADD, getCurSDLoc(),
01225                                 RetPtr.getValueType(), RetPtr,
01226                                 DAG.getIntPtrConstant(Offsets[i]));
01227       Chains[i] =
01228         DAG.getStore(Chain, getCurSDLoc(),
01229                      SDValue(RetOp.getNode(), RetOp.getResNo() + i),
01230                      // FIXME: better loc info would be nice.
01231                      Add, MachinePointerInfo(), false, false, 0);
01232     }
01233 
01234     Chain = DAG.getNode(ISD::TokenFactor, getCurSDLoc(),
01235                         MVT::Other, Chains);
01236   } else if (I.getNumOperands() != 0) {
01237     SmallVector<EVT, 4> ValueVTs;
01238     ComputeValueVTs(TLI, I.getOperand(0)->getType(), ValueVTs);
01239     unsigned NumValues = ValueVTs.size();
01240     if (NumValues) {
01241       SDValue RetOp = getValue(I.getOperand(0));
01242 
01243       const Function *F = I.getParent()->getParent();
01244 
01245       ISD::NodeType ExtendKind = ISD::ANY_EXTEND;
01246       if (F->getAttributes().hasAttribute(AttributeSet::ReturnIndex,
01247                                           Attribute::SExt))
01248         ExtendKind = ISD::SIGN_EXTEND;
01249       else if (F->getAttributes().hasAttribute(AttributeSet::ReturnIndex,
01250                                                Attribute::ZExt))
01251         ExtendKind = ISD::ZERO_EXTEND;
01252 
01253       LLVMContext &Context = F->getContext();
01254       bool RetInReg = F->getAttributes().hasAttribute(AttributeSet::ReturnIndex,
01255                                                       Attribute::InReg);
01256 
01257       for (unsigned j = 0; j != NumValues; ++j) {
01258         EVT VT = ValueVTs[j];
01259 
01260         if (ExtendKind != ISD::ANY_EXTEND && VT.isInteger())
01261           VT = TLI.getTypeForExtArgOrReturn(Context, VT, ExtendKind);
01262 
01263         unsigned NumParts = TLI.getNumRegisters(Context, VT);
01264         MVT PartVT = TLI.getRegisterType(Context, VT);
01265         SmallVector<SDValue, 4> Parts(NumParts);
01266         getCopyToParts(DAG, getCurSDLoc(),
01267                        SDValue(RetOp.getNode(), RetOp.getResNo() + j),
01268                        &Parts[0], NumParts, PartVT, &I, ExtendKind);
01269 
01270         // 'inreg' on function refers to return value
01271         ISD::ArgFlagsTy Flags = ISD::ArgFlagsTy();
01272         if (RetInReg)
01273           Flags.setInReg();
01274 
01275         // Propagate extension type if any
01276         if (ExtendKind == ISD::SIGN_EXTEND)
01277           Flags.setSExt();
01278         else if (ExtendKind == ISD::ZERO_EXTEND)
01279           Flags.setZExt();
01280 
01281         for (unsigned i = 0; i < NumParts; ++i) {
01282           Outs.push_back(ISD::OutputArg(Flags, Parts[i].getValueType(),
01283                                         VT, /*isfixed=*/true, 0, 0));
01284           OutVals.push_back(Parts[i]);
01285         }
01286       }
01287     }
01288   }
01289 
01290   bool isVarArg = DAG.getMachineFunction().getFunction()->isVarArg();
01291   CallingConv::ID CallConv =
01292     DAG.getMachineFunction().getFunction()->getCallingConv();
01293   Chain = DAG.getTargetLoweringInfo().LowerReturn(
01294       Chain, CallConv, isVarArg, Outs, OutVals, getCurSDLoc(), DAG);
01295 
01296   // Verify that the target's LowerReturn behaved as expected.
01297   assert(Chain.getNode() && Chain.getValueType() == MVT::Other &&
01298          "LowerReturn didn't return a valid chain!");
01299 
01300   // Update the DAG with the new chain value resulting from return lowering.
01301   DAG.setRoot(Chain);
01302 }
01303 
01304 /// CopyToExportRegsIfNeeded - If the given value has virtual registers
01305 /// created for it, emit nodes to copy the value into the virtual
01306 /// registers.
01307 void SelectionDAGBuilder::CopyToExportRegsIfNeeded(const Value *V) {
01308   // Skip empty types
01309   if (V->getType()->isEmptyTy())
01310     return;
01311 
01312   DenseMap<const Value *, unsigned>::iterator VMI = FuncInfo.ValueMap.find(V);
01313   if (VMI != FuncInfo.ValueMap.end()) {
01314     assert(!V->use_empty() && "Unused value assigned virtual registers!");
01315     CopyValueToVirtualRegister(V, VMI->second);
01316   }
01317 }
01318 
01319 /// ExportFromCurrentBlock - If this condition isn't known to be exported from
01320 /// the current basic block, add it to ValueMap now so that we'll get a
01321 /// CopyTo/FromReg.
01322 void SelectionDAGBuilder::ExportFromCurrentBlock(const Value *V) {
01323   // No need to export constants.
01324   if (!isa<Instruction>(V) && !isa<Argument>(V)) return;
01325 
01326   // Already exported?
01327   if (FuncInfo.isExportedInst(V)) return;
01328 
01329   unsigned Reg = FuncInfo.InitializeRegForValue(V);
01330   CopyValueToVirtualRegister(V, Reg);
01331 }
01332 
01333 bool SelectionDAGBuilder::isExportableFromCurrentBlock(const Value *V,
01334                                                      const BasicBlock *FromBB) {
01335   // The operands of the setcc have to be in this block.  We don't know
01336   // how to export them from some other block.
01337   if (const Instruction *VI = dyn_cast<Instruction>(V)) {
01338     // Can export from current BB.
01339     if (VI->getParent() == FromBB)
01340       return true;
01341 
01342     // Is already exported, noop.
01343     return FuncInfo.isExportedInst(V);
01344   }
01345 
01346   // If this is an argument, we can export it if the BB is the entry block or
01347   // if it is already exported.
01348   if (isa<Argument>(V)) {
01349     if (FromBB == &FromBB->getParent()->getEntryBlock())
01350       return true;
01351 
01352     // Otherwise, can only export this if it is already exported.
01353     return FuncInfo.isExportedInst(V);
01354   }
01355 
01356   // Otherwise, constants can always be exported.
01357   return true;
01358 }
01359 
01360 /// Return branch probability calculated by BranchProbabilityInfo for IR blocks.
01361 uint32_t SelectionDAGBuilder::getEdgeWeight(const MachineBasicBlock *Src,
01362                                             const MachineBasicBlock *Dst) const {
01363   BranchProbabilityInfo *BPI = FuncInfo.BPI;
01364   if (!BPI)
01365     return 0;
01366   const BasicBlock *SrcBB = Src->getBasicBlock();
01367   const BasicBlock *DstBB = Dst->getBasicBlock();
01368   return BPI->getEdgeWeight(SrcBB, DstBB);
01369 }
01370 
01371 void SelectionDAGBuilder::
01372 addSuccessorWithWeight(MachineBasicBlock *Src, MachineBasicBlock *Dst,
01373                        uint32_t Weight /* = 0 */) {
01374   if (!Weight)
01375     Weight = getEdgeWeight(Src, Dst);
01376   Src->addSuccessor(Dst, Weight);
01377 }
01378 
01379 
01380 static bool InBlock(const Value *V, const BasicBlock *BB) {
01381   if (const Instruction *I = dyn_cast<Instruction>(V))
01382     return I->getParent() == BB;
01383   return true;
01384 }
01385 
01386 /// EmitBranchForMergedCondition - Helper method for FindMergedConditions.
01387 /// This function emits a branch and is used at the leaves of an OR or an
01388 /// AND operator tree.
01389 ///
01390 void
01391 SelectionDAGBuilder::EmitBranchForMergedCondition(const Value *Cond,
01392                                                   MachineBasicBlock *TBB,
01393                                                   MachineBasicBlock *FBB,
01394                                                   MachineBasicBlock *CurBB,
01395                                                   MachineBasicBlock *SwitchBB,
01396                                                   uint32_t TWeight,
01397                                                   uint32_t FWeight) {
01398   const BasicBlock *BB = CurBB->getBasicBlock();
01399 
01400   // If the leaf of the tree is a comparison, merge the condition into
01401   // the caseblock.
01402   if (const CmpInst *BOp = dyn_cast<CmpInst>(Cond)) {
01403     // The operands of the cmp have to be in this block.  We don't know
01404     // how to export them from some other block.  If this is the first block
01405     // of the sequence, no exporting is needed.
01406     if (CurBB == SwitchBB ||
01407         (isExportableFromCurrentBlock(BOp->getOperand(0), BB) &&
01408          isExportableFromCurrentBlock(BOp->getOperand(1), BB))) {
01409       ISD::CondCode Condition;
01410       if (const ICmpInst *IC = dyn_cast<ICmpInst>(Cond)) {
01411         Condition = getICmpCondCode(IC->getPredicate());
01412       } else if (const FCmpInst *FC = dyn_cast<FCmpInst>(Cond)) {
01413         Condition = getFCmpCondCode(FC->getPredicate());
01414         if (TM.Options.NoNaNsFPMath)
01415           Condition = getFCmpCodeWithoutNaN(Condition);
01416       } else {
01417         (void)Condition; // silence warning.
01418         llvm_unreachable("Unknown compare instruction");
01419       }
01420 
01421       CaseBlock CB(Condition, BOp->getOperand(0), BOp->getOperand(1), nullptr,
01422                    TBB, FBB, CurBB, TWeight, FWeight);
01423       SwitchCases.push_back(CB);
01424       return;
01425     }
01426   }
01427 
01428   // Create a CaseBlock record representing this branch.
01429   CaseBlock CB(ISD::SETEQ, Cond, ConstantInt::getTrue(*DAG.getContext()),
01430                nullptr, TBB, FBB, CurBB, TWeight, FWeight);
01431   SwitchCases.push_back(CB);
01432 }
01433 
01434 /// Scale down both weights to fit into uint32_t.
01435 static void ScaleWeights(uint64_t &NewTrue, uint64_t &NewFalse) {
01436   uint64_t NewMax = (NewTrue > NewFalse) ? NewTrue : NewFalse;
01437   uint32_t Scale = (NewMax / UINT32_MAX) + 1;
01438   NewTrue = NewTrue / Scale;
01439   NewFalse = NewFalse / Scale;
01440 }
01441 
01442 /// FindMergedConditions - If Cond is an expression like
01443 void SelectionDAGBuilder::FindMergedConditions(const Value *Cond,
01444                                                MachineBasicBlock *TBB,
01445                                                MachineBasicBlock *FBB,
01446                                                MachineBasicBlock *CurBB,
01447                                                MachineBasicBlock *SwitchBB,
01448                                                unsigned Opc, uint32_t TWeight,
01449                                                uint32_t FWeight) {
01450   // If this node is not part of the or/and tree, emit it as a branch.
01451   const Instruction *BOp = dyn_cast<Instruction>(Cond);
01452   if (!BOp || !(isa<BinaryOperator>(BOp) || isa<CmpInst>(BOp)) ||
01453       (unsigned)BOp->getOpcode() != Opc || !BOp->hasOneUse() ||
01454       BOp->getParent() != CurBB->getBasicBlock() ||
01455       !InBlock(BOp->getOperand(0), CurBB->getBasicBlock()) ||
01456       !InBlock(BOp->getOperand(1), CurBB->getBasicBlock())) {
01457     EmitBranchForMergedCondition(Cond, TBB, FBB, CurBB, SwitchBB,
01458                                  TWeight, FWeight);
01459     return;
01460   }
01461 
01462   //  Create TmpBB after CurBB.
01463   MachineFunction::iterator BBI = CurBB;
01464   MachineFunction &MF = DAG.getMachineFunction();
01465   MachineBasicBlock *TmpBB = MF.CreateMachineBasicBlock(CurBB->getBasicBlock());
01466   CurBB->getParent()->insert(++BBI, TmpBB);
01467 
01468   if (Opc == Instruction::Or) {
01469     // Codegen X | Y as:
01470     // BB1:
01471     //   jmp_if_X TBB
01472     //   jmp TmpBB
01473     // TmpBB:
01474     //   jmp_if_Y TBB
01475     //   jmp FBB
01476     //
01477 
01478     // We have flexibility in setting Prob for BB1 and Prob for TmpBB.
01479     // The requirement is that
01480     //   TrueProb for BB1 + (FalseProb for BB1 * TrueProb for TmpBB)
01481     //     = TrueProb for orignal BB.
01482     // Assuming the orignal weights are A and B, one choice is to set BB1's
01483     // weights to A and A+2B, and set TmpBB's weights to A and 2B. This choice
01484     // assumes that
01485     //   TrueProb for BB1 == FalseProb for BB1 * TrueProb for TmpBB.
01486     // Another choice is to assume TrueProb for BB1 equals to TrueProb for
01487     // TmpBB, but the math is more complicated.
01488 
01489     uint64_t NewTrueWeight = TWeight;
01490     uint64_t NewFalseWeight = (uint64_t)TWeight + 2 * (uint64_t)FWeight;
01491     ScaleWeights(NewTrueWeight, NewFalseWeight);
01492     // Emit the LHS condition.
01493     FindMergedConditions(BOp->getOperand(0), TBB, TmpBB, CurBB, SwitchBB, Opc,
01494                          NewTrueWeight, NewFalseWeight);
01495 
01496     NewTrueWeight = TWeight;
01497     NewFalseWeight = 2 * (uint64_t)FWeight;
01498     ScaleWeights(NewTrueWeight, NewFalseWeight);
01499     // Emit the RHS condition into TmpBB.
01500     FindMergedConditions(BOp->getOperand(1), TBB, FBB, TmpBB, SwitchBB, Opc,
01501                          NewTrueWeight, NewFalseWeight);
01502   } else {
01503     assert(Opc == Instruction::And && "Unknown merge op!");
01504     // Codegen X & Y as:
01505     // BB1:
01506     //   jmp_if_X TmpBB
01507     //   jmp FBB
01508     // TmpBB:
01509     //   jmp_if_Y TBB
01510     //   jmp FBB
01511     //
01512     //  This requires creation of TmpBB after CurBB.
01513 
01514     // We have flexibility in setting Prob for BB1 and Prob for TmpBB.
01515     // The requirement is that
01516     //   FalseProb for BB1 + (TrueProb for BB1 * FalseProb for TmpBB)
01517     //     = FalseProb for orignal BB.
01518     // Assuming the orignal weights are A and B, one choice is to set BB1's
01519     // weights to 2A+B and B, and set TmpBB's weights to 2A and B. This choice
01520     // assumes that
01521     //   FalseProb for BB1 == TrueProb for BB1 * FalseProb for TmpBB.
01522 
01523     uint64_t NewTrueWeight = 2 * (uint64_t)TWeight + (uint64_t)FWeight;
01524     uint64_t NewFalseWeight = FWeight;
01525     ScaleWeights(NewTrueWeight, NewFalseWeight);
01526     // Emit the LHS condition.
01527     FindMergedConditions(BOp->getOperand(0), TmpBB, FBB, CurBB, SwitchBB, Opc,
01528                          NewTrueWeight, NewFalseWeight);
01529 
01530     NewTrueWeight = 2 * (uint64_t)TWeight;
01531     NewFalseWeight = FWeight;
01532     ScaleWeights(NewTrueWeight, NewFalseWeight);
01533     // Emit the RHS condition into TmpBB.
01534     FindMergedConditions(BOp->getOperand(1), TBB, FBB, TmpBB, SwitchBB, Opc,
01535                          NewTrueWeight, NewFalseWeight);
01536   }
01537 }
01538 
01539 /// If the set of cases should be emitted as a series of branches, return true.
01540 /// If we should emit this as a bunch of and/or'd together conditions, return
01541 /// false.
01542 bool
01543 SelectionDAGBuilder::ShouldEmitAsBranches(const std::vector<CaseBlock> &Cases) {
01544   if (Cases.size() != 2) return true;
01545 
01546   // If this is two comparisons of the same values or'd or and'd together, they
01547   // will get folded into a single comparison, so don't emit two blocks.
01548   if ((Cases[0].CmpLHS == Cases[1].CmpLHS &&
01549        Cases[0].CmpRHS == Cases[1].CmpRHS) ||
01550       (Cases[0].CmpRHS == Cases[1].CmpLHS &&
01551        Cases[0].CmpLHS == Cases[1].CmpRHS)) {
01552     return false;
01553   }
01554 
01555   // Handle: (X != null) | (Y != null) --> (X|Y) != 0
01556   // Handle: (X == null) & (Y == null) --> (X|Y) == 0
01557   if (Cases[0].CmpRHS == Cases[1].CmpRHS &&
01558       Cases[0].CC == Cases[1].CC &&
01559       isa<Constant>(Cases[0].CmpRHS) &&
01560       cast<Constant>(Cases[0].CmpRHS)->isNullValue()) {
01561     if (Cases[0].CC == ISD::SETEQ && Cases[0].TrueBB == Cases[1].ThisBB)
01562       return false;
01563     if (Cases[0].CC == ISD::SETNE && Cases[0].FalseBB == Cases[1].ThisBB)
01564       return false;
01565   }
01566 
01567   return true;
01568 }
01569 
01570 void SelectionDAGBuilder::visitBr(const BranchInst &I) {
01571   MachineBasicBlock *BrMBB = FuncInfo.MBB;
01572 
01573   // Update machine-CFG edges.
01574   MachineBasicBlock *Succ0MBB = FuncInfo.MBBMap[I.getSuccessor(0)];
01575 
01576   // Figure out which block is immediately after the current one.
01577   MachineBasicBlock *NextBlock = nullptr;
01578   MachineFunction::iterator BBI = BrMBB;
01579   if (++BBI != FuncInfo.MF->end())
01580     NextBlock = BBI;
01581 
01582   if (I.isUnconditional()) {
01583     // Update machine-CFG edges.
01584     BrMBB->addSuccessor(Succ0MBB);
01585 
01586     // If this is not a fall-through branch or optimizations are switched off,
01587     // emit the branch.
01588     if (Succ0MBB != NextBlock || TM.getOptLevel() == CodeGenOpt::None)
01589       DAG.setRoot(DAG.getNode(ISD::BR, getCurSDLoc(),
01590                               MVT::Other, getControlRoot(),
01591                               DAG.getBasicBlock(Succ0MBB)));
01592 
01593     return;
01594   }
01595 
01596   // If this condition is one of the special cases we handle, do special stuff
01597   // now.
01598   const Value *CondVal = I.getCondition();
01599   MachineBasicBlock *Succ1MBB = FuncInfo.MBBMap[I.getSuccessor(1)];
01600 
01601   // If this is a series of conditions that are or'd or and'd together, emit
01602   // this as a sequence of branches instead of setcc's with and/or operations.
01603   // As long as jumps are not expensive, this should improve performance.
01604   // For example, instead of something like:
01605   //     cmp A, B
01606   //     C = seteq
01607   //     cmp D, E
01608   //     F = setle
01609   //     or C, F
01610   //     jnz foo
01611   // Emit:
01612   //     cmp A, B
01613   //     je foo
01614   //     cmp D, E
01615   //     jle foo
01616   //
01617   if (const BinaryOperator *BOp = dyn_cast<BinaryOperator>(CondVal)) {
01618     if (!DAG.getTargetLoweringInfo().isJumpExpensive() &&
01619         BOp->hasOneUse() && (BOp->getOpcode() == Instruction::And ||
01620                              BOp->getOpcode() == Instruction::Or)) {
01621       FindMergedConditions(BOp, Succ0MBB, Succ1MBB, BrMBB, BrMBB,
01622                            BOp->getOpcode(), getEdgeWeight(BrMBB, Succ0MBB),
01623                            getEdgeWeight(BrMBB, Succ1MBB));
01624       // If the compares in later blocks need to use values not currently
01625       // exported from this block, export them now.  This block should always
01626       // be the first entry.
01627       assert(SwitchCases[0].ThisBB == BrMBB && "Unexpected lowering!");
01628 
01629       // Allow some cases to be rejected.
01630       if (ShouldEmitAsBranches(SwitchCases)) {
01631         for (unsigned i = 1, e = SwitchCases.size(); i != e; ++i) {
01632           ExportFromCurrentBlock(SwitchCases[i].CmpLHS);
01633           ExportFromCurrentBlock(SwitchCases[i].CmpRHS);
01634         }
01635 
01636         // Emit the branch for this block.
01637         visitSwitchCase(SwitchCases[0], BrMBB);
01638         SwitchCases.erase(SwitchCases.begin());
01639         return;
01640       }
01641 
01642       // Okay, we decided not to do this, remove any inserted MBB's and clear
01643       // SwitchCases.
01644       for (unsigned i = 1, e = SwitchCases.size(); i != e; ++i)
01645         FuncInfo.MF->erase(SwitchCases[i].ThisBB);
01646 
01647       SwitchCases.clear();
01648     }
01649   }
01650 
01651   // Create a CaseBlock record representing this branch.
01652   CaseBlock CB(ISD::SETEQ, CondVal, ConstantInt::getTrue(*DAG.getContext()),
01653                nullptr, Succ0MBB, Succ1MBB, BrMBB);
01654 
01655   // Use visitSwitchCase to actually insert the fast branch sequence for this
01656   // cond branch.
01657   visitSwitchCase(CB, BrMBB);
01658 }
01659 
01660 /// visitSwitchCase - Emits the necessary code to represent a single node in
01661 /// the binary search tree resulting from lowering a switch instruction.
01662 void SelectionDAGBuilder::visitSwitchCase(CaseBlock &CB,
01663                                           MachineBasicBlock *SwitchBB) {
01664   SDValue Cond;
01665   SDValue CondLHS = getValue(CB.CmpLHS);
01666   SDLoc dl = getCurSDLoc();
01667 
01668   // Build the setcc now.
01669   if (!CB.CmpMHS) {
01670     // Fold "(X == true)" to X and "(X == false)" to !X to
01671     // handle common cases produced by branch lowering.
01672     if (CB.CmpRHS == ConstantInt::getTrue(*DAG.getContext()) &&
01673         CB.CC == ISD::SETEQ)
01674       Cond = CondLHS;
01675     else if (CB.CmpRHS == ConstantInt::getFalse(*DAG.getContext()) &&
01676              CB.CC == ISD::SETEQ) {
01677       SDValue True = DAG.getConstant(1, CondLHS.getValueType());
01678       Cond = DAG.getNode(ISD::XOR, dl, CondLHS.getValueType(), CondLHS, True);
01679     } else
01680       Cond = DAG.getSetCC(dl, MVT::i1, CondLHS, getValue(CB.CmpRHS), CB.CC);
01681   } else {
01682     assert(CB.CC == ISD::SETLE && "Can handle only LE ranges now");
01683 
01684     const APInt& Low = cast<ConstantInt>(CB.CmpLHS)->getValue();
01685     const APInt& High  = cast<ConstantInt>(CB.CmpRHS)->getValue();
01686 
01687     SDValue CmpOp = getValue(CB.CmpMHS);
01688     EVT VT = CmpOp.getValueType();
01689 
01690     if (cast<ConstantInt>(CB.CmpLHS)->isMinValue(true)) {
01691       Cond = DAG.getSetCC(dl, MVT::i1, CmpOp, DAG.getConstant(High, VT),
01692                           ISD::SETLE);
01693     } else {
01694       SDValue SUB = DAG.getNode(ISD::SUB, dl,
01695                                 VT, CmpOp, DAG.getConstant(Low, VT));
01696       Cond = DAG.getSetCC(dl, MVT::i1, SUB,
01697                           DAG.getConstant(High-Low, VT), ISD::SETULE);
01698     }
01699   }
01700 
01701   // Update successor info
01702   addSuccessorWithWeight(SwitchBB, CB.TrueBB, CB.TrueWeight);
01703   // TrueBB and FalseBB are always different unless the incoming IR is
01704   // degenerate. This only happens when running llc on weird IR.
01705   if (CB.TrueBB != CB.FalseBB)
01706     addSuccessorWithWeight(SwitchBB, CB.FalseBB, CB.FalseWeight);
01707 
01708   // Set NextBlock to be the MBB immediately after the current one, if any.
01709   // This is used to avoid emitting unnecessary branches to the next block.
01710   MachineBasicBlock *NextBlock = nullptr;
01711   MachineFunction::iterator BBI = SwitchBB;
01712   if (++BBI != FuncInfo.MF->end())
01713     NextBlock = BBI;
01714 
01715   // If the lhs block is the next block, invert the condition so that we can
01716   // fall through to the lhs instead of the rhs block.
01717   if (CB.TrueBB == NextBlock) {
01718     std::swap(CB.TrueBB, CB.FalseBB);
01719     SDValue True = DAG.getConstant(1, Cond.getValueType());
01720     Cond = DAG.getNode(ISD::XOR, dl, Cond.getValueType(), Cond, True);
01721   }
01722 
01723   SDValue BrCond = DAG.getNode(ISD::BRCOND, dl,
01724                                MVT::Other, getControlRoot(), Cond,
01725                                DAG.getBasicBlock(CB.TrueBB));
01726 
01727   // Insert the false branch. Do this even if it's a fall through branch,
01728   // this makes it easier to do DAG optimizations which require inverting
01729   // the branch condition.
01730   BrCond = DAG.getNode(ISD::BR, dl, MVT::Other, BrCond,
01731                        DAG.getBasicBlock(CB.FalseBB));
01732 
01733   DAG.setRoot(BrCond);
01734 }
01735 
01736 /// visitJumpTable - Emit JumpTable node in the current MBB
01737 void SelectionDAGBuilder::visitJumpTable(JumpTable &JT) {
01738   // Emit the code for the jump table
01739   assert(JT.Reg != -1U && "Should lower JT Header first!");
01740   EVT PTy = DAG.getTargetLoweringInfo().getPointerTy();
01741   SDValue Index = DAG.getCopyFromReg(getControlRoot(), getCurSDLoc(),
01742                                      JT.Reg, PTy);
01743   SDValue Table = DAG.getJumpTable(JT.JTI, PTy);
01744   SDValue BrJumpTable = DAG.getNode(ISD::BR_JT, getCurSDLoc(),
01745                                     MVT::Other, Index.getValue(1),
01746                                     Table, Index);
01747   DAG.setRoot(BrJumpTable);
01748 }
01749 
01750 /// visitJumpTableHeader - This function emits necessary code to produce index
01751 /// in the JumpTable from switch case.
01752 void SelectionDAGBuilder::visitJumpTableHeader(JumpTable &JT,
01753                                                JumpTableHeader &JTH,
01754                                                MachineBasicBlock *SwitchBB) {
01755   // Subtract the lowest switch case value from the value being switched on and
01756   // conditional branch to default mbb if the result is greater than the
01757   // difference between smallest and largest cases.
01758   SDValue SwitchOp = getValue(JTH.SValue);
01759   EVT VT = SwitchOp.getValueType();
01760   SDValue Sub = DAG.getNode(ISD::SUB, getCurSDLoc(), VT, SwitchOp,
01761                             DAG.getConstant(JTH.First, VT));
01762 
01763   // The SDNode we just created, which holds the value being switched on minus
01764   // the smallest case value, needs to be copied to a virtual register so it
01765   // can be used as an index into the jump table in a subsequent basic block.
01766   // This value may be smaller or larger than the target's pointer type, and
01767   // therefore require extension or truncating.
01768   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
01769   SwitchOp = DAG.getZExtOrTrunc(Sub, getCurSDLoc(), TLI.getPointerTy());
01770 
01771   unsigned JumpTableReg = FuncInfo.CreateReg(TLI.getPointerTy());
01772   SDValue CopyTo = DAG.getCopyToReg(getControlRoot(), getCurSDLoc(),
01773                                     JumpTableReg, SwitchOp);
01774   JT.Reg = JumpTableReg;
01775 
01776   // Emit the range check for the jump table, and branch to the default block
01777   // for the switch statement if the value being switched on exceeds the largest
01778   // case in the switch.
01779   SDValue CMP =
01780       DAG.getSetCC(getCurSDLoc(), TLI.getSetCCResultType(*DAG.getContext(),
01781                                                          Sub.getValueType()),
01782                    Sub, DAG.getConstant(JTH.Last - JTH.First, VT), ISD::SETUGT);
01783 
01784   // Set NextBlock to be the MBB immediately after the current one, if any.
01785   // This is used to avoid emitting unnecessary branches to the next block.
01786   MachineBasicBlock *NextBlock = nullptr;
01787   MachineFunction::iterator BBI = SwitchBB;
01788 
01789   if (++BBI != FuncInfo.MF->end())
01790     NextBlock = BBI;
01791 
01792   SDValue BrCond = DAG.getNode(ISD::BRCOND, getCurSDLoc(),
01793                                MVT::Other, CopyTo, CMP,
01794                                DAG.getBasicBlock(JT.Default));
01795 
01796   if (JT.MBB != NextBlock)
01797     BrCond = DAG.getNode(ISD::BR, getCurSDLoc(), MVT::Other, BrCond,
01798                          DAG.getBasicBlock(JT.MBB));
01799 
01800   DAG.setRoot(BrCond);
01801 }
01802 
01803 /// Codegen a new tail for a stack protector check ParentMBB which has had its
01804 /// tail spliced into a stack protector check success bb.
01805 ///
01806 /// For a high level explanation of how this fits into the stack protector
01807 /// generation see the comment on the declaration of class
01808 /// StackProtectorDescriptor.
01809 void SelectionDAGBuilder::visitSPDescriptorParent(StackProtectorDescriptor &SPD,
01810                                                   MachineBasicBlock *ParentBB) {
01811 
01812   // First create the loads to the guard/stack slot for the comparison.
01813   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
01814   EVT PtrTy = TLI.getPointerTy();
01815 
01816   MachineFrameInfo *MFI = ParentBB->getParent()->getFrameInfo();
01817   int FI = MFI->getStackProtectorIndex();
01818 
01819   const Value *IRGuard = SPD.getGuard();
01820   SDValue GuardPtr = getValue(IRGuard);
01821   SDValue StackSlotPtr = DAG.getFrameIndex(FI, PtrTy);
01822 
01823   unsigned Align =
01824     TLI.getDataLayout()->getPrefTypeAlignment(IRGuard->getType());
01825 
01826   SDValue Guard;
01827 
01828   // If GuardReg is set and useLoadStackGuardNode returns true, retrieve the
01829   // guard value from the virtual register holding the value. Otherwise, emit a
01830   // volatile load to retrieve the stack guard value.
01831   unsigned GuardReg = SPD.getGuardReg();
01832 
01833   if (GuardReg && TLI.useLoadStackGuardNode())
01834     Guard = DAG.getCopyFromReg(DAG.getEntryNode(), getCurSDLoc(), GuardReg,
01835                                PtrTy);
01836   else
01837     Guard = DAG.getLoad(PtrTy, getCurSDLoc(), DAG.getEntryNode(),
01838                         GuardPtr, MachinePointerInfo(IRGuard, 0),
01839                         true, false, false, Align);
01840 
01841   SDValue StackSlot = DAG.getLoad(PtrTy, getCurSDLoc(), DAG.getEntryNode(),
01842                                   StackSlotPtr,
01843                                   MachinePointerInfo::getFixedStack(FI),
01844                                   true, false, false, Align);
01845 
01846   // Perform the comparison via a subtract/getsetcc.
01847   EVT VT = Guard.getValueType();
01848   SDValue Sub = DAG.getNode(ISD::SUB, getCurSDLoc(), VT, Guard, StackSlot);
01849 
01850   SDValue Cmp =
01851       DAG.getSetCC(getCurSDLoc(), TLI.getSetCCResultType(*DAG.getContext(),
01852                                                          Sub.getValueType()),
01853                    Sub, DAG.getConstant(0, VT), ISD::SETNE);
01854 
01855   // If the sub is not 0, then we know the guard/stackslot do not equal, so
01856   // branch to failure MBB.
01857   SDValue BrCond = DAG.getNode(ISD::BRCOND, getCurSDLoc(),
01858                                MVT::Other, StackSlot.getOperand(0),
01859                                Cmp, DAG.getBasicBlock(SPD.getFailureMBB()));
01860   // Otherwise branch to success MBB.
01861   SDValue Br = DAG.getNode(ISD::BR, getCurSDLoc(),
01862                            MVT::Other, BrCond,
01863                            DAG.getBasicBlock(SPD.getSuccessMBB()));
01864 
01865   DAG.setRoot(Br);
01866 }
01867 
01868 /// Codegen the failure basic block for a stack protector check.
01869 ///
01870 /// A failure stack protector machine basic block consists simply of a call to
01871 /// __stack_chk_fail().
01872 ///
01873 /// For a high level explanation of how this fits into the stack protector
01874 /// generation see the comment on the declaration of class
01875 /// StackProtectorDescriptor.
01876 void
01877 SelectionDAGBuilder::visitSPDescriptorFailure(StackProtectorDescriptor &SPD) {
01878   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
01879   SDValue Chain =
01880       TLI.makeLibCall(DAG, RTLIB::STACKPROTECTOR_CHECK_FAIL, MVT::isVoid,
01881                       nullptr, 0, false, getCurSDLoc(), false, false).second;
01882   DAG.setRoot(Chain);
01883 }
01884 
01885 /// visitBitTestHeader - This function emits necessary code to produce value
01886 /// suitable for "bit tests"
01887 void SelectionDAGBuilder::visitBitTestHeader(BitTestBlock &B,
01888                                              MachineBasicBlock *SwitchBB) {
01889   // Subtract the minimum value
01890   SDValue SwitchOp = getValue(B.SValue);
01891   EVT VT = SwitchOp.getValueType();
01892   SDValue Sub = DAG.getNode(ISD::SUB, getCurSDLoc(), VT, SwitchOp,
01893                             DAG.getConstant(B.First, VT));
01894 
01895   // Check range
01896   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
01897   SDValue RangeCmp =
01898       DAG.getSetCC(getCurSDLoc(), TLI.getSetCCResultType(*DAG.getContext(),
01899                                                          Sub.getValueType()),
01900                    Sub, DAG.getConstant(B.Range, VT), ISD::SETUGT);
01901 
01902   // Determine the type of the test operands.
01903   bool UsePtrType = false;
01904   if (!TLI.isTypeLegal(VT))
01905     UsePtrType = true;
01906   else {
01907     for (unsigned i = 0, e = B.Cases.size(); i != e; ++i)
01908       if (!isUIntN(VT.getSizeInBits(), B.Cases[i].Mask)) {
01909         // Switch table case range are encoded into series of masks.
01910         // Just use pointer type, it's guaranteed to fit.
01911         UsePtrType = true;
01912         break;
01913       }
01914   }
01915   if (UsePtrType) {
01916     VT = TLI.getPointerTy();
01917     Sub = DAG.getZExtOrTrunc(Sub, getCurSDLoc(), VT);
01918   }
01919 
01920   B.RegVT = VT.getSimpleVT();
01921   B.Reg = FuncInfo.CreateReg(B.RegVT);
01922   SDValue CopyTo = DAG.getCopyToReg(getControlRoot(), getCurSDLoc(),
01923                                     B.Reg, Sub);
01924 
01925   // Set NextBlock to be the MBB immediately after the current one, if any.
01926   // This is used to avoid emitting unnecessary branches to the next block.
01927   MachineBasicBlock *NextBlock = nullptr;
01928   MachineFunction::iterator BBI = SwitchBB;
01929   if (++BBI != FuncInfo.MF->end())
01930     NextBlock = BBI;
01931 
01932   MachineBasicBlock* MBB = B.Cases[0].ThisBB;
01933 
01934   addSuccessorWithWeight(SwitchBB, B.Default);
01935   addSuccessorWithWeight(SwitchBB, MBB);
01936 
01937   SDValue BrRange = DAG.getNode(ISD::BRCOND, getCurSDLoc(),
01938                                 MVT::Other, CopyTo, RangeCmp,
01939                                 DAG.getBasicBlock(B.Default));
01940 
01941   if (MBB != NextBlock)
01942     BrRange = DAG.getNode(ISD::BR, getCurSDLoc(), MVT::Other, CopyTo,
01943                           DAG.getBasicBlock(MBB));
01944 
01945   DAG.setRoot(BrRange);
01946 }
01947 
01948 /// visitBitTestCase - this function produces one "bit test"
01949 void SelectionDAGBuilder::visitBitTestCase(BitTestBlock &BB,
01950                                            MachineBasicBlock* NextMBB,
01951                                            uint32_t BranchWeightToNext,
01952                                            unsigned Reg,
01953                                            BitTestCase &B,
01954                                            MachineBasicBlock *SwitchBB) {
01955   MVT VT = BB.RegVT;
01956   SDValue ShiftOp = DAG.getCopyFromReg(getControlRoot(), getCurSDLoc(),
01957                                        Reg, VT);
01958   SDValue Cmp;
01959   unsigned PopCount = countPopulation(B.Mask);
01960   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
01961   if (PopCount == 1) {
01962     // Testing for a single bit; just compare the shift count with what it
01963     // would need to be to shift a 1 bit in that position.
01964     Cmp = DAG.getSetCC(
01965         getCurSDLoc(), TLI.getSetCCResultType(*DAG.getContext(), VT), ShiftOp,
01966         DAG.getConstant(countTrailingZeros(B.Mask), VT), ISD::SETEQ);
01967   } else if (PopCount == BB.Range) {
01968     // There is only one zero bit in the range, test for it directly.
01969     Cmp = DAG.getSetCC(
01970         getCurSDLoc(), TLI.getSetCCResultType(*DAG.getContext(), VT), ShiftOp,
01971         DAG.getConstant(countTrailingOnes(B.Mask), VT), ISD::SETNE);
01972   } else {
01973     // Make desired shift
01974     SDValue SwitchVal = DAG.getNode(ISD::SHL, getCurSDLoc(), VT,
01975                                     DAG.getConstant(1, VT), ShiftOp);
01976 
01977     // Emit bit tests and jumps
01978     SDValue AndOp = DAG.getNode(ISD::AND, getCurSDLoc(),
01979                                 VT, SwitchVal, DAG.getConstant(B.Mask, VT));
01980     Cmp = DAG.getSetCC(getCurSDLoc(),
01981                        TLI.getSetCCResultType(*DAG.getContext(), VT), AndOp,
01982                        DAG.getConstant(0, VT), ISD::SETNE);
01983   }
01984 
01985   // The branch weight from SwitchBB to B.TargetBB is B.ExtraWeight.
01986   addSuccessorWithWeight(SwitchBB, B.TargetBB, B.ExtraWeight);
01987   // The branch weight from SwitchBB to NextMBB is BranchWeightToNext.
01988   addSuccessorWithWeight(SwitchBB, NextMBB, BranchWeightToNext);
01989 
01990   SDValue BrAnd = DAG.getNode(ISD::BRCOND, getCurSDLoc(),
01991                               MVT::Other, getControlRoot(),
01992                               Cmp, DAG.getBasicBlock(B.TargetBB));
01993 
01994   // Set NextBlock to be the MBB immediately after the current one, if any.
01995   // This is used to avoid emitting unnecessary branches to the next block.
01996   MachineBasicBlock *NextBlock = nullptr;
01997   MachineFunction::iterator BBI = SwitchBB;
01998   if (++BBI != FuncInfo.MF->end())
01999     NextBlock = BBI;
02000 
02001   if (NextMBB != NextBlock)
02002     BrAnd = DAG.getNode(ISD::BR, getCurSDLoc(), MVT::Other, BrAnd,
02003                         DAG.getBasicBlock(NextMBB));
02004 
02005   DAG.setRoot(BrAnd);
02006 }
02007 
02008 void SelectionDAGBuilder::visitInvoke(const InvokeInst &I) {
02009   MachineBasicBlock *InvokeMBB = FuncInfo.MBB;
02010 
02011   // Retrieve successors.
02012   MachineBasicBlock *Return = FuncInfo.MBBMap[I.getSuccessor(0)];
02013   MachineBasicBlock *LandingPad = FuncInfo.MBBMap[I.getSuccessor(1)];
02014 
02015   const Value *Callee(I.getCalledValue());
02016   const Function *Fn = dyn_cast<Function>(Callee);
02017   if (isa<InlineAsm>(Callee))
02018     visitInlineAsm(&I);
02019   else if (Fn && Fn->isIntrinsic()) {
02020     switch (Fn->getIntrinsicID()) {
02021     default:
02022       llvm_unreachable("Cannot invoke this intrinsic");
02023     case Intrinsic::donothing:
02024       // Ignore invokes to @llvm.donothing: jump directly to the next BB.
02025       break;
02026     case Intrinsic::experimental_patchpoint_void:
02027     case Intrinsic::experimental_patchpoint_i64:
02028       visitPatchpoint(&I, LandingPad);
02029       break;
02030     }
02031   } else
02032     LowerCallTo(&I, getValue(Callee), false, LandingPad);
02033 
02034   // If the value of the invoke is used outside of its defining block, make it
02035   // available as a virtual register.
02036   CopyToExportRegsIfNeeded(&I);
02037 
02038   // Update successor info
02039   addSuccessorWithWeight(InvokeMBB, Return);
02040   addSuccessorWithWeight(InvokeMBB, LandingPad);
02041 
02042   // Drop into normal successor.
02043   DAG.setRoot(DAG.getNode(ISD::BR, getCurSDLoc(),
02044                           MVT::Other, getControlRoot(),
02045                           DAG.getBasicBlock(Return)));
02046 }
02047 
02048 void SelectionDAGBuilder::visitResume(const ResumeInst &RI) {
02049   llvm_unreachable("SelectionDAGBuilder shouldn't visit resume instructions!");
02050 }
02051 
02052 void SelectionDAGBuilder::visitLandingPad(const LandingPadInst &LP) {
02053   assert(FuncInfo.MBB->isLandingPad() &&
02054          "Call to landingpad not in landing pad!");
02055 
02056   MachineBasicBlock *MBB = FuncInfo.MBB;
02057   MachineModuleInfo &MMI = DAG.getMachineFunction().getMMI();
02058   AddLandingPadInfo(LP, MMI, MBB);
02059 
02060   // If there aren't registers to copy the values into (e.g., during SjLj
02061   // exceptions), then don't bother to create these DAG nodes.
02062   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
02063   if (TLI.getExceptionPointerRegister() == 0 &&
02064       TLI.getExceptionSelectorRegister() == 0)
02065     return;
02066 
02067   SmallVector<EVT, 2> ValueVTs;
02068   ComputeValueVTs(TLI, LP.getType(), ValueVTs);
02069   assert(ValueVTs.size() == 2 && "Only two-valued landingpads are supported");
02070 
02071   // Get the two live-in registers as SDValues. The physregs have already been
02072   // copied into virtual registers.
02073   SDValue Ops[2];
02074   if (FuncInfo.ExceptionPointerVirtReg) {
02075     Ops[0] = DAG.getZExtOrTrunc(
02076         DAG.getCopyFromReg(DAG.getEntryNode(), getCurSDLoc(),
02077                            FuncInfo.ExceptionPointerVirtReg, TLI.getPointerTy()),
02078         getCurSDLoc(), ValueVTs[0]);
02079   } else {
02080     Ops[0] = DAG.getConstant(0, TLI.getPointerTy());
02081   }
02082   Ops[1] = DAG.getZExtOrTrunc(
02083       DAG.getCopyFromReg(DAG.getEntryNode(), getCurSDLoc(),
02084                          FuncInfo.ExceptionSelectorVirtReg, TLI.getPointerTy()),
02085       getCurSDLoc(), ValueVTs[1]);
02086 
02087   // Merge into one.
02088   SDValue Res = DAG.getNode(ISD::MERGE_VALUES, getCurSDLoc(),
02089                             DAG.getVTList(ValueVTs), Ops);
02090   setValue(&LP, Res);
02091 }
02092 
02093 unsigned
02094 SelectionDAGBuilder::visitLandingPadClauseBB(GlobalValue *ClauseGV,
02095                                              MachineBasicBlock *LPadBB) {
02096   SDValue Chain = getControlRoot();
02097 
02098   // Get the typeid that we will dispatch on later.
02099   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
02100   const TargetRegisterClass *RC = TLI.getRegClassFor(TLI.getPointerTy());
02101   unsigned VReg = FuncInfo.MF->getRegInfo().createVirtualRegister(RC);
02102   unsigned TypeID = DAG.getMachineFunction().getMMI().getTypeIDFor(ClauseGV);
02103   SDValue Sel = DAG.getConstant(TypeID, TLI.getPointerTy());
02104   Chain = DAG.getCopyToReg(Chain, getCurSDLoc(), VReg, Sel);
02105 
02106   // Branch to the main landing pad block.
02107   MachineBasicBlock *ClauseMBB = FuncInfo.MBB;
02108   ClauseMBB->addSuccessor(LPadBB);
02109   DAG.setRoot(DAG.getNode(ISD::BR, getCurSDLoc(), MVT::Other, Chain,
02110                           DAG.getBasicBlock(LPadBB)));
02111   return VReg;
02112 }
02113 
02114 /// handleSmallSwitchCaseRange - Emit a series of specific tests (suitable for
02115 /// small case ranges).
02116 bool SelectionDAGBuilder::handleSmallSwitchRange(CaseRec& CR,
02117                                                  CaseRecVector& WorkList,
02118                                                  const Value* SV,
02119                                                  MachineBasicBlock *Default,
02120                                                  MachineBasicBlock *SwitchBB) {
02121   // Size is the number of Cases represented by this range.
02122   size_t Size = CR.Range.second - CR.Range.first;
02123   if (Size > 3)
02124     return false;
02125 
02126   // Get the MachineFunction which holds the current MBB.  This is used when
02127   // inserting any additional MBBs necessary to represent the switch.
02128   MachineFunction *CurMF = FuncInfo.MF;
02129 
02130   // Figure out which block is immediately after the current one.
02131   MachineBasicBlock *NextBlock = nullptr;
02132   MachineFunction::iterator BBI = CR.CaseBB;
02133 
02134   if (++BBI != FuncInfo.MF->end())
02135     NextBlock = BBI;
02136 
02137   BranchProbabilityInfo *BPI = FuncInfo.BPI;
02138   // If any two of the cases has the same destination, and if one value
02139   // is the same as the other, but has one bit unset that the other has set,
02140   // use bit manipulation to do two compares at once.  For example:
02141   // "if (X == 6 || X == 4)" -> "if ((X|2) == 6)"
02142   // TODO: This could be extended to merge any 2 cases in switches with 3 cases.
02143   // TODO: Handle cases where CR.CaseBB != SwitchBB.
02144   if (Size == 2 && CR.CaseBB == SwitchBB) {
02145     Case &Small = *CR.Range.first;
02146     Case &Big = *(CR.Range.second-1);
02147 
02148     if (Small.Low == Small.High && Big.Low == Big.High && Small.BB == Big.BB) {
02149       const APInt& SmallValue = cast<ConstantInt>(Small.Low)->getValue();
02150       const APInt& BigValue = cast<ConstantInt>(Big.Low)->getValue();
02151 
02152       // Check that there is only one bit different.
02153       if (BigValue.countPopulation() == SmallValue.countPopulation() + 1 &&
02154           (SmallValue | BigValue) == BigValue) {
02155         // Isolate the common bit.
02156         APInt CommonBit = BigValue & ~SmallValue;
02157         assert((SmallValue | CommonBit) == BigValue &&
02158                CommonBit.countPopulation() == 1 && "Not a common bit?");
02159 
02160         SDValue CondLHS = getValue(SV);
02161         EVT VT = CondLHS.getValueType();
02162         SDLoc DL = getCurSDLoc();
02163 
02164         SDValue Or = DAG.getNode(ISD::OR, DL, VT, CondLHS,
02165                                  DAG.getConstant(CommonBit, VT));
02166         SDValue Cond = DAG.getSetCC(DL, MVT::i1,
02167                                     Or, DAG.getConstant(BigValue, VT),
02168                                     ISD::SETEQ);
02169 
02170         // Update successor info.
02171         // Both Small and Big will jump to Small.BB, so we sum up the weights.
02172         addSuccessorWithWeight(SwitchBB, Small.BB,
02173                                Small.ExtraWeight + Big.ExtraWeight);
02174         addSuccessorWithWeight(SwitchBB, Default,
02175           // The default destination is the first successor in IR.
02176           BPI ? BPI->getEdgeWeight(SwitchBB->getBasicBlock(), (unsigned)0) : 0);
02177 
02178         // Insert the true branch.
02179         SDValue BrCond = DAG.getNode(ISD::BRCOND, DL, MVT::Other,
02180                                      getControlRoot(), Cond,
02181                                      DAG.getBasicBlock(Small.BB));
02182 
02183         // Insert the false branch.
02184         BrCond = DAG.getNode(ISD::BR, DL, MVT::Other, BrCond,
02185                              DAG.getBasicBlock(Default));
02186 
02187         DAG.setRoot(BrCond);
02188         return true;
02189       }
02190     }
02191   }
02192 
02193   // Order cases by weight so the most likely case will be checked first.
02194   uint32_t UnhandledWeights = 0;
02195   if (BPI) {
02196     for (CaseItr I = CR.Range.first, IE = CR.Range.second; I != IE; ++I) {
02197       uint32_t IWeight = I->ExtraWeight;
02198       UnhandledWeights += IWeight;
02199       for (CaseItr J = CR.Range.first; J < I; ++J) {
02200         uint32_t JWeight = J->ExtraWeight;
02201         if (IWeight > JWeight)
02202           std::swap(*I, *J);
02203       }
02204     }
02205   }
02206   // Rearrange the case blocks so that the last one falls through if possible.
02207   Case &BackCase = *(CR.Range.second-1);
02208   if (Size > 1 &&
02209       NextBlock && Default != NextBlock && BackCase.BB != NextBlock) {
02210     // The last case block won't fall through into 'NextBlock' if we emit the
02211     // branches in this order.  See if rearranging a case value would help.
02212     // We start at the bottom as it's the case with the least weight.
02213     for (Case *I = &*(CR.Range.second-2), *E = &*CR.Range.first-1; I != E; --I)
02214       if (I->BB == NextBlock) {
02215         std::swap(*I, BackCase);
02216         break;
02217       }
02218   }
02219 
02220   // Create a CaseBlock record representing a conditional branch to
02221   // the Case's target mbb if the value being switched on SV is equal
02222   // to C.
02223   MachineBasicBlock *CurBlock = CR.CaseBB;
02224   for (CaseItr I = CR.Range.first, E = CR.Range.second; I != E; ++I) {
02225     MachineBasicBlock *FallThrough;
02226     if (I != E-1) {
02227       FallThrough = CurMF->CreateMachineBasicBlock(CurBlock->getBasicBlock());
02228       CurMF->insert(BBI, FallThrough);
02229 
02230       // Put SV in a virtual register to make it available from the new blocks.
02231       ExportFromCurrentBlock(SV);
02232     } else {
02233       // If the last case doesn't match, go to the default block.
02234       FallThrough = Default;
02235     }
02236 
02237     const Value *RHS, *LHS, *MHS;
02238     ISD::CondCode CC;
02239     if (I->High == I->Low) {
02240       // This is just small small case range :) containing exactly 1 case
02241       CC = ISD::SETEQ;
02242       LHS = SV; RHS = I->High; MHS = nullptr;
02243     } else {
02244       CC = ISD::SETLE;
02245       LHS = I->Low; MHS = SV; RHS = I->High;
02246     }
02247 
02248     // The false weight should be sum of all un-handled cases.
02249     UnhandledWeights -= I->ExtraWeight;
02250     CaseBlock CB(CC, LHS, RHS, MHS, /* truebb */ I->BB, /* falsebb */ FallThrough,
02251                  /* me */ CurBlock,
02252                  /* trueweight */ I->ExtraWeight,
02253                  /* falseweight */ UnhandledWeights);
02254 
02255     // If emitting the first comparison, just call visitSwitchCase to emit the
02256     // code into the current block.  Otherwise, push the CaseBlock onto the
02257     // vector to be later processed by SDISel, and insert the node's MBB
02258     // before the next MBB.
02259     if (CurBlock == SwitchBB)
02260       visitSwitchCase(CB, SwitchBB);
02261     else
02262       SwitchCases.push_back(CB);
02263 
02264     CurBlock = FallThrough;
02265   }
02266 
02267   return true;
02268 }
02269 
02270 static inline bool areJTsAllowed(const TargetLowering &TLI) {
02271   return TLI.isOperationLegalOrCustom(ISD::BR_JT, MVT::Other) ||
02272          TLI.isOperationLegalOrCustom(ISD::BRIND, MVT::Other);
02273 }
02274 
02275 static APInt ComputeRange(const APInt &First, const APInt &Last) {
02276   uint32_t BitWidth = std::max(Last.getBitWidth(), First.getBitWidth()) + 1;
02277   APInt LastExt = Last.sext(BitWidth), FirstExt = First.sext(BitWidth);
02278   return (LastExt - FirstExt + 1ULL);
02279 }
02280 
02281 /// handleJTSwitchCase - Emit jumptable for current switch case range
02282 bool SelectionDAGBuilder::handleJTSwitchCase(CaseRec &CR,
02283                                              CaseRecVector &WorkList,
02284                                              const Value *SV,
02285                                              MachineBasicBlock *Default,
02286                                              MachineBasicBlock *SwitchBB) {
02287   Case& FrontCase = *CR.Range.first;
02288   Case& BackCase  = *(CR.Range.second-1);
02289 
02290   const APInt &First = cast<ConstantInt>(FrontCase.Low)->getValue();
02291   const APInt &Last  = cast<ConstantInt>(BackCase.High)->getValue();
02292 
02293   APInt TSize(First.getBitWidth(), 0);
02294   for (CaseItr I = CR.Range.first, E = CR.Range.second; I != E; ++I)
02295     TSize += I->size();
02296 
02297   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
02298   if (!areJTsAllowed(TLI) || TSize.ult(TLI.getMinimumJumpTableEntries()))
02299     return false;
02300 
02301   APInt Range = ComputeRange(First, Last);
02302   // The density is TSize / Range. Require at least 40%.
02303   // It should not be possible for IntTSize to saturate for sane code, but make
02304   // sure we handle Range saturation correctly.
02305   uint64_t IntRange = Range.getLimitedValue(UINT64_MAX/10);
02306   uint64_t IntTSize = TSize.getLimitedValue(UINT64_MAX/10);
02307   if (IntTSize * 10 < IntRange * 4)
02308     return false;
02309 
02310   DEBUG(dbgs() << "Lowering jump table\n"
02311                << "First entry: " << First << ". Last entry: " << Last << '\n'
02312                << "Range: " << Range << ". Size: " << TSize << ".\n\n");
02313 
02314   // Get the MachineFunction which holds the current MBB.  This is used when
02315   // inserting any additional MBBs necessary to represent the switch.
02316   MachineFunction *CurMF = FuncInfo.MF;
02317 
02318   // Figure out which block is immediately after the current one.
02319   MachineFunction::iterator BBI = CR.CaseBB;
02320   ++BBI;
02321 
02322   const BasicBlock *LLVMBB = CR.CaseBB->getBasicBlock();
02323 
02324   // Create a new basic block to hold the code for loading the address
02325   // of the jump table, and jumping to it.  Update successor information;
02326   // we will either branch to the default case for the switch, or the jump
02327   // table.
02328   MachineBasicBlock *JumpTableBB = CurMF->CreateMachineBasicBlock(LLVMBB);
02329   CurMF->insert(BBI, JumpTableBB);
02330 
02331   addSuccessorWithWeight(CR.CaseBB, Default);
02332   addSuccessorWithWeight(CR.CaseBB, JumpTableBB);
02333 
02334   // Build a vector of destination BBs, corresponding to each target
02335   // of the jump table. If the value of the jump table slot corresponds to
02336   // a case statement, push the case's BB onto the vector, otherwise, push
02337   // the default BB.
02338   std::vector<MachineBasicBlock*> DestBBs;
02339   APInt TEI = First;
02340   for (CaseItr I = CR.Range.first, E = CR.Range.second; I != E; ++TEI) {
02341     const APInt &Low = cast<ConstantInt>(I->Low)->getValue();
02342     const APInt &High = cast<ConstantInt>(I->High)->getValue();
02343 
02344     if (Low.sle(TEI) && TEI.sle(High)) {
02345       DestBBs.push_back(I->BB);
02346       if (TEI==High)
02347         ++I;
02348     } else {
02349       DestBBs.push_back(Default);
02350     }
02351   }
02352 
02353   // Calculate weight for each unique destination in CR.
02354   DenseMap<MachineBasicBlock*, uint32_t> DestWeights;
02355   if (FuncInfo.BPI)
02356     for (CaseItr I = CR.Range.first, E = CR.Range.second; I != E; ++I) {
02357       DenseMap<MachineBasicBlock*, uint32_t>::iterator Itr =
02358           DestWeights.find(I->BB);
02359       if (Itr != DestWeights.end())
02360         Itr->second += I->ExtraWeight;
02361       else
02362         DestWeights[I->BB] = I->ExtraWeight;
02363     }
02364 
02365   // Update successor info. Add one edge to each unique successor.
02366   BitVector SuccsHandled(CR.CaseBB->getParent()->getNumBlockIDs());
02367   for (std::vector<MachineBasicBlock*>::iterator I = DestBBs.begin(),
02368          E = DestBBs.end(); I != E; ++I) {
02369     if (!SuccsHandled[(*I)->getNumber()]) {
02370       SuccsHandled[(*I)->getNumber()] = true;
02371       DenseMap<MachineBasicBlock*, uint32_t>::iterator Itr =
02372           DestWeights.find(*I);
02373       addSuccessorWithWeight(JumpTableBB, *I,
02374                              Itr != DestWeights.end() ? Itr->second : 0);
02375     }
02376   }
02377 
02378   // Create a jump table index for this jump table.
02379   unsigned JTEncoding = TLI.getJumpTableEncoding();
02380   unsigned JTI = CurMF->getOrCreateJumpTableInfo(JTEncoding)
02381                        ->createJumpTableIndex(DestBBs);
02382 
02383   // Set the jump table information so that we can codegen it as a second
02384   // MachineBasicBlock
02385   JumpTable JT(-1U, JTI, JumpTableBB, Default);
02386   JumpTableHeader JTH(First, Last, SV, CR.CaseBB, (CR.CaseBB == SwitchBB));
02387   if (CR.CaseBB == SwitchBB)
02388     visitJumpTableHeader(JT, JTH, SwitchBB);
02389 
02390   JTCases.push_back(JumpTableBlock(JTH, JT));
02391   return true;
02392 }
02393 
02394 /// handleBTSplitSwitchCase - emit comparison and split binary search tree into
02395 /// 2 subtrees.
02396 bool SelectionDAGBuilder::handleBTSplitSwitchCase(CaseRec& CR,
02397                                                   CaseRecVector& WorkList,
02398                                                   const Value* SV,
02399                                                   MachineBasicBlock* SwitchBB) {
02400   Case& FrontCase = *CR.Range.first;
02401   Case& BackCase  = *(CR.Range.second-1);
02402 
02403   // Size is the number of Cases represented by this range.
02404   unsigned Size = CR.Range.second - CR.Range.first;
02405 
02406   const APInt &First = cast<ConstantInt>(FrontCase.Low)->getValue();
02407   const APInt &Last  = cast<ConstantInt>(BackCase.High)->getValue();
02408   double FMetric = 0;
02409   CaseItr Pivot = CR.Range.first + Size/2;
02410 
02411   // Select optimal pivot, maximizing sum density of LHS and RHS. This will
02412   // (heuristically) allow us to emit JumpTable's later.
02413   APInt TSize(First.getBitWidth(), 0);
02414   for (CaseItr I = CR.Range.first, E = CR.Range.second;
02415        I!=E; ++I)
02416     TSize += I->size();
02417 
02418   APInt LSize = FrontCase.size();
02419   APInt RSize = TSize-LSize;
02420   DEBUG(dbgs() << "Selecting best pivot: \n"
02421                << "First: " << First << ", Last: " << Last <<'\n'
02422                << "LSize: " << LSize << ", RSize: " << RSize << '\n');
02423   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
02424   for (CaseItr I = CR.Range.first, J=I+1, E = CR.Range.second;
02425        J!=E; ++I, ++J) {
02426     const APInt &LEnd = cast<ConstantInt>(I->High)->getValue();
02427     const APInt &RBegin = cast<ConstantInt>(J->Low)->getValue();
02428     APInt Range = ComputeRange(LEnd, RBegin);
02429     assert((Range - 2ULL).isNonNegative() &&
02430            "Invalid case distance");
02431     // Use volatile double here to avoid excess precision issues on some hosts,
02432     // e.g. that use 80-bit X87 registers.
02433     // Only consider the density of sub-ranges that actually have sufficient
02434     // entries to be lowered as a jump table.
02435     volatile double LDensity =
02436         LSize.ult(TLI.getMinimumJumpTableEntries())
02437             ? 0.0
02438             : LSize.roundToDouble() / (LEnd - First + 1ULL).roundToDouble();
02439     volatile double RDensity =
02440         RSize.ult(TLI.getMinimumJumpTableEntries())
02441             ? 0.0
02442             : RSize.roundToDouble() / (Last - RBegin + 1ULL).roundToDouble();
02443     volatile double Metric = Range.logBase2() * (LDensity + RDensity);
02444     // Should always split in some non-trivial place
02445     DEBUG(dbgs() <<"=>Step\n"
02446                  << "LEnd: " << LEnd << ", RBegin: " << RBegin << '\n'
02447                  << "LDensity: " << LDensity
02448                  << ", RDensity: " << RDensity << '\n'
02449                  << "Metric: " << Metric << '\n');
02450     if (FMetric < Metric) {
02451       Pivot = J;
02452       FMetric = Metric;
02453       DEBUG(dbgs() << "Current metric set to: " << FMetric << '\n');
02454     }
02455 
02456     LSize += J->size();
02457     RSize -= J->size();
02458   }
02459 
02460   if (FMetric == 0 || !areJTsAllowed(TLI))
02461     Pivot = CR.Range.first + Size/2;
02462   splitSwitchCase(CR, Pivot, WorkList, SV, SwitchBB);
02463   return true;
02464 }
02465 
02466 void SelectionDAGBuilder::splitSwitchCase(CaseRec &CR, CaseItr Pivot,
02467                                           CaseRecVector &WorkList,
02468                                           const Value *SV,
02469                                           MachineBasicBlock *SwitchBB) {
02470   // Get the MachineFunction which holds the current MBB.  This is used when
02471   // inserting any additional MBBs necessary to represent the switch.
02472   MachineFunction *CurMF = FuncInfo.MF;
02473 
02474   // Figure out which block is immediately after the current one.
02475   MachineFunction::iterator BBI = CR.CaseBB;
02476   ++BBI;
02477 
02478   const BasicBlock *LLVMBB = CR.CaseBB->getBasicBlock();
02479 
02480   CaseRange LHSR(CR.Range.first, Pivot);
02481   CaseRange RHSR(Pivot, CR.Range.second);
02482   const Constant *C = Pivot->Low;
02483   MachineBasicBlock *FalseBB = nullptr, *TrueBB = nullptr;
02484 
02485   // We know that we branch to the LHS if the Value being switched on is
02486   // less than the Pivot value, C.  We use this to optimize our binary
02487   // tree a bit, by recognizing that if SV is greater than or equal to the
02488   // LHS's Case Value, and that Case Value is exactly one less than the
02489   // Pivot's Value, then we can branch directly to the LHS's Target,
02490   // rather than creating a leaf node for it.
02491   if ((LHSR.second - LHSR.first) == 1 && LHSR.first->High == CR.GE &&
02492       cast<ConstantInt>(C)->getValue() ==
02493           (cast<ConstantInt>(CR.GE)->getValue() + 1LL)) {
02494     TrueBB = LHSR.first->BB;
02495   } else {
02496     TrueBB = CurMF->CreateMachineBasicBlock(LLVMBB);
02497     CurMF->insert(BBI, TrueBB);
02498     WorkList.push_back(CaseRec(TrueBB, C, CR.GE, LHSR));
02499 
02500     // Put SV in a virtual register to make it available from the new blocks.
02501     ExportFromCurrentBlock(SV);
02502   }
02503 
02504   // Similar to the optimization above, if the Value being switched on is
02505   // known to be less than the Constant CR.LT, and the current Case Value
02506   // is CR.LT - 1, then we can branch directly to the target block for
02507   // the current Case Value, rather than emitting a RHS leaf node for it.
02508   if ((RHSR.second - RHSR.first) == 1 && CR.LT &&
02509       cast<ConstantInt>(RHSR.first->Low)->getValue() ==
02510           (cast<ConstantInt>(CR.LT)->getValue() - 1LL)) {
02511     FalseBB = RHSR.first->BB;
02512   } else {
02513     FalseBB = CurMF->CreateMachineBasicBlock(LLVMBB);
02514     CurMF->insert(BBI, FalseBB);
02515     WorkList.push_back(CaseRec(FalseBB, CR.LT, C, RHSR));
02516 
02517     // Put SV in a virtual register to make it available from the new blocks.
02518     ExportFromCurrentBlock(SV);
02519   }
02520 
02521   // Create a CaseBlock record representing a conditional branch to
02522   // the LHS node if the value being switched on SV is less than C.
02523   // Otherwise, branch to LHS.
02524   CaseBlock CB(ISD::SETLT, SV, C, nullptr, TrueBB, FalseBB, CR.CaseBB);
02525 
02526   if (CR.CaseBB == SwitchBB)
02527     visitSwitchCase(CB, SwitchBB);
02528   else
02529     SwitchCases.push_back(CB);
02530 }
02531 
02532 /// handleBitTestsSwitchCase - if current case range has few destination and
02533 /// range span less, than machine word bitwidth, encode case range into series
02534 /// of masks and emit bit tests with these masks.
02535 bool SelectionDAGBuilder::handleBitTestsSwitchCase(CaseRec& CR,
02536                                                    CaseRecVector& WorkList,
02537                                                    const Value* SV,
02538                                                    MachineBasicBlock* Default,
02539                                                    MachineBasicBlock* SwitchBB) {
02540   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
02541   EVT PTy = TLI.getPointerTy();
02542   unsigned IntPtrBits = PTy.getSizeInBits();
02543 
02544   Case& FrontCase = *CR.Range.first;
02545   Case& BackCase  = *(CR.Range.second-1);
02546 
02547   // Get the MachineFunction which holds the current MBB.  This is used when
02548   // inserting any additional MBBs necessary to represent the switch.
02549   MachineFunction *CurMF = FuncInfo.MF;
02550 
02551   // If target does not have legal shift left, do not emit bit tests at all.
02552   if (!TLI.isOperationLegal(ISD::SHL, PTy))
02553     return false;
02554 
02555   size_t numCmps = 0;
02556   for (CaseItr I = CR.Range.first, E = CR.Range.second; I != E; ++I) {
02557     // Single case counts one, case range - two.
02558     numCmps += (I->Low == I->High ? 1 : 2);
02559   }
02560 
02561   // Count unique destinations
02562   SmallSet<MachineBasicBlock*, 4> Dests;
02563   for (CaseItr I = CR.Range.first, E = CR.Range.second; I != E; ++I) {
02564     Dests.insert(I->BB);
02565     if (Dests.size() > 3)
02566       // Don't bother the code below, if there are too much unique destinations
02567       return false;
02568   }
02569   DEBUG(dbgs() << "Total number of unique destinations: "
02570         << Dests.size() << '\n'
02571         << "Total number of comparisons: " << numCmps << '\n');
02572 
02573   // Compute span of values.
02574   const APInt& minValue = cast<ConstantInt>(FrontCase.Low)->getValue();
02575   const APInt& maxValue = cast<ConstantInt>(BackCase.High)->getValue();
02576   APInt cmpRange = maxValue - minValue;
02577 
02578   DEBUG(dbgs() << "Compare range: " << cmpRange << '\n'
02579                << "Low bound: " << minValue << '\n'
02580                << "High bound: " << maxValue << '\n');
02581 
02582   if (cmpRange.uge(IntPtrBits) ||
02583       (!(Dests.size() == 1 && numCmps >= 3) &&
02584        !(Dests.size() == 2 && numCmps >= 5) &&
02585        !(Dests.size() >= 3 && numCmps >= 6)))
02586     return false;
02587 
02588   DEBUG(dbgs() << "Emitting bit tests\n");
02589   APInt lowBound = APInt::getNullValue(cmpRange.getBitWidth());
02590 
02591   // Optimize the case where all the case values fit in a
02592   // word without having to subtract minValue. In this case,
02593   // we can optimize away the subtraction.
02594   if (minValue.isNonNegative() && maxValue.slt(IntPtrBits)) {
02595     cmpRange = maxValue;
02596   } else {
02597     lowBound = minValue;
02598   }
02599 
02600   CaseBitsVector CasesBits;
02601   unsigned i, count = 0;
02602 
02603   for (CaseItr I = CR.Range.first, E = CR.Range.second; I!=E; ++I) {
02604     MachineBasicBlock* Dest = I->BB;
02605     for (i = 0; i < count; ++i)
02606       if (Dest == CasesBits[i].BB)
02607         break;
02608 
02609     if (i == count) {
02610       assert((count < 3) && "Too much destinations to test!");
02611       CasesBits.push_back(CaseBits(0, Dest, 0, 0/*Weight*/));
02612       count++;
02613     }
02614 
02615     const APInt& lowValue = cast<ConstantInt>(I->Low)->getValue();
02616     const APInt& highValue = cast<ConstantInt>(I->High)->getValue();
02617 
02618     uint64_t lo = (lowValue - lowBound).getZExtValue();
02619     uint64_t hi = (highValue - lowBound).getZExtValue();
02620     CasesBits[i].ExtraWeight += I->ExtraWeight;
02621 
02622     for (uint64_t j = lo; j <= hi; j++) {
02623       CasesBits[i].Mask |=  1ULL << j;
02624       CasesBits[i].Bits++;
02625     }
02626 
02627   }
02628   std::sort(CasesBits.begin(), CasesBits.end(), CaseBitsCmp());
02629 
02630   BitTestInfo BTC;
02631 
02632   // Figure out which block is immediately after the current one.
02633   MachineFunction::iterator BBI = CR.CaseBB;
02634   ++BBI;
02635 
02636   const BasicBlock *LLVMBB = CR.CaseBB->getBasicBlock();
02637 
02638   DEBUG(dbgs() << "Cases:\n");
02639   for (unsigned i = 0, e = CasesBits.size(); i!=e; ++i) {
02640     DEBUG(dbgs() << "Mask: " << CasesBits[i].Mask
02641                  << ", Bits: " << CasesBits[i].Bits
02642                  << ", BB: " << CasesBits[i].BB << '\n');
02643 
02644     MachineBasicBlock *CaseBB = CurMF->CreateMachineBasicBlock(LLVMBB);
02645     CurMF->insert(BBI, CaseBB);
02646     BTC.push_back(BitTestCase(CasesBits[i].Mask,
02647                               CaseBB,
02648                               CasesBits[i].BB, CasesBits[i].ExtraWeight));
02649 
02650     // Put SV in a virtual register to make it available from the new blocks.
02651     ExportFromCurrentBlock(SV);
02652   }
02653 
02654   BitTestBlock BTB(lowBound, cmpRange, SV,
02655                    -1U, MVT::Other, (CR.CaseBB == SwitchBB),
02656                    CR.CaseBB, Default, std::move(BTC));
02657 
02658   if (CR.CaseBB == SwitchBB)
02659     visitBitTestHeader(BTB, SwitchBB);
02660 
02661   BitTestCases.push_back(std::move(BTB));
02662 
02663   return true;
02664 }
02665 
02666 /// Clusterify - Transform simple list of Cases into list of CaseRange's
02667 void SelectionDAGBuilder::Clusterify(CaseVector& Cases,
02668                                      const SwitchInst& SI) {
02669   BranchProbabilityInfo *BPI = FuncInfo.BPI;
02670   // Start with "simple" cases.
02671   for (SwitchInst::ConstCaseIt i : SI.cases()) {
02672     const BasicBlock *SuccBB = i.getCaseSuccessor();
02673     MachineBasicBlock *SMBB = FuncInfo.MBBMap[SuccBB];
02674 
02675     uint32_t ExtraWeight =
02676       BPI ? BPI->getEdgeWeight(SI.getParent(), i.getSuccessorIndex()) : 0;
02677 
02678     Cases.push_back(Case(i.getCaseValue(), i.getCaseValue(),
02679                          SMBB, ExtraWeight));
02680   }
02681   std::sort(Cases.begin(), Cases.end(), CaseCmp());
02682 
02683   // Merge case into clusters
02684   if (Cases.size() >= 2)
02685     // Must recompute end() each iteration because it may be
02686     // invalidated by erase if we hold on to it
02687     for (CaseItr I = Cases.begin(), J = std::next(Cases.begin());
02688          J != Cases.end(); ) {
02689       const APInt& nextValue = cast<ConstantInt>(J->Low)->getValue();
02690       const APInt& currentValue = cast<ConstantInt>(I->High)->getValue();
02691       MachineBasicBlock* nextBB = J->BB;
02692       MachineBasicBlock* currentBB = I->BB;
02693 
02694       // If the two neighboring cases go to the same destination, merge them
02695       // into a single case.
02696       if ((nextValue - currentValue == 1) && (currentBB == nextBB)) {
02697         I->High = J->High;
02698         I->ExtraWeight += J->ExtraWeight;
02699         J = Cases.erase(J);
02700       } else {
02701         I = J++;
02702       }
02703     }
02704 
02705   DEBUG({
02706       size_t numCmps = 0;
02707       for (auto &I : Cases)
02708         // A range counts double, since it requires two compares.
02709         numCmps += I.Low != I.High ? 2 : 1;
02710 
02711       dbgs() << "Clusterify finished. Total clusters: " << Cases.size()
02712              << ". Total compares: " << numCmps << '\n';
02713     });
02714 }
02715 
02716 void SelectionDAGBuilder::UpdateSplitBlock(MachineBasicBlock *First,
02717                                            MachineBasicBlock *Last) {
02718   // Update JTCases.
02719   for (unsigned i = 0, e = JTCases.size(); i != e; ++i)
02720     if (JTCases[i].first.HeaderBB == First)
02721       JTCases[i].first.HeaderBB = Last;
02722 
02723   // Update BitTestCases.
02724   for (unsigned i = 0, e = BitTestCases.size(); i != e; ++i)
02725     if (BitTestCases[i].Parent == First)
02726       BitTestCases[i].Parent = Last;
02727 }
02728 
02729 void SelectionDAGBuilder::visitSwitch(const SwitchInst &SI) {
02730   MachineBasicBlock *SwitchMBB = FuncInfo.MBB;
02731 
02732   // Figure out which block is immediately after the current one.
02733   MachineBasicBlock *NextBlock = nullptr;
02734   if (SwitchMBB + 1 != FuncInfo.MF->end())
02735     NextBlock = SwitchMBB + 1;
02736 
02737 
02738   // Create a vector of Cases, sorted so that we can efficiently create a binary
02739   // search tree from them.
02740   CaseVector Cases;
02741   Clusterify(Cases, SI);
02742 
02743   // Get the default destination MBB.
02744   MachineBasicBlock *Default = FuncInfo.MBBMap[SI.getDefaultDest()];
02745 
02746   if (isa<UnreachableInst>(SI.getDefaultDest()->getFirstNonPHIOrDbg()) &&
02747       !Cases.empty()) {
02748     // Replace an unreachable default destination with the most popular case
02749     // destination.
02750     DenseMap<const BasicBlock *, unsigned> Popularity;
02751     unsigned MaxPop = 0;
02752     const BasicBlock *MaxBB = nullptr;
02753     for (auto I : SI.cases()) {
02754       const BasicBlock *BB = I.getCaseSuccessor();
02755       if (++Popularity[BB] > MaxPop) {
02756         MaxPop = Popularity[BB];
02757         MaxBB = BB;
02758       }
02759     }
02760 
02761     // Set new default.
02762     assert(MaxPop > 0);
02763     assert(MaxBB);
02764     Default = FuncInfo.MBBMap[MaxBB];
02765 
02766     // Remove cases that were pointing to the destination that is now the default.
02767     Cases.erase(std::remove_if(Cases.begin(), Cases.end(),
02768                                [&](const Case &C) { return C.BB == Default; }),
02769                 Cases.end());
02770   }
02771 
02772   // If there is only the default destination, go there directly.
02773   if (Cases.empty()) {
02774     // Update machine-CFG edges.
02775     SwitchMBB->addSuccessor(Default);
02776 
02777     // If this is not a fall-through branch, emit the branch.
02778     if (Default != NextBlock) {
02779       DAG.setRoot(DAG.getNode(ISD::BR, getCurSDLoc(), MVT::Other,
02780                               getControlRoot(), DAG.getBasicBlock(Default)));
02781     }
02782     return;
02783   }
02784 
02785   // Get the Value to be switched on.
02786   const Value *SV = SI.getCondition();
02787 
02788   // Push the initial CaseRec onto the worklist
02789   CaseRecVector WorkList;
02790   WorkList.push_back(CaseRec(SwitchMBB,nullptr,nullptr,
02791                              CaseRange(Cases.begin(),Cases.end())));
02792 
02793   while (!WorkList.empty()) {
02794     // Grab a record representing a case range to process off the worklist
02795     CaseRec CR = WorkList.back();
02796     WorkList.pop_back();
02797 
02798     if (handleBitTestsSwitchCase(CR, WorkList, SV, Default, SwitchMBB))
02799       continue;
02800 
02801     // If the range has few cases (two or less) emit a series of specific
02802     // tests.
02803     if (handleSmallSwitchRange(CR, WorkList, SV, Default, SwitchMBB))
02804       continue;
02805 
02806     // If the switch has more than N blocks, and is at least 40% dense, and the
02807     // target supports indirect branches, then emit a jump table rather than
02808     // lowering the switch to a binary tree of conditional branches.
02809     // N defaults to 4 and is controlled via TLS.getMinimumJumpTableEntries().
02810     if (handleJTSwitchCase(CR, WorkList, SV, Default, SwitchMBB))
02811       continue;
02812 
02813     // Emit binary tree. We need to pick a pivot, and push left and right ranges
02814     // onto the worklist. Leafs are handled via handleSmallSwitchRange() call.
02815     handleBTSplitSwitchCase(CR, WorkList, SV, SwitchMBB);
02816   }
02817 }
02818 
02819 void SelectionDAGBuilder::visitIndirectBr(const IndirectBrInst &I) {
02820   MachineBasicBlock *IndirectBrMBB = FuncInfo.MBB;
02821 
02822   // Update machine-CFG edges with unique successors.
02823   SmallSet<BasicBlock*, 32> Done;
02824   for (unsigned i = 0, e = I.getNumSuccessors(); i != e; ++i) {
02825     BasicBlock *BB = I.getSuccessor(i);
02826     bool Inserted = Done.insert(BB).second;
02827     if (!Inserted)
02828         continue;
02829 
02830     MachineBasicBlock *Succ = FuncInfo.MBBMap[BB];
02831     addSuccessorWithWeight(IndirectBrMBB, Succ);
02832   }
02833 
02834   DAG.setRoot(DAG.getNode(ISD::BRIND, getCurSDLoc(),
02835                           MVT::Other, getControlRoot(),
02836                           getValue(I.getAddress())));
02837 }
02838 
02839 void SelectionDAGBuilder::visitUnreachable(const UnreachableInst &I) {
02840   if (DAG.getTarget().Options.TrapUnreachable)
02841     DAG.setRoot(DAG.getNode(ISD::TRAP, getCurSDLoc(), MVT::Other, DAG.getRoot()));
02842 }
02843 
02844 void SelectionDAGBuilder::visitFSub(const User &I) {
02845   // -0.0 - X --> fneg
02846   Type *Ty = I.getType();
02847   if (isa<Constant>(I.getOperand(0)) &&
02848       I.getOperand(0) == ConstantFP::getZeroValueForNegation(Ty)) {
02849     SDValue Op2 = getValue(I.getOperand(1));
02850     setValue(&I, DAG.getNode(ISD::FNEG, getCurSDLoc(),
02851                              Op2.getValueType(), Op2));
02852     return;
02853   }
02854 
02855   visitBinary(I, ISD::FSUB);
02856 }
02857 
02858 void SelectionDAGBuilder::visitBinary(const User &I, unsigned OpCode) {
02859   SDValue Op1 = getValue(I.getOperand(0));
02860   SDValue Op2 = getValue(I.getOperand(1));
02861 
02862   bool nuw = false;
02863   bool nsw = false;
02864   bool exact = false;
02865   if (const OverflowingBinaryOperator *OFBinOp =
02866           dyn_cast<const OverflowingBinaryOperator>(&I)) {
02867     nuw = OFBinOp->hasNoUnsignedWrap();
02868     nsw = OFBinOp->hasNoSignedWrap();
02869   }
02870   if (const PossiblyExactOperator *ExactOp =
02871           dyn_cast<const PossiblyExactOperator>(&I))
02872     exact = ExactOp->isExact();
02873 
02874   SDValue BinNodeValue = DAG.getNode(OpCode, getCurSDLoc(), Op1.getValueType(),
02875                                      Op1, Op2, nuw, nsw, exact);
02876   setValue(&I, BinNodeValue);
02877 }
02878 
02879 void SelectionDAGBuilder::visitShift(const User &I, unsigned Opcode) {
02880   SDValue Op1 = getValue(I.getOperand(0));
02881   SDValue Op2 = getValue(I.getOperand(1));
02882 
02883   EVT ShiftTy =
02884       DAG.getTargetLoweringInfo().getShiftAmountTy(Op2.getValueType());
02885 
02886   // Coerce the shift amount to the right type if we can.
02887   if (!I.getType()->isVectorTy() && Op2.getValueType() != ShiftTy) {
02888     unsigned ShiftSize = ShiftTy.getSizeInBits();
02889     unsigned Op2Size = Op2.getValueType().getSizeInBits();
02890     SDLoc DL = getCurSDLoc();
02891 
02892     // If the operand is smaller than the shift count type, promote it.
02893     if (ShiftSize > Op2Size)
02894       Op2 = DAG.getNode(ISD::ZERO_EXTEND, DL, ShiftTy, Op2);
02895 
02896     // If the operand is larger than the shift count type but the shift
02897     // count type has enough bits to represent any shift value, truncate
02898     // it now. This is a common case and it exposes the truncate to
02899     // optimization early.
02900     else if (ShiftSize >= Log2_32_Ceil(Op2.getValueType().getSizeInBits()))
02901       Op2 = DAG.getNode(ISD::TRUNCATE, DL, ShiftTy, Op2);
02902     // Otherwise we'll need to temporarily settle for some other convenient
02903     // type.  Type legalization will make adjustments once the shiftee is split.
02904     else
02905       Op2 = DAG.getZExtOrTrunc(Op2, DL, MVT::i32);
02906   }
02907 
02908   bool nuw = false;
02909   bool nsw = false;
02910   bool exact = false;
02911 
02912   if (Opcode == ISD::SRL || Opcode == ISD::SRA || Opcode == ISD::SHL) {
02913 
02914     if (const OverflowingBinaryOperator *OFBinOp =
02915             dyn_cast<const OverflowingBinaryOperator>(&I)) {
02916       nuw = OFBinOp->hasNoUnsignedWrap();
02917       nsw = OFBinOp->hasNoSignedWrap();
02918     }
02919     if (const PossiblyExactOperator *ExactOp =
02920             dyn_cast<const PossiblyExactOperator>(&I))
02921       exact = ExactOp->isExact();
02922   }
02923 
02924   SDValue Res = DAG.getNode(Opcode, getCurSDLoc(), Op1.getValueType(), Op1, Op2,
02925                             nuw, nsw, exact);
02926   setValue(&I, Res);
02927 }
02928 
02929 void SelectionDAGBuilder::visitSDiv(const User &I) {
02930   SDValue Op1 = getValue(I.getOperand(0));
02931   SDValue Op2 = getValue(I.getOperand(1));
02932 
02933   // Turn exact SDivs into multiplications.
02934   // FIXME: This should be in DAGCombiner, but it doesn't have access to the
02935   // exact bit.
02936   if (isa<BinaryOperator>(&I) && cast<BinaryOperator>(&I)->isExact() &&
02937       !isa<ConstantSDNode>(Op1) &&
02938       isa<ConstantSDNode>(Op2) && !cast<ConstantSDNode>(Op2)->isNullValue())
02939     setValue(&I, DAG.getTargetLoweringInfo()
02940                      .BuildExactSDIV(Op1, Op2, getCurSDLoc(), DAG));
02941   else
02942     setValue(&I, DAG.getNode(ISD::SDIV, getCurSDLoc(), Op1.getValueType(),
02943                              Op1, Op2));
02944 }
02945 
02946 void SelectionDAGBuilder::visitICmp(const User &I) {
02947   ICmpInst::Predicate predicate = ICmpInst::BAD_ICMP_PREDICATE;
02948   if (const ICmpInst *IC = dyn_cast<ICmpInst>(&I))
02949     predicate = IC->getPredicate();
02950   else if (const ConstantExpr *IC = dyn_cast<ConstantExpr>(&I))
02951     predicate = ICmpInst::Predicate(IC->getPredicate());
02952   SDValue Op1 = getValue(I.getOperand(0));
02953   SDValue Op2 = getValue(I.getOperand(1));
02954   ISD::CondCode Opcode = getICmpCondCode(predicate);
02955 
02956   EVT DestVT = DAG.getTargetLoweringInfo().getValueType(I.getType());
02957   setValue(&I, DAG.getSetCC(getCurSDLoc(), DestVT, Op1, Op2, Opcode));
02958 }
02959 
02960 void SelectionDAGBuilder::visitFCmp(const User &I) {
02961   FCmpInst::Predicate predicate = FCmpInst::BAD_FCMP_PREDICATE;
02962   if (const FCmpInst *FC = dyn_cast<FCmpInst>(&I))
02963     predicate = FC->getPredicate();
02964   else if (const ConstantExpr *FC = dyn_cast<ConstantExpr>(&I))
02965     predicate = FCmpInst::Predicate(FC->getPredicate());
02966   SDValue Op1 = getValue(I.getOperand(0));
02967   SDValue Op2 = getValue(I.getOperand(1));
02968   ISD::CondCode Condition = getFCmpCondCode(predicate);
02969   if (TM.Options.NoNaNsFPMath)
02970     Condition = getFCmpCodeWithoutNaN(Condition);
02971   EVT DestVT = DAG.getTargetLoweringInfo().getValueType(I.getType());
02972   setValue(&I, DAG.getSetCC(getCurSDLoc(), DestVT, Op1, Op2, Condition));
02973 }
02974 
02975 void SelectionDAGBuilder::visitSelect(const User &I) {
02976   SmallVector<EVT, 4> ValueVTs;
02977   ComputeValueVTs(DAG.getTargetLoweringInfo(), I.getType(), ValueVTs);
02978   unsigned NumValues = ValueVTs.size();
02979   if (NumValues == 0) return;
02980 
02981   SmallVector<SDValue, 4> Values(NumValues);
02982   SDValue Cond     = getValue(I.getOperand(0));
02983   SDValue TrueVal  = getValue(I.getOperand(1));
02984   SDValue FalseVal = getValue(I.getOperand(2));
02985   ISD::NodeType OpCode = Cond.getValueType().isVector() ?
02986     ISD::VSELECT : ISD::SELECT;
02987 
02988   for (unsigned i = 0; i != NumValues; ++i)
02989     Values[i] = DAG.getNode(OpCode, getCurSDLoc(),
02990                             TrueVal.getNode()->getValueType(TrueVal.getResNo()+i),
02991                             Cond,
02992                             SDValue(TrueVal.getNode(),
02993                                     TrueVal.getResNo() + i),
02994                             SDValue(FalseVal.getNode(),
02995                                     FalseVal.getResNo() + i));
02996 
02997   setValue(&I, DAG.getNode(ISD::MERGE_VALUES, getCurSDLoc(),
02998                            DAG.getVTList(ValueVTs), Values));
02999 }
03000 
03001 void SelectionDAGBuilder::visitTrunc(const User &I) {
03002   // TruncInst cannot be a no-op cast because sizeof(src) > sizeof(dest).
03003   SDValue N = getValue(I.getOperand(0));
03004   EVT DestVT = DAG.getTargetLoweringInfo().getValueType(I.getType());
03005   setValue(&I, DAG.getNode(ISD::TRUNCATE, getCurSDLoc(), DestVT, N));
03006 }
03007 
03008 void SelectionDAGBuilder::visitZExt(const User &I) {
03009   // ZExt cannot be a no-op cast because sizeof(src) < sizeof(dest).
03010   // ZExt also can't be a cast to bool for same reason. So, nothing much to do
03011   SDValue N = getValue(I.getOperand(0));
03012   EVT DestVT = DAG.getTargetLoweringInfo().getValueType(I.getType());
03013   setValue(&I, DAG.getNode(ISD::ZERO_EXTEND, getCurSDLoc(), DestVT, N));
03014 }
03015 
03016 void SelectionDAGBuilder::visitSExt(const User &I) {
03017   // SExt cannot be a no-op cast because sizeof(src) < sizeof(dest).
03018   // SExt also can't be a cast to bool for same reason. So, nothing much to do
03019   SDValue N = getValue(I.getOperand(0));
03020   EVT DestVT = DAG.getTargetLoweringInfo().getValueType(I.getType());
03021   setValue(&I, DAG.getNode(ISD::SIGN_EXTEND, getCurSDLoc(), DestVT, N));
03022 }
03023 
03024 void SelectionDAGBuilder::visitFPTrunc(const User &I) {
03025   // FPTrunc is never a no-op cast, no need to check
03026   SDValue N = getValue(I.getOperand(0));
03027   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
03028   EVT DestVT = TLI.getValueType(I.getType());
03029   setValue(&I, DAG.getNode(ISD::FP_ROUND, getCurSDLoc(), DestVT, N,
03030                            DAG.getTargetConstant(0, TLI.getPointerTy())));
03031 }
03032 
03033 void SelectionDAGBuilder::visitFPExt(const User &I) {
03034   // FPExt is never a no-op cast, no need to check
03035   SDValue N = getValue(I.getOperand(0));
03036   EVT DestVT = DAG.getTargetLoweringInfo().getValueType(I.getType());
03037   setValue(&I, DAG.getNode(ISD::FP_EXTEND, getCurSDLoc(), DestVT, N));
03038 }
03039 
03040 void SelectionDAGBuilder::visitFPToUI(const User &I) {
03041   // FPToUI is never a no-op cast, no need to check
03042   SDValue N = getValue(I.getOperand(0));
03043   EVT DestVT = DAG.getTargetLoweringInfo().getValueType(I.getType());
03044   setValue(&I, DAG.getNode(ISD::FP_TO_UINT, getCurSDLoc(), DestVT, N));
03045 }
03046 
03047 void SelectionDAGBuilder::visitFPToSI(const User &I) {
03048   // FPToSI is never a no-op cast, no need to check
03049   SDValue N = getValue(I.getOperand(0));
03050   EVT DestVT = DAG.getTargetLoweringInfo().getValueType(I.getType());
03051   setValue(&I, DAG.getNode(ISD::FP_TO_SINT, getCurSDLoc(), DestVT, N));
03052 }
03053 
03054 void SelectionDAGBuilder::visitUIToFP(const User &I) {
03055   // UIToFP is never a no-op cast, no need to check
03056   SDValue N = getValue(I.getOperand(0));
03057   EVT DestVT = DAG.getTargetLoweringInfo().getValueType(I.getType());
03058   setValue(&I, DAG.getNode(ISD::UINT_TO_FP, getCurSDLoc(), DestVT, N));
03059 }
03060 
03061 void SelectionDAGBuilder::visitSIToFP(const User &I) {
03062   // SIToFP is never a no-op cast, no need to check
03063   SDValue N = getValue(I.getOperand(0));
03064   EVT DestVT = DAG.getTargetLoweringInfo().getValueType(I.getType());
03065   setValue(&I, DAG.getNode(ISD::SINT_TO_FP, getCurSDLoc(), DestVT, N));
03066 }
03067 
03068 void SelectionDAGBuilder::visitPtrToInt(const User &I) {
03069   // What to do depends on the size of the integer and the size of the pointer.
03070   // We can either truncate, zero extend, or no-op, accordingly.
03071   SDValue N = getValue(I.getOperand(0));
03072   EVT DestVT = DAG.getTargetLoweringInfo().getValueType(I.getType());
03073   setValue(&I, DAG.getZExtOrTrunc(N, getCurSDLoc(), DestVT));
03074 }
03075 
03076 void SelectionDAGBuilder::visitIntToPtr(const User &I) {
03077   // What to do depends on the size of the integer and the size of the pointer.
03078   // We can either truncate, zero extend, or no-op, accordingly.
03079   SDValue N = getValue(I.getOperand(0));
03080   EVT DestVT = DAG.getTargetLoweringInfo().getValueType(I.getType());
03081   setValue(&I, DAG.getZExtOrTrunc(N, getCurSDLoc(), DestVT));
03082 }
03083 
03084 void SelectionDAGBuilder::visitBitCast(const User &I) {
03085   SDValue N = getValue(I.getOperand(0));
03086   EVT DestVT = DAG.getTargetLoweringInfo().getValueType(I.getType());
03087 
03088   // BitCast assures us that source and destination are the same size so this is
03089   // either a BITCAST or a no-op.
03090   if (DestVT != N.getValueType())
03091     setValue(&I, DAG.getNode(ISD::BITCAST, getCurSDLoc(),
03092                              DestVT, N)); // convert types.
03093   // Check if the original LLVM IR Operand was a ConstantInt, because getValue()
03094   // might fold any kind of constant expression to an integer constant and that
03095   // is not what we are looking for. Only regcognize a bitcast of a genuine
03096   // constant integer as an opaque constant.
03097   else if(ConstantInt *C = dyn_cast<ConstantInt>(I.getOperand(0)))
03098     setValue(&I, DAG.getConstant(C->getValue(), DestVT, /*isTarget=*/false,
03099                                  /*isOpaque*/true));
03100   else
03101     setValue(&I, N);            // noop cast.
03102 }
03103 
03104 void SelectionDAGBuilder::visitAddrSpaceCast(const User &I) {
03105   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
03106   const Value *SV = I.getOperand(0);
03107   SDValue N = getValue(SV);
03108   EVT DestVT = TLI.getValueType(I.getType());
03109 
03110   unsigned SrcAS = SV->getType()->getPointerAddressSpace();
03111   unsigned DestAS = I.getType()->getPointerAddressSpace();
03112 
03113   if (!TLI.isNoopAddrSpaceCast(SrcAS, DestAS))
03114     N = DAG.getAddrSpaceCast(getCurSDLoc(), DestVT, N, SrcAS, DestAS);
03115 
03116   setValue(&I, N);
03117 }
03118 
03119 void SelectionDAGBuilder::visitInsertElement(const User &I) {
03120   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
03121   SDValue InVec = getValue(I.getOperand(0));
03122   SDValue InVal = getValue(I.getOperand(1));
03123   SDValue InIdx = DAG.getSExtOrTrunc(getValue(I.getOperand(2)),
03124                                      getCurSDLoc(), TLI.getVectorIdxTy());
03125   setValue(&I, DAG.getNode(ISD::INSERT_VECTOR_ELT, getCurSDLoc(),
03126                            TLI.getValueType(I.getType()), InVec, InVal, InIdx));
03127 }
03128 
03129 void SelectionDAGBuilder::visitExtractElement(const User &I) {
03130   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
03131   SDValue InVec = getValue(I.getOperand(0));
03132   SDValue InIdx = DAG.getSExtOrTrunc(getValue(I.getOperand(1)),
03133                                      getCurSDLoc(), TLI.getVectorIdxTy());
03134   setValue(&I, DAG.getNode(ISD::EXTRACT_VECTOR_ELT, getCurSDLoc(),
03135                            TLI.getValueType(I.getType()), InVec, InIdx));
03136 }
03137 
03138 // Utility for visitShuffleVector - Return true if every element in Mask,
03139 // beginning from position Pos and ending in Pos+Size, falls within the
03140 // specified sequential range [L, L+Pos). or is undef.
03141 static bool isSequentialInRange(const SmallVectorImpl<int> &Mask,
03142                                 unsigned Pos, unsigned Size, int Low) {
03143   for (unsigned i = Pos, e = Pos+Size; i != e; ++i, ++Low)
03144     if (Mask[i] >= 0 && Mask[i] != Low)
03145       return false;
03146   return true;
03147 }
03148 
03149 void SelectionDAGBuilder::visitShuffleVector(const User &I) {
03150   SDValue Src1 = getValue(I.getOperand(0));
03151   SDValue Src2 = getValue(I.getOperand(1));
03152 
03153   SmallVector<int, 8> Mask;
03154   ShuffleVectorInst::getShuffleMask(cast<Constant>(I.getOperand(2)), Mask);
03155   unsigned MaskNumElts = Mask.size();
03156 
03157   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
03158   EVT VT = TLI.getValueType(I.getType());
03159   EVT SrcVT = Src1.getValueType();
03160   unsigned SrcNumElts = SrcVT.getVectorNumElements();
03161 
03162   if (SrcNumElts == MaskNumElts) {
03163     setValue(&I, DAG.getVectorShuffle(VT, getCurSDLoc(), Src1, Src2,
03164                                       &Mask[0]));
03165     return;
03166   }
03167 
03168   // Normalize the shuffle vector since mask and vector length don't match.
03169   if (SrcNumElts < MaskNumElts && MaskNumElts % SrcNumElts == 0) {
03170     // Mask is longer than the source vectors and is a multiple of the source
03171     // vectors.  We can use concatenate vector to make the mask and vectors
03172     // lengths match.
03173     if (SrcNumElts*2 == MaskNumElts) {
03174       // First check for Src1 in low and Src2 in high
03175       if (isSequentialInRange(Mask, 0, SrcNumElts, 0) &&
03176           isSequentialInRange(Mask, SrcNumElts, SrcNumElts, SrcNumElts)) {
03177         // The shuffle is concatenating two vectors together.
03178         setValue(&I, DAG.getNode(ISD::CONCAT_VECTORS, getCurSDLoc(),
03179                                  VT, Src1, Src2));
03180         return;
03181       }
03182       // Then check for Src2 in low and Src1 in high
03183       if (isSequentialInRange(Mask, 0, SrcNumElts, SrcNumElts) &&
03184           isSequentialInRange(Mask, SrcNumElts, SrcNumElts, 0)) {
03185         // The shuffle is concatenating two vectors together.
03186         setValue(&I, DAG.getNode(ISD::CONCAT_VECTORS, getCurSDLoc(),
03187                                  VT, Src2, Src1));
03188         return;
03189       }
03190     }
03191 
03192     // Pad both vectors with undefs to make them the same length as the mask.
03193     unsigned NumConcat = MaskNumElts / SrcNumElts;
03194     bool Src1U = Src1.getOpcode() == ISD::UNDEF;
03195     bool Src2U = Src2.getOpcode() == ISD::UNDEF;
03196     SDValue UndefVal = DAG.getUNDEF(SrcVT);
03197 
03198     SmallVector<SDValue, 8> MOps1(NumConcat, UndefVal);
03199     SmallVector<SDValue, 8> MOps2(NumConcat, UndefVal);
03200     MOps1[0] = Src1;
03201     MOps2[0] = Src2;
03202 
03203     Src1 = Src1U ? DAG.getUNDEF(VT) : DAG.getNode(ISD::CONCAT_VECTORS,
03204                                                   getCurSDLoc(), VT, MOps1);
03205     Src2 = Src2U ? DAG.getUNDEF(VT) : DAG.getNode(ISD::CONCAT_VECTORS,
03206                                                   getCurSDLoc(), VT, MOps2);
03207 
03208     // Readjust mask for new input vector length.
03209     SmallVector<int, 8> MappedOps;
03210     for (unsigned i = 0; i != MaskNumElts; ++i) {
03211       int Idx = Mask[i];
03212       if (Idx >= (int)SrcNumElts)
03213         Idx -= SrcNumElts - MaskNumElts;
03214       MappedOps.push_back(Idx);
03215     }
03216 
03217     setValue(&I, DAG.getVectorShuffle(VT, getCurSDLoc(), Src1, Src2,
03218                                       &MappedOps[0]));
03219     return;
03220   }
03221 
03222   if (SrcNumElts > MaskNumElts) {
03223     // Analyze the access pattern of the vector to see if we can extract
03224     // two subvectors and do the shuffle. The analysis is done by calculating
03225     // the range of elements the mask access on both vectors.
03226     int MinRange[2] = { static_cast<int>(SrcNumElts),
03227                         static_cast<int>(SrcNumElts)};
03228     int MaxRange[2] = {-1, -1};
03229 
03230     for (unsigned i = 0; i != MaskNumElts; ++i) {
03231       int Idx = Mask[i];
03232       unsigned Input = 0;
03233       if (Idx < 0)
03234         continue;
03235 
03236       if (Idx >= (int)SrcNumElts) {
03237         Input = 1;
03238         Idx -= SrcNumElts;
03239       }
03240       if (Idx > MaxRange[Input])
03241         MaxRange[Input] = Idx;
03242       if (Idx < MinRange[Input])
03243         MinRange[Input] = Idx;
03244     }
03245 
03246     // Check if the access is smaller than the vector size and can we find
03247     // a reasonable extract index.
03248     int RangeUse[2] = { -1, -1 };  // 0 = Unused, 1 = Extract, -1 = Can not
03249                                    // Extract.
03250     int StartIdx[2];  // StartIdx to extract from
03251     for (unsigned Input = 0; Input < 2; ++Input) {
03252       if (MinRange[Input] >= (int)SrcNumElts && MaxRange[Input] < 0) {
03253         RangeUse[Input] = 0; // Unused
03254         StartIdx[Input] = 0;
03255         continue;
03256       }
03257 
03258       // Find a good start index that is a multiple of the mask length. Then
03259       // see if the rest of the elements are in range.
03260       StartIdx[Input] = (MinRange[Input]/MaskNumElts)*MaskNumElts;
03261       if (MaxRange[Input] - StartIdx[Input] < (int)MaskNumElts &&
03262           StartIdx[Input] + MaskNumElts <= SrcNumElts)
03263         RangeUse[Input] = 1; // Extract from a multiple of the mask length.
03264     }
03265 
03266     if (RangeUse[0] == 0 && RangeUse[1] == 0) {
03267       setValue(&I, DAG.getUNDEF(VT)); // Vectors are not used.
03268       return;
03269     }
03270     if (RangeUse[0] >= 0 && RangeUse[1] >= 0) {
03271       // Extract appropriate subvector and generate a vector shuffle
03272       for (unsigned Input = 0; Input < 2; ++Input) {
03273         SDValue &Src = Input == 0 ? Src1 : Src2;
03274         if (RangeUse[Input] == 0)
03275           Src = DAG.getUNDEF(VT);
03276         else
03277           Src = DAG.getNode(
03278               ISD::EXTRACT_SUBVECTOR, getCurSDLoc(), VT, Src,
03279               DAG.getConstant(StartIdx[Input], TLI.getVectorIdxTy()));
03280       }
03281 
03282       // Calculate new mask.
03283       SmallVector<int, 8> MappedOps;
03284       for (unsigned i = 0; i != MaskNumElts; ++i) {
03285         int Idx = Mask[i];
03286         if (Idx >= 0) {
03287           if (Idx < (int)SrcNumElts)
03288             Idx -= StartIdx[0];
03289           else
03290             Idx -= SrcNumElts + StartIdx[1] - MaskNumElts;
03291         }
03292         MappedOps.push_back(Idx);
03293       }
03294 
03295       setValue(&I, DAG.getVectorShuffle(VT, getCurSDLoc(), Src1, Src2,
03296                                         &MappedOps[0]));
03297       return;
03298     }
03299   }
03300 
03301   // We can't use either concat vectors or extract subvectors so fall back to
03302   // replacing the shuffle with extract and build vector.
03303   // to insert and build vector.
03304   EVT EltVT = VT.getVectorElementType();
03305   EVT IdxVT = TLI.getVectorIdxTy();
03306   SmallVector<SDValue,8> Ops;
03307   for (unsigned i = 0; i != MaskNumElts; ++i) {
03308     int Idx = Mask[i];
03309     SDValue Res;
03310 
03311     if (Idx < 0) {
03312       Res = DAG.getUNDEF(EltVT);
03313     } else {
03314       SDValue &Src = Idx < (int)SrcNumElts ? Src1 : Src2;
03315       if (Idx >= (int)SrcNumElts) Idx -= SrcNumElts;
03316 
03317       Res = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, getCurSDLoc(),
03318                         EltVT, Src, DAG.getConstant(Idx, IdxVT));
03319     }
03320 
03321     Ops.push_back(Res);
03322   }
03323 
03324   setValue(&I, DAG.getNode(ISD::BUILD_VECTOR, getCurSDLoc(), VT, Ops));
03325 }
03326 
03327 void SelectionDAGBuilder::visitInsertValue(const InsertValueInst &I) {
03328   const Value *Op0 = I.getOperand(0);
03329   const Value *Op1 = I.getOperand(1);
03330   Type *AggTy = I.getType();
03331   Type *ValTy = Op1->getType();
03332   bool IntoUndef = isa<UndefValue>(Op0);
03333   bool FromUndef = isa<UndefValue>(Op1);
03334 
03335   unsigned LinearIndex = ComputeLinearIndex(AggTy, I.getIndices());
03336 
03337   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
03338   SmallVector<EVT, 4> AggValueVTs;
03339   ComputeValueVTs(TLI, AggTy, AggValueVTs);
03340   SmallVector<EVT, 4> ValValueVTs;
03341   ComputeValueVTs(TLI, ValTy, ValValueVTs);
03342 
03343   unsigned NumAggValues = AggValueVTs.size();
03344   unsigned NumValValues = ValValueVTs.size();
03345   SmallVector<SDValue, 4> Values(NumAggValues);
03346 
03347   // Ignore an insertvalue that produces an empty object
03348   if (!NumAggValues) {
03349     setValue(&I, DAG.getUNDEF(MVT(MVT::Other)));
03350     return;
03351   }
03352 
03353   SDValue Agg = getValue(Op0);
03354   unsigned i = 0;
03355   // Copy the beginning value(s) from the original aggregate.
03356   for (; i != LinearIndex; ++i)
03357     Values[i] = IntoUndef ? DAG.getUNDEF(AggValueVTs[i]) :
03358                 SDValue(Agg.getNode(), Agg.getResNo() + i);
03359   // Copy values from the inserted value(s).
03360   if (NumValValues) {
03361     SDValue Val = getValue(Op1);
03362     for (; i != LinearIndex + NumValValues; ++i)
03363       Values[i] = FromUndef ? DAG.getUNDEF(AggValueVTs[i]) :
03364                   SDValue(Val.getNode(), Val.getResNo() + i - LinearIndex);
03365   }
03366   // Copy remaining value(s) from the original aggregate.
03367   for (; i != NumAggValues; ++i)
03368     Values[i] = IntoUndef ? DAG.getUNDEF(AggValueVTs[i]) :
03369                 SDValue(Agg.getNode(), Agg.getResNo() + i);
03370 
03371   setValue(&I, DAG.getNode(ISD::MERGE_VALUES, getCurSDLoc(),
03372                            DAG.getVTList(AggValueVTs), Values));
03373 }
03374 
03375 void SelectionDAGBuilder::visitExtractValue(const ExtractValueInst &I) {
03376   const Value *Op0 = I.getOperand(0);
03377   Type *AggTy = Op0->getType();
03378   Type *ValTy = I.getType();
03379   bool OutOfUndef = isa<UndefValue>(Op0);
03380 
03381   unsigned LinearIndex = ComputeLinearIndex(AggTy, I.getIndices());
03382 
03383   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
03384   SmallVector<EVT, 4> ValValueVTs;
03385   ComputeValueVTs(TLI, ValTy, ValValueVTs);
03386 
03387   unsigned NumValValues = ValValueVTs.size();
03388 
03389   // Ignore a extractvalue that produces an empty object
03390   if (!NumValValues) {
03391     setValue(&I, DAG.getUNDEF(MVT(MVT::Other)));
03392     return;
03393   }
03394 
03395   SmallVector<SDValue, 4> Values(NumValValues);
03396 
03397   SDValue Agg = getValue(Op0);
03398   // Copy out the selected value(s).
03399   for (unsigned i = LinearIndex; i != LinearIndex + NumValValues; ++i)
03400     Values[i - LinearIndex] =
03401       OutOfUndef ?
03402         DAG.getUNDEF(Agg.getNode()->getValueType(Agg.getResNo() + i)) :
03403         SDValue(Agg.getNode(), Agg.getResNo() + i);
03404 
03405   setValue(&I, DAG.getNode(ISD::MERGE_VALUES, getCurSDLoc(),
03406                            DAG.getVTList(ValValueVTs), Values));
03407 }
03408 
03409 void SelectionDAGBuilder::visitGetElementPtr(const User &I) {
03410   Value *Op0 = I.getOperand(0);
03411   // Note that the pointer operand may be a vector of pointers. Take the scalar
03412   // element which holds a pointer.
03413   Type *Ty = Op0->getType()->getScalarType();
03414   unsigned AS = Ty->getPointerAddressSpace();
03415   SDValue N = getValue(Op0);
03416 
03417   for (GetElementPtrInst::const_op_iterator OI = I.op_begin()+1, E = I.op_end();
03418        OI != E; ++OI) {
03419     const Value *Idx = *OI;
03420     if (StructType *StTy = dyn_cast<StructType>(Ty)) {
03421       unsigned Field = cast<Constant>(Idx)->getUniqueInteger().getZExtValue();
03422       if (Field) {
03423         // N = N + Offset
03424         uint64_t Offset = DL->getStructLayout(StTy)->getElementOffset(Field);
03425         N = DAG.getNode(ISD::ADD, getCurSDLoc(), N.getValueType(), N,
03426                         DAG.getConstant(Offset, N.getValueType()));
03427       }
03428 
03429       Ty = StTy->getElementType(Field);
03430     } else {
03431       Ty = cast<SequentialType>(Ty)->getElementType();
03432 
03433       // If this is a constant subscript, handle it quickly.
03434       const TargetLowering &TLI = DAG.getTargetLoweringInfo();
03435       if (const ConstantInt *CI = dyn_cast<ConstantInt>(Idx)) {
03436         if (CI->isZero()) continue;
03437         uint64_t Offs =
03438             DL->getTypeAllocSize(Ty)*cast<ConstantInt>(CI)->getSExtValue();
03439         SDValue OffsVal;
03440         EVT PTy = TLI.getPointerTy(AS);
03441         unsigned PtrBits = PTy.getSizeInBits();
03442         if (PtrBits < 64)
03443           OffsVal = DAG.getNode(ISD::TRUNCATE, getCurSDLoc(), PTy,
03444                                 DAG.getConstant(Offs, MVT::i64));
03445         else
03446           OffsVal = DAG.getConstant(Offs, PTy);
03447 
03448         N = DAG.getNode(ISD::ADD, getCurSDLoc(), N.getValueType(), N,
03449                         OffsVal);
03450         continue;
03451       }
03452 
03453       // N = N + Idx * ElementSize;
03454       APInt ElementSize =
03455           APInt(TLI.getPointerSizeInBits(AS), DL->getTypeAllocSize(Ty));
03456       SDValue IdxN = getValue(Idx);
03457 
03458       // If the index is smaller or larger than intptr_t, truncate or extend
03459       // it.
03460       IdxN = DAG.getSExtOrTrunc(IdxN, getCurSDLoc(), N.getValueType());
03461 
03462       // If this is a multiply by a power of two, turn it into a shl
03463       // immediately.  This is a very common case.
03464       if (ElementSize != 1) {
03465         if (ElementSize.isPowerOf2()) {
03466           unsigned Amt = ElementSize.logBase2();
03467           IdxN = DAG.getNode(ISD::SHL, getCurSDLoc(),
03468                              N.getValueType(), IdxN,
03469                              DAG.getConstant(Amt, IdxN.getValueType()));
03470         } else {
03471           SDValue Scale = DAG.getConstant(ElementSize, IdxN.getValueType());
03472           IdxN = DAG.getNode(ISD::MUL, getCurSDLoc(),
03473                              N.getValueType(), IdxN, Scale);
03474         }
03475       }
03476 
03477       N = DAG.getNode(ISD::ADD, getCurSDLoc(),
03478                       N.getValueType(), N, IdxN);
03479     }
03480   }
03481 
03482   setValue(&I, N);
03483 }
03484 
03485 void SelectionDAGBuilder::visitAlloca(const AllocaInst &I) {
03486   // If this is a fixed sized alloca in the entry block of the function,
03487   // allocate it statically on the stack.
03488   if (FuncInfo.StaticAllocaMap.count(&I))
03489     return;   // getValue will auto-populate this.
03490 
03491   Type *Ty = I.getAllocatedType();
03492   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
03493   uint64_t TySize = TLI.getDataLayout()->getTypeAllocSize(Ty);
03494   unsigned Align =
03495       std::max((unsigned)TLI.getDataLayout()->getPrefTypeAlignment(Ty),
03496                I.getAlignment());
03497 
03498   SDValue AllocSize = getValue(I.getArraySize());
03499 
03500   EVT IntPtr = TLI.getPointerTy();
03501   if (AllocSize.getValueType() != IntPtr)
03502     AllocSize = DAG.getZExtOrTrunc(AllocSize, getCurSDLoc(), IntPtr);
03503 
03504   AllocSize = DAG.getNode(ISD::MUL, getCurSDLoc(), IntPtr,
03505                           AllocSize,
03506                           DAG.getConstant(TySize, IntPtr));
03507 
03508   // Handle alignment.  If the requested alignment is less than or equal to
03509   // the stack alignment, ignore it.  If the size is greater than or equal to
03510   // the stack alignment, we note this in the DYNAMIC_STACKALLOC node.
03511   unsigned StackAlign =
03512       DAG.getSubtarget().getFrameLowering()->getStackAlignment();
03513   if (Align <= StackAlign)
03514     Align = 0;
03515 
03516   // Round the size of the allocation up to the stack alignment size
03517   // by add SA-1 to the size.
03518   AllocSize = DAG.getNode(ISD::ADD, getCurSDLoc(),
03519                           AllocSize.getValueType(), AllocSize,
03520                           DAG.getIntPtrConstant(StackAlign-1));
03521 
03522   // Mask out the low bits for alignment purposes.
03523   AllocSize = DAG.getNode(ISD::AND, getCurSDLoc(),
03524                           AllocSize.getValueType(), AllocSize,
03525                           DAG.getIntPtrConstant(~(uint64_t)(StackAlign-1)));
03526 
03527   SDValue Ops[] = { getRoot(), AllocSize, DAG.getIntPtrConstant(Align) };
03528   SDVTList VTs = DAG.getVTList(AllocSize.getValueType(), MVT::Other);
03529   SDValue DSA = DAG.getNode(ISD::DYNAMIC_STACKALLOC, getCurSDLoc(), VTs, Ops);
03530   setValue(&I, DSA);
03531   DAG.setRoot(DSA.getValue(1));
03532 
03533   assert(FuncInfo.MF->getFrameInfo()->hasVarSizedObjects());
03534 }
03535 
03536 void SelectionDAGBuilder::visitLoad(const LoadInst &I) {
03537   if (I.isAtomic())
03538     return visitAtomicLoad(I);
03539 
03540   const Value *SV = I.getOperand(0);
03541   SDValue Ptr = getValue(SV);
03542 
03543   Type *Ty = I.getType();
03544 
03545   bool isVolatile = I.isVolatile();
03546   bool isNonTemporal = I.getMetadata(LLVMContext::MD_nontemporal) != nullptr;
03547   bool isInvariant = I.getMetadata(LLVMContext::MD_invariant_load) != nullptr;
03548   unsigned Alignment = I.getAlignment();
03549 
03550   AAMDNodes AAInfo;
03551   I.getAAMetadata(AAInfo);
03552   const MDNode *Ranges = I.getMetadata(LLVMContext::MD_range);
03553 
03554   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
03555   SmallVector<EVT, 4> ValueVTs;
03556   SmallVector<uint64_t, 4> Offsets;
03557   ComputeValueVTs(TLI, Ty, ValueVTs, &Offsets);
03558   unsigned NumValues = ValueVTs.size();
03559   if (NumValues == 0)
03560     return;
03561 
03562   SDValue Root;
03563   bool ConstantMemory = false;
03564   if (isVolatile || NumValues > MaxParallelChains)
03565     // Serialize volatile loads with other side effects.
03566     Root = getRoot();
03567   else if (AA->pointsToConstantMemory(
03568              AliasAnalysis::Location(SV, AA->getTypeStoreSize(Ty), AAInfo))) {
03569     // Do not serialize (non-volatile) loads of constant memory with anything.
03570     Root = DAG.getEntryNode();
03571     ConstantMemory = true;
03572   } else {
03573     // Do not serialize non-volatile loads against each other.
03574     Root = DAG.getRoot();
03575   }
03576 
03577   if (isVolatile)
03578     Root = TLI.prepareVolatileOrAtomicLoad(Root, getCurSDLoc(), DAG);
03579 
03580   SmallVector<SDValue, 4> Values(NumValues);
03581   SmallVector<SDValue, 4> Chains(std::min(unsigned(MaxParallelChains),
03582                                           NumValues));
03583   EVT PtrVT = Ptr.getValueType();
03584   unsigned ChainI = 0;
03585   for (unsigned i = 0; i != NumValues; ++i, ++ChainI) {
03586     // Serializing loads here may result in excessive register pressure, and
03587     // TokenFactor places arbitrary choke points on the scheduler. SD scheduling
03588     // could recover a bit by hoisting nodes upward in the chain by recognizing
03589     // they are side-effect free or do not alias. The optimizer should really
03590     // avoid this case by converting large object/array copies to llvm.memcpy
03591     // (MaxParallelChains should always remain as failsafe).
03592     if (ChainI == MaxParallelChains) {
03593       assert(PendingLoads.empty() && "PendingLoads must be serialized first");
03594       SDValue Chain = DAG.getNode(ISD::TokenFactor, getCurSDLoc(), MVT::Other,
03595                                   makeArrayRef(Chains.data(), ChainI));
03596       Root = Chain;
03597       ChainI = 0;
03598     }
03599     SDValue A = DAG.getNode(ISD::ADD, getCurSDLoc(),
03600                             PtrVT, Ptr,
03601                             DAG.getConstant(Offsets[i], PtrVT));
03602     SDValue L = DAG.getLoad(ValueVTs[i], getCurSDLoc(), Root,
03603                             A, MachinePointerInfo(SV, Offsets[i]), isVolatile,
03604                             isNonTemporal, isInvariant, Alignment, AAInfo,
03605                             Ranges);
03606 
03607     Values[i] = L;
03608     Chains[ChainI] = L.getValue(1);
03609   }
03610 
03611   if (!ConstantMemory) {
03612     SDValue Chain = DAG.getNode(ISD::TokenFactor, getCurSDLoc(), MVT::Other,
03613                                 makeArrayRef(Chains.data(), ChainI));
03614     if (isVolatile)
03615       DAG.setRoot(Chain);
03616     else
03617       PendingLoads.push_back(Chain);
03618   }
03619 
03620   setValue(&I, DAG.getNode(ISD::MERGE_VALUES, getCurSDLoc(),
03621                            DAG.getVTList(ValueVTs), Values));
03622 }
03623 
03624 void SelectionDAGBuilder::visitStore(const StoreInst &I) {
03625   if (I.isAtomic())
03626     return visitAtomicStore(I);
03627 
03628   const Value *SrcV = I.getOperand(0);
03629   const Value *PtrV = I.getOperand(1);
03630 
03631   SmallVector<EVT, 4> ValueVTs;
03632   SmallVector<uint64_t, 4> Offsets;
03633   ComputeValueVTs(DAG.getTargetLoweringInfo(), SrcV->getType(),
03634                   ValueVTs, &Offsets);
03635   unsigned NumValues = ValueVTs.size();
03636   if (NumValues == 0)
03637     return;
03638 
03639   // Get the lowered operands. Note that we do this after
03640   // checking if NumResults is zero, because with zero results
03641   // the operands won't have values in the map.
03642   SDValue Src = getValue(SrcV);
03643   SDValue Ptr = getValue(PtrV);
03644 
03645   SDValue Root = getRoot();
03646   SmallVector<SDValue, 4> Chains(std::min(unsigned(MaxParallelChains),
03647                                           NumValues));
03648   EVT PtrVT = Ptr.getValueType();
03649   bool isVolatile = I.isVolatile();
03650   bool isNonTemporal = I.getMetadata(LLVMContext::MD_nontemporal) != nullptr;
03651   unsigned Alignment = I.getAlignment();
03652 
03653   AAMDNodes AAInfo;
03654   I.getAAMetadata(AAInfo);
03655 
03656   unsigned ChainI = 0;
03657   for (unsigned i = 0; i != NumValues; ++i, ++ChainI) {
03658     // See visitLoad comments.
03659     if (ChainI == MaxParallelChains) {
03660       SDValue Chain = DAG.getNode(ISD::TokenFactor, getCurSDLoc(), MVT::Other,
03661                                   makeArrayRef(Chains.data(), ChainI));
03662       Root = Chain;
03663       ChainI = 0;
03664     }
03665     SDValue Add = DAG.getNode(ISD::ADD, getCurSDLoc(), PtrVT, Ptr,
03666                               DAG.getConstant(Offsets[i], PtrVT));
03667     SDValue St = DAG.getStore(Root, getCurSDLoc(),
03668                               SDValue(Src.getNode(), Src.getResNo() + i),
03669                               Add, MachinePointerInfo(PtrV, Offsets[i]),
03670                               isVolatile, isNonTemporal, Alignment, AAInfo);
03671     Chains[ChainI] = St;
03672   }
03673 
03674   SDValue StoreNode = DAG.getNode(ISD::TokenFactor, getCurSDLoc(), MVT::Other,
03675                                   makeArrayRef(Chains.data(), ChainI));
03676   DAG.setRoot(StoreNode);
03677 }
03678 
03679 void SelectionDAGBuilder::visitMaskedStore(const CallInst &I) {
03680   SDLoc sdl = getCurSDLoc();
03681 
03682   // llvm.masked.store.*(Src0, Ptr, alignemt, Mask)
03683   Value  *PtrOperand = I.getArgOperand(1);
03684   SDValue Ptr = getValue(PtrOperand);
03685   SDValue Src0 = getValue(I.getArgOperand(0));
03686   SDValue Mask = getValue(I.getArgOperand(3));
03687   EVT VT = Src0.getValueType();
03688   unsigned Alignment = (cast<ConstantInt>(I.getArgOperand(2)))->getZExtValue();
03689   if (!Alignment)
03690     Alignment = DAG.getEVTAlignment(VT);
03691 
03692   AAMDNodes AAInfo;
03693   I.getAAMetadata(AAInfo);
03694 
03695   MachineMemOperand *MMO =
03696     DAG.getMachineFunction().
03697     getMachineMemOperand(MachinePointerInfo(PtrOperand),
03698                           MachineMemOperand::MOStore,  VT.getStoreSize(),
03699                           Alignment, AAInfo);
03700   SDValue StoreNode = DAG.getMaskedStore(getRoot(), sdl, Src0, Ptr, Mask, VT,
03701                                          MMO, false);
03702   DAG.setRoot(StoreNode);
03703   setValue(&I, StoreNode);
03704 }
03705 
03706 void SelectionDAGBuilder::visitMaskedLoad(const CallInst &I) {
03707   SDLoc sdl = getCurSDLoc();
03708 
03709   // @llvm.masked.load.*(Ptr, alignment, Mask, Src0)
03710   Value  *PtrOperand = I.getArgOperand(0);
03711   SDValue Ptr = getValue(PtrOperand);
03712   SDValue Src0 = getValue(I.getArgOperand(3));
03713   SDValue Mask = getValue(I.getArgOperand(2));
03714 
03715   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
03716   EVT VT = TLI.getValueType(I.getType());
03717   unsigned Alignment = (cast<ConstantInt>(I.getArgOperand(1)))->getZExtValue();
03718   if (!Alignment)
03719     Alignment = DAG.getEVTAlignment(VT);
03720 
03721   AAMDNodes AAInfo;
03722   I.getAAMetadata(AAInfo);
03723   const MDNode *Ranges = I.getMetadata(LLVMContext::MD_range);
03724 
03725   SDValue InChain = DAG.getRoot();
03726   if (AA->pointsToConstantMemory(
03727       AliasAnalysis::Location(PtrOperand,
03728                               AA->getTypeStoreSize(I.getType()),
03729                               AAInfo))) {
03730     // Do not serialize (non-volatile) loads of constant memory with anything.
03731     InChain = DAG.getEntryNode();
03732   }
03733 
03734   MachineMemOperand *MMO =
03735     DAG.getMachineFunction().
03736     getMachineMemOperand(MachinePointerInfo(PtrOperand),
03737                           MachineMemOperand::MOLoad,  VT.getStoreSize(),
03738                           Alignment, AAInfo, Ranges);
03739 
03740   SDValue Load = DAG.getMaskedLoad(VT, sdl, InChain, Ptr, Mask, Src0, VT, MMO,
03741                                    ISD::NON_EXTLOAD);
03742   SDValue OutChain = Load.getValue(1);
03743   DAG.setRoot(OutChain);
03744   setValue(&I, Load);
03745 }
03746 
03747 void SelectionDAGBuilder::visitAtomicCmpXchg(const AtomicCmpXchgInst &I) {
03748   SDLoc dl = getCurSDLoc();
03749   AtomicOrdering SuccessOrder = I.getSuccessOrdering();
03750   AtomicOrdering FailureOrder = I.getFailureOrdering();
03751   SynchronizationScope Scope = I.getSynchScope();
03752 
03753   SDValue InChain = getRoot();
03754 
03755   MVT MemVT = getValue(I.getCompareOperand()).getSimpleValueType();
03756   SDVTList VTs = DAG.getVTList(MemVT, MVT::i1, MVT::Other);
03757   SDValue L = DAG.getAtomicCmpSwap(
03758       ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS, dl, MemVT, VTs, InChain,
03759       getValue(I.getPointerOperand()), getValue(I.getCompareOperand()),
03760       getValue(I.getNewValOperand()), MachinePointerInfo(I.getPointerOperand()),
03761       /*Alignment=*/ 0, SuccessOrder, FailureOrder, Scope);
03762 
03763   SDValue OutChain = L.getValue(2);
03764 
03765   setValue(&I, L);
03766   DAG.setRoot(OutChain);
03767 }
03768 
03769 void SelectionDAGBuilder::visitAtomicRMW(const AtomicRMWInst &I) {
03770   SDLoc dl = getCurSDLoc();
03771   ISD::NodeType NT;
03772   switch (I.getOperation()) {
03773   default: llvm_unreachable("Unknown atomicrmw operation");
03774   case AtomicRMWInst::Xchg: NT = ISD::ATOMIC_SWAP; break;
03775   case AtomicRMWInst::Add:  NT = ISD::ATOMIC_LOAD_ADD; break;
03776   case AtomicRMWInst::Sub:  NT = ISD::ATOMIC_LOAD_SUB; break;
03777   case AtomicRMWInst::And:  NT = ISD::ATOMIC_LOAD_AND; break;
03778   case AtomicRMWInst::Nand: NT = ISD::ATOMIC_LOAD_NAND; break;
03779   case AtomicRMWInst::Or:   NT = ISD::ATOMIC_LOAD_OR; break;
03780   case AtomicRMWInst::Xor:  NT = ISD::ATOMIC_LOAD_XOR; break;
03781   case AtomicRMWInst::Max:  NT = ISD::ATOMIC_LOAD_MAX; break;
03782   case AtomicRMWInst::Min:  NT = ISD::ATOMIC_LOAD_MIN; break;
03783   case AtomicRMWInst::UMax: NT = ISD::ATOMIC_LOAD_UMAX; break;
03784   case AtomicRMWInst::UMin: NT = ISD::ATOMIC_LOAD_UMIN; break;
03785   }
03786   AtomicOrdering Order = I.getOrdering();
03787   SynchronizationScope Scope = I.getSynchScope();
03788 
03789   SDValue InChain = getRoot();
03790 
03791   SDValue L =
03792     DAG.getAtomic(NT, dl,
03793                   getValue(I.getValOperand()).getSimpleValueType(),
03794                   InChain,
03795                   getValue(I.getPointerOperand()),
03796                   getValue(I.getValOperand()),
03797                   I.getPointerOperand(),
03798                   /* Alignment=*/ 0, Order, Scope);
03799 
03800   SDValue OutChain = L.getValue(1);
03801 
03802   setValue(&I, L);
03803   DAG.setRoot(OutChain);
03804 }
03805 
03806 void SelectionDAGBuilder::visitFence(const FenceInst &I) {
03807   SDLoc dl = getCurSDLoc();
03808   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
03809   SDValue Ops[3];
03810   Ops[0] = getRoot();
03811   Ops[1] = DAG.getConstant(I.getOrdering(), TLI.getPointerTy());
03812   Ops[2] = DAG.getConstant(I.getSynchScope(), TLI.getPointerTy());
03813   DAG.setRoot(DAG.getNode(ISD::ATOMIC_FENCE, dl, MVT::Other, Ops));
03814 }
03815 
03816 void SelectionDAGBuilder::visitAtomicLoad(const LoadInst &I) {
03817   SDLoc dl = getCurSDLoc();
03818   AtomicOrdering Order = I.getOrdering();
03819   SynchronizationScope Scope = I.getSynchScope();
03820 
03821   SDValue InChain = getRoot();
03822 
03823   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
03824   EVT VT = TLI.getValueType(I.getType());
03825 
03826   if (I.getAlignment() < VT.getSizeInBits() / 8)
03827     report_fatal_error("Cannot generate unaligned atomic load");
03828 
03829   MachineMemOperand *MMO =
03830       DAG.getMachineFunction().
03831       getMachineMemOperand(MachinePointerInfo(I.getPointerOperand()),
03832                            MachineMemOperand::MOVolatile |
03833                            MachineMemOperand::MOLoad,
03834                            VT.getStoreSize(),
03835                            I.getAlignment() ? I.getAlignment() :
03836                                               DAG.getEVTAlignment(VT));
03837 
03838   InChain = TLI.prepareVolatileOrAtomicLoad(InChain, dl, DAG);
03839   SDValue L =
03840       DAG.getAtomic(ISD::ATOMIC_LOAD, dl, VT, VT, InChain,
03841                     getValue(I.getPointerOperand()), MMO,
03842                     Order, Scope);
03843 
03844   SDValue OutChain = L.getValue(1);
03845 
03846   setValue(&I, L);
03847   DAG.setRoot(OutChain);
03848 }
03849 
03850 void SelectionDAGBuilder::visitAtomicStore(const StoreInst &I) {
03851   SDLoc dl = getCurSDLoc();
03852 
03853   AtomicOrdering Order = I.getOrdering();
03854   SynchronizationScope Scope = I.getSynchScope();
03855 
03856   SDValue InChain = getRoot();
03857 
03858   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
03859   EVT VT = TLI.getValueType(I.getValueOperand()->getType());
03860 
03861   if (I.getAlignment() < VT.getSizeInBits() / 8)
03862     report_fatal_error("Cannot generate unaligned atomic store");
03863 
03864   SDValue OutChain =
03865     DAG.getAtomic(ISD::ATOMIC_STORE, dl, VT,
03866                   InChain,
03867                   getValue(I.getPointerOperand()),
03868                   getValue(I.getValueOperand()),
03869                   I.getPointerOperand(), I.getAlignment(),
03870                   Order, Scope);
03871 
03872   DAG.setRoot(OutChain);
03873 }
03874 
03875 /// visitTargetIntrinsic - Lower a call of a target intrinsic to an INTRINSIC
03876 /// node.
03877 void SelectionDAGBuilder::visitTargetIntrinsic(const CallInst &I,
03878                                                unsigned Intrinsic) {
03879   bool HasChain = !I.doesNotAccessMemory();
03880   bool OnlyLoad = HasChain && I.onlyReadsMemory();
03881 
03882   // Build the operand list.
03883   SmallVector<SDValue, 8> Ops;
03884   if (HasChain) {  // If this intrinsic has side-effects, chainify it.
03885     if (OnlyLoad) {
03886       // We don't need to serialize loads against other loads.
03887       Ops.push_back(DAG.getRoot());
03888     } else {
03889       Ops.push_back(getRoot());
03890     }
03891   }
03892 
03893   // Info is set by getTgtMemInstrinsic
03894   TargetLowering::IntrinsicInfo Info;
03895   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
03896   bool IsTgtIntrinsic = TLI.getTgtMemIntrinsic(Info, I, Intrinsic);
03897 
03898   // Add the intrinsic ID as an integer operand if it's not a target intrinsic.
03899   if (!IsTgtIntrinsic || Info.opc == ISD::INTRINSIC_VOID ||
03900       Info.opc == ISD::INTRINSIC_W_CHAIN)
03901     Ops.push_back(DAG.getTargetConstant(Intrinsic, TLI.getPointerTy()));
03902 
03903   // Add all operands of the call to the operand list.
03904   for (unsigned i = 0, e = I.getNumArgOperands(); i != e; ++i) {
03905     SDValue Op = getValue(I.getArgOperand(i));
03906     Ops.push_back(Op);
03907   }
03908 
03909   SmallVector<EVT, 4> ValueVTs;
03910   ComputeValueVTs(TLI, I.getType(), ValueVTs);
03911 
03912   if (HasChain)
03913     ValueVTs.push_back(MVT::Other);
03914 
03915   SDVTList VTs = DAG.getVTList(ValueVTs);
03916 
03917   // Create the node.
03918   SDValue Result;
03919   if (IsTgtIntrinsic) {
03920     // This is target intrinsic that touches memory
03921     Result = DAG.getMemIntrinsicNode(Info.opc, getCurSDLoc(),
03922                                      VTs, Ops, Info.memVT,
03923                                    MachinePointerInfo(Info.ptrVal, Info.offset),
03924                                      Info.align, Info.vol,
03925                                      Info.readMem, Info.writeMem, Info.size);
03926   } else if (!HasChain) {
03927     Result = DAG.getNode(ISD::INTRINSIC_WO_CHAIN, getCurSDLoc(), VTs, Ops);
03928   } else if (!I.getType()->isVoidTy()) {
03929     Result = DAG.getNode(ISD::INTRINSIC_W_CHAIN, getCurSDLoc(), VTs, Ops);
03930   } else {
03931     Result = DAG.getNode(ISD::INTRINSIC_VOID, getCurSDLoc(), VTs, Ops);
03932   }
03933 
03934   if (HasChain) {
03935     SDValue Chain = Result.getValue(Result.getNode()->getNumValues()-1);
03936     if (OnlyLoad)
03937       PendingLoads.push_back(Chain);
03938     else
03939       DAG.setRoot(Chain);
03940   }
03941 
03942   if (!I.getType()->isVoidTy()) {
03943     if (VectorType *PTy = dyn_cast<VectorType>(I.getType())) {
03944       EVT VT = TLI.getValueType(PTy);
03945       Result = DAG.getNode(ISD::BITCAST, getCurSDLoc(), VT, Result);
03946     }
03947 
03948     setValue(&I, Result);
03949   }
03950 }
03951 
03952 /// GetSignificand - Get the significand and build it into a floating-point
03953 /// number with exponent of 1:
03954 ///
03955 ///   Op = (Op & 0x007fffff) | 0x3f800000;
03956 ///
03957 /// where Op is the hexadecimal representation of floating point value.
03958 static SDValue
03959 GetSignificand(SelectionDAG &DAG, SDValue Op, SDLoc dl) {
03960   SDValue t1 = DAG.getNode(ISD::AND, dl, MVT::i32, Op,
03961                            DAG.getConstant(0x007fffff, MVT::i32));
03962   SDValue t2 = DAG.getNode(ISD::OR, dl, MVT::i32, t1,
03963                            DAG.getConstant(0x3f800000, MVT::i32));
03964   return DAG.getNode(ISD::BITCAST, dl, MVT::f32, t2);
03965 }
03966 
03967 /// GetExponent - Get the exponent:
03968 ///
03969 ///   (float)(int)(((Op & 0x7f800000) >> 23) - 127);
03970 ///
03971 /// where Op is the hexadecimal representation of floating point value.
03972 static SDValue
03973 GetExponent(SelectionDAG &DAG, SDValue Op, const TargetLowering &TLI,
03974             SDLoc dl) {
03975   SDValue t0 = DAG.getNode(ISD::AND, dl, MVT::i32, Op,
03976                            DAG.getConstant(0x7f800000, MVT::i32));
03977   SDValue t1 = DAG.getNode(ISD::SRL, dl, MVT::i32, t0,
03978                            DAG.getConstant(23, TLI.getPointerTy()));
03979   SDValue t2 = DAG.getNode(ISD::SUB, dl, MVT::i32, t1,
03980                            DAG.getConstant(127, MVT::i32));
03981   return DAG.getNode(ISD::SINT_TO_FP, dl, MVT::f32, t2);
03982 }
03983 
03984 /// getF32Constant - Get 32-bit floating point constant.
03985 static SDValue
03986 getF32Constant(SelectionDAG &DAG, unsigned Flt) {
03987   return DAG.getConstantFP(APFloat(APFloat::IEEEsingle, APInt(32, Flt)),
03988                            MVT::f32);
03989 }
03990 
03991 /// expandExp - Lower an exp intrinsic. Handles the special sequences for
03992 /// limited-precision mode.
03993 static SDValue expandExp(SDLoc dl, SDValue Op, SelectionDAG &DAG,
03994                          const TargetLowering &TLI) {
03995   if (Op.getValueType() == MVT::f32 &&
03996       LimitFloatPrecision > 0 && LimitFloatPrecision <= 18) {
03997 
03998     // Put the exponent in the right bit position for later addition to the
03999     // final result:
04000     //
04001     //   #define LOG2OFe 1.4426950f
04002     //   IntegerPartOfX = ((int32_t)(X * LOG2OFe));
04003     SDValue t0 = DAG.getNode(ISD::FMUL, dl, MVT::f32, Op,
04004                              getF32Constant(DAG, 0x3fb8aa3b));
04005     SDValue IntegerPartOfX = DAG.getNode(ISD::FP_TO_SINT, dl, MVT::i32, t0);
04006 
04007     //   FractionalPartOfX = (X * LOG2OFe) - (float)IntegerPartOfX;
04008     SDValue t1 = DAG.getNode(ISD::SINT_TO_FP, dl, MVT::f32, IntegerPartOfX);
04009     SDValue X = DAG.getNode(ISD::FSUB, dl, MVT::f32, t0, t1);
04010 
04011     //   IntegerPartOfX <<= 23;
04012     IntegerPartOfX = DAG.getNode(ISD::SHL, dl, MVT::i32, IntegerPartOfX,
04013                                  DAG.getConstant(23, TLI.getPointerTy()));
04014 
04015     SDValue TwoToFracPartOfX;
04016     if (LimitFloatPrecision <= 6) {
04017       // For floating-point precision of 6:
04018       //
04019       //   TwoToFractionalPartOfX =
04020       //     0.997535578f +
04021       //       (0.735607626f + 0.252464424f * x) * x;
04022       //
04023       // error 0.0144103317, which is 6 bits
04024       SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X,
04025                                getF32Constant(DAG, 0x3e814304));
04026       SDValue t3 = DAG.getNode(ISD::FADD, dl, MVT::f32, t2,
04027                                getF32Constant(DAG, 0x3f3c50c8));
04028       SDValue t4 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t3, X);
04029       TwoToFracPartOfX = DAG.getNode(ISD::FADD, dl, MVT::f32, t4,
04030                                      getF32Constant(DAG, 0x3f7f5e7e));
04031     } else if (LimitFloatPrecision <= 12) {
04032       // For floating-point precision of 12:
04033       //
04034       //   TwoToFractionalPartOfX =
04035       //     0.999892986f +
04036       //       (0.696457318f +
04037       //         (0.224338339f + 0.792043434e-1f * x) * x) * x;
04038       //
04039       // 0.000107046256 error, which is 13 to 14 bits
04040       SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X,
04041                                getF32Constant(DAG, 0x3da235e3));
04042       SDValue t3 = DAG.getNode(ISD::FADD, dl, MVT::f32, t2,
04043                                getF32Constant(DAG, 0x3e65b8f3));
04044       SDValue t4 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t3, X);
04045       SDValue t5 = DAG.getNode(ISD::FADD, dl, MVT::f32, t4,
04046                                getF32Constant(DAG, 0x3f324b07));
04047       SDValue t6 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t5, X);
04048       TwoToFracPartOfX = DAG.getNode(ISD::FADD, dl, MVT::f32, t6,
04049                                      getF32Constant(DAG, 0x3f7ff8fd));
04050     } else { // LimitFloatPrecision <= 18
04051       // For floating-point precision of 18:
04052       //
04053       //   TwoToFractionalPartOfX =
04054       //     0.999999982f +
04055       //       (0.693148872f +
04056       //         (0.240227044f +
04057       //           (0.554906021e-1f +
04058       //             (0.961591928e-2f +
04059       //               (0.136028312e-2f + 0.157059148e-3f *x)*x)*x)*x)*x)*x;
04060       //
04061       // error 2.47208000*10^(-7), which is better than 18 bits
04062       SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X,
04063                                getF32Constant(DAG, 0x3924b03e));
04064       SDValue t3 = DAG.getNode(ISD::FADD, dl, MVT::f32, t2,
04065                                getF32Constant(DAG, 0x3ab24b87));
04066       SDValue t4 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t3, X);
04067       SDValue t5 = DAG.getNode(ISD::FADD, dl, MVT::f32, t4,
04068                                getF32Constant(DAG, 0x3c1d8c17));
04069       SDValue t6 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t5, X);
04070       SDValue t7 = DAG.getNode(ISD::FADD, dl, MVT::f32, t6,
04071                                getF32Constant(DAG, 0x3d634a1d));
04072       SDValue t8 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t7, X);
04073       SDValue t9 = DAG.getNode(ISD::FADD, dl, MVT::f32, t8,
04074                                getF32Constant(DAG, 0x3e75fe14));
04075       SDValue t10 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t9, X);
04076       SDValue t11 = DAG.getNode(ISD::FADD, dl, MVT::f32, t10,
04077                                 getF32Constant(DAG, 0x3f317234));
04078       SDValue t12 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t11, X);
04079       TwoToFracPartOfX = DAG.getNode(ISD::FADD, dl, MVT::f32, t12,
04080                                      getF32Constant(DAG, 0x3f800000));
04081     }
04082 
04083     // Add the exponent into the result in integer domain.
04084     SDValue t13 = DAG.getNode(ISD::BITCAST, dl, MVT::i32, TwoToFracPartOfX);
04085     return DAG.getNode(ISD::BITCAST, dl, MVT::f32,
04086                        DAG.getNode(ISD::ADD, dl, MVT::i32,
04087                                    t13, IntegerPartOfX));
04088   }
04089 
04090   // No special expansion.
04091   return DAG.getNode(ISD::FEXP, dl, Op.getValueType(), Op);
04092 }
04093 
04094 /// expandLog - Lower a log intrinsic. Handles the special sequences for
04095 /// limited-precision mode.
04096 static SDValue expandLog(SDLoc dl, SDValue Op, SelectionDAG &DAG,
04097                          const TargetLowering &TLI) {
04098   if (Op.getValueType() == MVT::f32 &&
04099       LimitFloatPrecision > 0 && LimitFloatPrecision <= 18) {
04100     SDValue Op1 = DAG.getNode(ISD::BITCAST, dl, MVT::i32, Op);
04101 
04102     // Scale the exponent by log(2) [0.69314718f].
04103     SDValue Exp = GetExponent(DAG, Op1, TLI, dl);
04104     SDValue LogOfExponent = DAG.getNode(ISD::FMUL, dl, MVT::f32, Exp,
04105                                         getF32Constant(DAG, 0x3f317218));
04106 
04107     // Get the significand and build it into a floating-point number with
04108     // exponent of 1.
04109     SDValue X = GetSignificand(DAG, Op1, dl);
04110 
04111     SDValue LogOfMantissa;
04112     if (LimitFloatPrecision <= 6) {
04113       // For floating-point precision of 6:
04114       //
04115       //   LogofMantissa =
04116       //     -1.1609546f +
04117       //       (1.4034025f - 0.23903021f * x) * x;
04118       //
04119       // error 0.0034276066, which is better than 8 bits
04120       SDValue t0 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X,
04121                                getF32Constant(DAG, 0xbe74c456));
04122       SDValue t1 = DAG.getNode(ISD::FADD, dl, MVT::f32, t0,
04123                                getF32Constant(DAG, 0x3fb3a2b1));
04124       SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t1, X);
04125       LogOfMantissa = DAG.getNode(ISD::FSUB, dl, MVT::f32, t2,
04126                                   getF32Constant(DAG, 0x3f949a29));
04127     } else if (LimitFloatPrecision <= 12) {
04128       // For floating-point precision of 12:
04129       //
04130       //   LogOfMantissa =
04131       //     -1.7417939f +
04132       //       (2.8212026f +
04133       //         (-1.4699568f +
04134       //           (0.44717955f - 0.56570851e-1f * x) * x) * x) * x;
04135       //
04136       // error 0.000061011436, which is 14 bits
04137       SDValue t0 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X,
04138                                getF32Constant(DAG, 0xbd67b6d6));
04139       SDValue t1 = DAG.getNode(ISD::FADD, dl, MVT::f32, t0,
04140                                getF32Constant(DAG, 0x3ee4f4b8));
04141       SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t1, X);
04142       SDValue t3 = DAG.getNode(ISD::FSUB, dl, MVT::f32, t2,
04143                                getF32Constant(DAG, 0x3fbc278b));
04144       SDValue t4 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t3, X);
04145       SDValue t5 = DAG.getNode(ISD::FADD, dl, MVT::f32, t4,
04146                                getF32Constant(DAG, 0x40348e95));
04147       SDValue t6 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t5, X);
04148       LogOfMantissa = DAG.getNode(ISD::FSUB, dl, MVT::f32, t6,
04149                                   getF32Constant(DAG, 0x3fdef31a));
04150     } else { // LimitFloatPrecision <= 18
04151       // For floating-point precision of 18:
04152       //
04153       //   LogOfMantissa =
04154       //     -2.1072184f +
04155       //       (4.2372794f +
04156       //         (-3.7029485f +
04157       //           (2.2781945f +
04158       //             (-0.87823314f +
04159       //               (0.19073739f - 0.17809712e-1f * x) * x) * x) * x) * x)*x;
04160       //
04161       // error 0.0000023660568, which is better than 18 bits
04162       SDValue t0 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X,
04163                                getF32Constant(DAG, 0xbc91e5ac));
04164       SDValue t1 = DAG.getNode(ISD::FADD, dl, MVT::f32, t0,
04165                                getF32Constant(DAG, 0x3e4350aa));
04166       SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t1, X);
04167       SDValue t3 = DAG.getNode(ISD::FSUB, dl, MVT::f32, t2,
04168                                getF32Constant(DAG, 0x3f60d3e3));
04169       SDValue t4 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t3, X);
04170       SDValue t5 = DAG.getNode(ISD::FADD, dl, MVT::f32, t4,
04171                                getF32Constant(DAG, 0x4011cdf0));
04172       SDValue t6 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t5, X);
04173       SDValue t7 = DAG.getNode(ISD::FSUB, dl, MVT::f32, t6,
04174                                getF32Constant(DAG, 0x406cfd1c));
04175       SDValue t8 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t7, X);
04176       SDValue t9 = DAG.getNode(ISD::FADD, dl, MVT::f32, t8,
04177                                getF32Constant(DAG, 0x408797cb));
04178       SDValue t10 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t9, X);
04179       LogOfMantissa = DAG.getNode(ISD::FSUB, dl, MVT::f32, t10,
04180                                   getF32Constant(DAG, 0x4006dcab));
04181     }
04182 
04183     return DAG.getNode(ISD::FADD, dl, MVT::f32, LogOfExponent, LogOfMantissa);
04184   }
04185 
04186   // No special expansion.
04187   return DAG.getNode(ISD::FLOG, dl, Op.getValueType(), Op);
04188 }
04189 
04190 /// expandLog2 - Lower a log2 intrinsic. Handles the special sequences for
04191 /// limited-precision mode.
04192 static SDValue expandLog2(SDLoc dl, SDValue Op, SelectionDAG &DAG,
04193                           const TargetLowering &TLI) {
04194   if (Op.getValueType() == MVT::f32 &&
04195       LimitFloatPrecision > 0 && LimitFloatPrecision <= 18) {
04196     SDValue Op1 = DAG.getNode(ISD::BITCAST, dl, MVT::i32, Op);
04197 
04198     // Get the exponent.
04199     SDValue LogOfExponent = GetExponent(DAG, Op1, TLI, dl);
04200 
04201     // Get the significand and build it into a floating-point number with
04202     // exponent of 1.
04203     SDValue X = GetSignificand(DAG, Op1, dl);
04204 
04205     // Different possible minimax approximations of significand in
04206     // floating-point for various degrees of accuracy over [1,2].
04207     SDValue Log2ofMantissa;
04208     if (LimitFloatPrecision <= 6) {
04209       // For floating-point precision of 6:
04210       //
04211       //   Log2ofMantissa = -1.6749035f + (2.0246817f - .34484768f * x) * x;
04212       //
04213       // error 0.0049451742, which is more than 7 bits
04214       SDValue t0 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X,
04215                                getF32Constant(DAG, 0xbeb08fe0));
04216       SDValue t1 = DAG.getNode(ISD::FADD, dl, MVT::f32, t0,
04217                                getF32Constant(DAG, 0x40019463));
04218       SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t1, X);
04219       Log2ofMantissa = DAG.getNode(ISD::FSUB, dl, MVT::f32, t2,
04220                                    getF32Constant(DAG, 0x3fd6633d));
04221     } else if (LimitFloatPrecision <= 12) {
04222       // For floating-point precision of 12:
04223       //
04224       //   Log2ofMantissa =
04225       //     -2.51285454f +
04226       //       (4.07009056f +
04227       //         (-2.12067489f +
04228       //           (.645142248f - 0.816157886e-1f * x) * x) * x) * x;
04229       //
04230       // error 0.0000876136000, which is better than 13 bits
04231       SDValue t0 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X,
04232                                getF32Constant(DAG, 0xbda7262e));
04233       SDValue t1 = DAG.getNode(ISD::FADD, dl, MVT::f32, t0,
04234                                getF32Constant(DAG, 0x3f25280b));
04235       SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t1, X);
04236       SDValue t3 = DAG.getNode(ISD::FSUB, dl, MVT::f32, t2,
04237                                getF32Constant(DAG, 0x4007b923));
04238       SDValue t4 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t3, X);
04239       SDValue t5 = DAG.getNode(ISD::FADD, dl, MVT::f32, t4,
04240                                getF32Constant(DAG, 0x40823e2f));
04241       SDValue t6 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t5, X);
04242       Log2ofMantissa = DAG.getNode(ISD::FSUB, dl, MVT::f32, t6,
04243                                    getF32Constant(DAG, 0x4020d29c));
04244     } else { // LimitFloatPrecision <= 18
04245       // For floating-point precision of 18:
04246       //
04247       //   Log2ofMantissa =
04248       //     -3.0400495f +
04249       //       (6.1129976f +
04250       //         (-5.3420409f +
04251       //           (3.2865683f +
04252       //             (-1.2669343f +
04253       //               (0.27515199f -
04254       //                 0.25691327e-1f * x) * x) * x) * x) * x) * x;
04255       //
04256       // error 0.0000018516, which is better than 18 bits
04257       SDValue t0 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X,
04258                                getF32Constant(DAG, 0xbcd2769e));
04259       SDValue t1 = DAG.getNode(ISD::FADD, dl, MVT::f32, t0,
04260                                getF32Constant(DAG, 0x3e8ce0b9));
04261       SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t1, X);
04262       SDValue t3 = DAG.getNode(ISD::FSUB, dl, MVT::f32, t2,
04263                                getF32Constant(DAG, 0x3fa22ae7));
04264       SDValue t4 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t3, X);
04265       SDValue t5 = DAG.getNode(ISD::FADD, dl, MVT::f32, t4,
04266                                getF32Constant(DAG, 0x40525723));
04267       SDValue t6 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t5, X);
04268       SDValue t7 = DAG.getNode(ISD::FSUB, dl, MVT::f32, t6,
04269                                getF32Constant(DAG, 0x40aaf200));
04270       SDValue t8 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t7, X);
04271       SDValue t9 = DAG.getNode(ISD::FADD, dl, MVT::f32, t8,
04272                                getF32Constant(DAG, 0x40c39dad));
04273       SDValue t10 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t9, X);
04274       Log2ofMantissa = DAG.getNode(ISD::FSUB, dl, MVT::f32, t10,
04275                                    getF32Constant(DAG, 0x4042902c));
04276     }
04277 
04278     return DAG.getNode(ISD::FADD, dl, MVT::f32, LogOfExponent, Log2ofMantissa);
04279   }
04280 
04281   // No special expansion.
04282   return DAG.getNode(ISD::FLOG2, dl, Op.getValueType(), Op);
04283 }
04284 
04285 /// expandLog10 - Lower a log10 intrinsic. Handles the special sequences for
04286 /// limited-precision mode.
04287 static SDValue expandLog10(SDLoc dl, SDValue Op, SelectionDAG &DAG,
04288                            const TargetLowering &TLI) {
04289   if (Op.getValueType() == MVT::f32 &&
04290       LimitFloatPrecision > 0 && LimitFloatPrecision <= 18) {
04291     SDValue Op1 = DAG.getNode(ISD::BITCAST, dl, MVT::i32, Op);
04292 
04293     // Scale the exponent by log10(2) [0.30102999f].
04294     SDValue Exp = GetExponent(DAG, Op1, TLI, dl);
04295     SDValue LogOfExponent = DAG.getNode(ISD::FMUL, dl, MVT::f32, Exp,
04296                                         getF32Constant(DAG, 0x3e9a209a));
04297 
04298     // Get the significand and build it into a floating-point number with
04299     // exponent of 1.
04300     SDValue X = GetSignificand(DAG, Op1, dl);
04301 
04302     SDValue Log10ofMantissa;
04303     if (LimitFloatPrecision <= 6) {
04304       // For floating-point precision of 6:
04305       //
04306       //   Log10ofMantissa =
04307       //     -0.50419619f +
04308       //       (0.60948995f - 0.10380950f * x) * x;
04309       //
04310       // error 0.0014886165, which is 6 bits
04311       SDValue t0 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X,
04312                                getF32Constant(DAG, 0xbdd49a13));
04313       SDValue t1 = DAG.getNode(ISD::FADD, dl, MVT::f32, t0,
04314                                getF32Constant(DAG, 0x3f1c0789));
04315       SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t1, X);
04316       Log10ofMantissa = DAG.getNode(ISD::FSUB, dl, MVT::f32, t2,
04317                                     getF32Constant(DAG, 0x3f011300));
04318     } else if (LimitFloatPrecision <= 12) {
04319       // For floating-point precision of 12:
04320       //
04321       //   Log10ofMantissa =
04322       //     -0.64831180f +
04323       //       (0.91751397f +
04324       //         (-0.31664806f + 0.47637168e-1f * x) * x) * x;
04325       //
04326       // error 0.00019228036, which is better than 12 bits
04327       SDValue t0 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X,
04328                                getF32Constant(DAG, 0x3d431f31));
04329       SDValue t1 = DAG.getNode(ISD::FSUB, dl, MVT::f32, t0,
04330                                getF32Constant(DAG, 0x3ea21fb2));
04331       SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t1, X);
04332       SDValue t3 = DAG.getNode(ISD::FADD, dl, MVT::f32, t2,
04333                                getF32Constant(DAG, 0x3f6ae232));
04334       SDValue t4 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t3, X);
04335       Log10ofMantissa = DAG.getNode(ISD::FSUB, dl, MVT::f32, t4,
04336                                     getF32Constant(DAG, 0x3f25f7c3));
04337     } else { // LimitFloatPrecision <= 18
04338       // For floating-point precision of 18:
04339       //
04340       //   Log10ofMantissa =
04341       //     -0.84299375f +
04342       //       (1.5327582f +
04343       //         (-1.0688956f +
04344       //           (0.49102474f +
04345       //             (-0.12539807f + 0.13508273e-1f * x) * x) * x) * x) * x;
04346       //
04347       // error 0.0000037995730, which is better than 18 bits
04348       SDValue t0 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X,
04349                                getF32Constant(DAG, 0x3c5d51ce));
04350       SDValue t1 = DAG.getNode(ISD::FSUB, dl, MVT::f32, t0,
04351                                getF32Constant(DAG, 0x3e00685a));
04352       SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t1, X);
04353       SDValue t3 = DAG.getNode(ISD::FADD, dl, MVT::f32, t2,
04354                                getF32Constant(DAG, 0x3efb6798));
04355       SDValue t4 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t3, X);
04356       SDValue t5 = DAG.getNode(ISD::FSUB, dl, MVT::f32, t4,
04357                                getF32Constant(DAG, 0x3f88d192));
04358       SDValue t6 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t5, X);
04359       SDValue t7 = DAG.getNode(ISD::FADD, dl, MVT::f32, t6,
04360                                getF32Constant(DAG, 0x3fc4316c));
04361       SDValue t8 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t7, X);
04362       Log10ofMantissa = DAG.getNode(ISD::FSUB, dl, MVT::f32, t8,
04363                                     getF32Constant(DAG, 0x3f57ce70));
04364     }
04365 
04366     return DAG.getNode(ISD::FADD, dl, MVT::f32, LogOfExponent, Log10ofMantissa);
04367   }
04368 
04369   // No special expansion.
04370   return DAG.getNode(ISD::FLOG10, dl, Op.getValueType(), Op);
04371 }
04372 
04373 /// expandExp2 - Lower an exp2 intrinsic. Handles the special sequences for
04374 /// limited-precision mode.
04375 static SDValue expandExp2(SDLoc dl, SDValue Op, SelectionDAG &DAG,
04376                           const TargetLowering &TLI) {
04377   if (Op.getValueType() == MVT::f32 &&
04378       LimitFloatPrecision > 0 && LimitFloatPrecision <= 18) {
04379     SDValue IntegerPartOfX = DAG.getNode(ISD::FP_TO_SINT, dl, MVT::i32, Op);
04380 
04381     //   FractionalPartOfX = x - (float)IntegerPartOfX;
04382     SDValue t1 = DAG.getNode(ISD::SINT_TO_FP, dl, MVT::f32, IntegerPartOfX);
04383     SDValue X = DAG.getNode(ISD::FSUB, dl, MVT::f32, Op, t1);
04384 
04385     //   IntegerPartOfX <<= 23;
04386     IntegerPartOfX = DAG.getNode(ISD::SHL, dl, MVT::i32, IntegerPartOfX,
04387                                  DAG.getConstant(23, TLI.getPointerTy()));
04388 
04389     SDValue TwoToFractionalPartOfX;
04390     if (LimitFloatPrecision <= 6) {
04391       // For floating-point precision of 6:
04392       //
04393       //   TwoToFractionalPartOfX =
04394       //     0.997535578f +
04395       //       (0.735607626f + 0.252464424f * x) * x;
04396       //
04397       // error 0.0144103317, which is 6 bits
04398       SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X,
04399                                getF32Constant(DAG, 0x3e814304));
04400       SDValue t3 = DAG.getNode(ISD::FADD, dl, MVT::f32, t2,
04401                                getF32Constant(DAG, 0x3f3c50c8));
04402       SDValue t4 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t3, X);
04403       TwoToFractionalPartOfX = DAG.getNode(ISD::FADD, dl, MVT::f32, t4,
04404                                            getF32Constant(DAG, 0x3f7f5e7e));
04405     } else if (LimitFloatPrecision <= 12) {
04406       // For floating-point precision of 12:
04407       //
04408       //   TwoToFractionalPartOfX =
04409       //     0.999892986f +
04410       //       (0.696457318f +
04411       //         (0.224338339f + 0.792043434e-1f * x) * x) * x;
04412       //
04413       // error 0.000107046256, which is 13 to 14 bits
04414       SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X,
04415                                getF32Constant(DAG, 0x3da235e3));
04416       SDValue t3 = DAG.getNode(ISD::FADD, dl, MVT::f32, t2,
04417                                getF32Constant(DAG, 0x3e65b8f3));
04418       SDValue t4 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t3, X);
04419       SDValue t5 = DAG.getNode(ISD::FADD, dl, MVT::f32, t4,
04420                                getF32Constant(DAG, 0x3f324b07));
04421       SDValue t6 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t5, X);
04422       TwoToFractionalPartOfX = DAG.getNode(ISD::FADD, dl, MVT::f32, t6,
04423                                            getF32Constant(DAG, 0x3f7ff8fd));
04424     } else { // LimitFloatPrecision <= 18
04425       // For floating-point precision of 18:
04426       //
04427       //   TwoToFractionalPartOfX =
04428       //     0.999999982f +
04429       //       (0.693148872f +
04430       //         (0.240227044f +
04431       //           (0.554906021e-1f +
04432       //             (0.961591928e-2f +
04433       //               (0.136028312e-2f + 0.157059148e-3f *x)*x)*x)*x)*x)*x;
04434       // error 2.47208000*10^(-7), which is better than 18 bits
04435       SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X,
04436                                getF32Constant(DAG, 0x3924b03e));
04437       SDValue t3 = DAG.getNode(ISD::FADD, dl, MVT::f32, t2,
04438                                getF32Constant(DAG, 0x3ab24b87));
04439       SDValue t4 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t3, X);
04440       SDValue t5 = DAG.getNode(ISD::FADD, dl, MVT::f32, t4,
04441                                getF32Constant(DAG, 0x3c1d8c17));
04442       SDValue t6 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t5, X);
04443       SDValue t7 = DAG.getNode(ISD::FADD, dl, MVT::f32, t6,
04444                                getF32Constant(DAG, 0x3d634a1d));
04445       SDValue t8 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t7, X);
04446       SDValue t9 = DAG.getNode(ISD::FADD, dl, MVT::f32, t8,
04447                                getF32Constant(DAG, 0x3e75fe14));
04448       SDValue t10 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t9, X);
04449       SDValue t11 = DAG.getNode(ISD::FADD, dl, MVT::f32, t10,
04450                                 getF32Constant(DAG, 0x3f317234));
04451       SDValue t12 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t11, X);
04452       TwoToFractionalPartOfX = DAG.getNode(ISD::FADD, dl, MVT::f32, t12,
04453                                            getF32Constant(DAG, 0x3f800000));
04454     }
04455 
04456     // Add the exponent into the result in integer domain.
04457     SDValue t13 = DAG.getNode(ISD::BITCAST, dl, MVT::i32,
04458                               TwoToFractionalPartOfX);
04459     return DAG.getNode(ISD::BITCAST, dl, MVT::f32,
04460                        DAG.getNode(ISD::ADD, dl, MVT::i32,
04461                                    t13, IntegerPartOfX));
04462   }
04463 
04464   // No special expansion.
04465   return DAG.getNode(ISD::FEXP2, dl, Op.getValueType(), Op);
04466 }
04467 
04468 /// visitPow - Lower a pow intrinsic. Handles the special sequences for
04469 /// limited-precision mode with x == 10.0f.
04470 static SDValue expandPow(SDLoc dl, SDValue LHS, SDValue RHS,
04471                          SelectionDAG &DAG, const TargetLowering &TLI) {
04472   bool IsExp10 = false;
04473   if (LHS.getValueType() == MVT::f32 && RHS.getValueType() == MVT::f32 &&
04474       LimitFloatPrecision > 0 && LimitFloatPrecision <= 18) {
04475     if (ConstantFPSDNode *LHSC = dyn_cast<ConstantFPSDNode>(LHS)) {
04476       APFloat Ten(10.0f);
04477       IsExp10 = LHSC->isExactlyValue(Ten);
04478     }
04479   }
04480 
04481   if (IsExp10) {
04482     // Put the exponent in the right bit position for later addition to the
04483     // final result:
04484     //
04485     //   #define LOG2OF10 3.3219281f
04486     //   IntegerPartOfX = (int32_t)(x * LOG2OF10);
04487     SDValue t0 = DAG.getNode(ISD::FMUL, dl, MVT::f32, RHS,
04488                              getF32Constant(DAG, 0x40549a78));
04489     SDValue IntegerPartOfX = DAG.getNode(ISD::FP_TO_SINT, dl, MVT::i32, t0);
04490 
04491     //   FractionalPartOfX = x - (float)IntegerPartOfX;
04492     SDValue t1 = DAG.getNode(ISD::SINT_TO_FP, dl, MVT::f32, IntegerPartOfX);
04493     SDValue X = DAG.getNode(ISD::FSUB, dl, MVT::f32, t0, t1);
04494 
04495     //   IntegerPartOfX <<= 23;
04496     IntegerPartOfX = DAG.getNode(ISD::SHL, dl, MVT::i32, IntegerPartOfX,
04497                                  DAG.getConstant(23, TLI.getPointerTy()));
04498 
04499     SDValue TwoToFractionalPartOfX;
04500     if (LimitFloatPrecision <= 6) {
04501       // For floating-point precision of 6:
04502       //
04503       //   twoToFractionalPartOfX =
04504       //     0.997535578f +
04505       //       (0.735607626f + 0.252464424f * x) * x;
04506       //
04507       // error 0.0144103317, which is 6 bits
04508       SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X,
04509                                getF32Constant(DAG, 0x3e814304));
04510       SDValue t3 = DAG.getNode(ISD::FADD, dl, MVT::f32, t2,
04511                                getF32Constant(DAG, 0x3f3c50c8));
04512       SDValue t4 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t3, X);
04513       TwoToFractionalPartOfX = DAG.getNode(ISD::FADD, dl, MVT::f32, t4,
04514                                            getF32Constant(DAG, 0x3f7f5e7e));
04515     } else if (LimitFloatPrecision <= 12) {
04516       // For floating-point precision of 12:
04517       //
04518       //   TwoToFractionalPartOfX =
04519       //     0.999892986f +
04520       //       (0.696457318f +
04521       //         (0.224338339f + 0.792043434e-1f * x) * x) * x;
04522       //
04523       // error 0.000107046256, which is 13 to 14 bits
04524       SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X,
04525                                getF32Constant(DAG, 0x3da235e3));
04526       SDValue t3 = DAG.getNode(ISD::FADD, dl, MVT::f32, t2,
04527                                getF32Constant(DAG, 0x3e65b8f3));
04528       SDValue t4 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t3, X);
04529       SDValue t5 = DAG.getNode(ISD::FADD, dl, MVT::f32, t4,
04530                                getF32Constant(DAG, 0x3f324b07));
04531       SDValue t6 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t5, X);
04532       TwoToFractionalPartOfX = DAG.getNode(ISD::FADD, dl, MVT::f32, t6,
04533                                            getF32Constant(DAG, 0x3f7ff8fd));
04534     } else { // LimitFloatPrecision <= 18
04535       // For floating-point precision of 18:
04536       //
04537       //   TwoToFractionalPartOfX =
04538       //     0.999999982f +
04539       //       (0.693148872f +
04540       //         (0.240227044f +
04541       //           (0.554906021e-1f +
04542       //             (0.961591928e-2f +
04543       //               (0.136028312e-2f + 0.157059148e-3f *x)*x)*x)*x)*x)*x;
04544       // error 2.47208000*10^(-7), which is better than 18 bits
04545       SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X,
04546                                getF32Constant(DAG, 0x3924b03e));
04547       SDValue t3 = DAG.getNode(ISD::FADD, dl, MVT::f32, t2,
04548                                getF32Constant(DAG, 0x3ab24b87));
04549       SDValue t4 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t3, X);
04550       SDValue t5 = DAG.getNode(ISD::FADD, dl, MVT::f32, t4,
04551                                getF32Constant(DAG, 0x3c1d8c17));
04552       SDValue t6 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t5, X);
04553       SDValue t7 = DAG.getNode(ISD::FADD, dl, MVT::f32, t6,
04554                                getF32Constant(DAG, 0x3d634a1d));
04555       SDValue t8 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t7, X);
04556       SDValue t9 = DAG.getNode(ISD::FADD, dl, MVT::f32, t8,
04557                                getF32Constant(DAG, 0x3e75fe14));
04558       SDValue t10 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t9, X);
04559       SDValue t11 = DAG.getNode(ISD::FADD, dl, MVT::f32, t10,
04560                                 getF32Constant(DAG, 0x3f317234));
04561       SDValue t12 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t11, X);
04562       TwoToFractionalPartOfX = DAG.getNode(ISD::FADD, dl, MVT::f32, t12,
04563                                            getF32Constant(DAG, 0x3f800000));
04564     }
04565 
04566     SDValue t13 = DAG.getNode(ISD::BITCAST, dl,MVT::i32,TwoToFractionalPartOfX);
04567     return DAG.getNode(ISD::BITCAST, dl, MVT::f32,
04568                        DAG.getNode(ISD::ADD, dl, MVT::i32,
04569                                    t13, IntegerPartOfX));
04570   }
04571 
04572   // No special expansion.
04573   return DAG.getNode(ISD::FPOW, dl, LHS.getValueType(), LHS, RHS);
04574 }
04575 
04576 
04577 /// ExpandPowI - Expand a llvm.powi intrinsic.
04578 static SDValue ExpandPowI(SDLoc DL, SDValue LHS, SDValue RHS,
04579                           SelectionDAG &DAG) {
04580   // If RHS is a constant, we can expand this out to a multiplication tree,
04581   // otherwise we end up lowering to a call to __powidf2 (for example).  When
04582   // optimizing for size, we only want to do this if the expansion would produce
04583   // a small number of multiplies, otherwise we do the full expansion.
04584   if (ConstantSDNode *RHSC = dyn_cast<ConstantSDNode>(RHS)) {
04585     // Get the exponent as a positive value.
04586     unsigned Val = RHSC->getSExtValue();
04587     if ((int)Val < 0) Val = -Val;
04588 
04589     // powi(x, 0) -> 1.0
04590     if (Val == 0)
04591       return DAG.getConstantFP(1.0, LHS.getValueType());
04592 
04593     const Function *F = DAG.getMachineFunction().getFunction();
04594     if (!F->hasFnAttribute(Attribute::OptimizeForSize) ||
04595         // If optimizing for size, don't insert too many multiplies.  This
04596         // inserts up to 5 multiplies.
04597         countPopulation(Val) + Log2_32(Val) < 7) {
04598       // We use the simple binary decomposition method to generate the multiply
04599       // sequence.  There are more optimal ways to do this (for example,
04600       // powi(x,15) generates one more multiply than it should), but this has
04601       // the benefit of being both really simple and much better than a libcall.
04602       SDValue Res;  // Logically starts equal to 1.0
04603       SDValue CurSquare = LHS;
04604       while (Val) {
04605         if (Val & 1) {
04606           if (Res.getNode())
04607             Res = DAG.getNode(ISD::FMUL, DL,Res.getValueType(), Res, CurSquare);
04608           else
04609             Res = CurSquare;  // 1.0*CurSquare.
04610         }
04611 
04612         CurSquare = DAG.getNode(ISD::FMUL, DL, CurSquare.getValueType(),
04613                                 CurSquare, CurSquare);
04614         Val >>= 1;
04615       }
04616 
04617       // If the original was negative, invert the result, producing 1/(x*x*x).
04618       if (RHSC->getSExtValue() < 0)
04619         Res = DAG.getNode(ISD::FDIV, DL, LHS.getValueType(),
04620                           DAG.getConstantFP(1.0, LHS.getValueType()), Res);
04621       return Res;
04622     }
04623   }
04624 
04625   // Otherwise, expand to a libcall.
04626   return DAG.getNode(ISD::FPOWI, DL, LHS.getValueType(), LHS, RHS);
04627 }
04628 
04629 // getTruncatedArgReg - Find underlying register used for an truncated
04630 // argument.
04631 static unsigned getTruncatedArgReg(const SDValue &N) {
04632   if (N.getOpcode() != ISD::TRUNCATE)
04633     return 0;
04634 
04635   const SDValue &Ext = N.getOperand(0);
04636   if (Ext.getOpcode() == ISD::AssertZext ||
04637       Ext.getOpcode() == ISD::AssertSext) {
04638     const SDValue &CFR = Ext.getOperand(0);
04639     if (CFR.getOpcode() == ISD::CopyFromReg)
04640       return cast<RegisterSDNode>(CFR.getOperand(1))->getReg();
04641     if (CFR.getOpcode() == ISD::TRUNCATE)
04642       return getTruncatedArgReg(CFR);
04643   }
04644   return 0;
04645 }
04646 
04647 /// EmitFuncArgumentDbgValue - If the DbgValueInst is a dbg_value of a function
04648 /// argument, create the corresponding DBG_VALUE machine instruction for it now.
04649 /// At the end of instruction selection, they will be inserted to the entry BB.
04650 bool SelectionDAGBuilder::EmitFuncArgumentDbgValue(const Value *V,
04651                                                    MDNode *Variable,
04652                                                    MDNode *Expr, int64_t Offset,
04653                                                    bool IsIndirect,
04654                                                    const SDValue &N) {
04655   const Argument *Arg = dyn_cast<Argument>(V);
04656   if (!Arg)
04657     return false;
04658 
04659   MachineFunction &MF = DAG.getMachineFunction();
04660   const TargetInstrInfo *TII = DAG.getSubtarget().getInstrInfo();
04661 
04662   // Ignore inlined function arguments here.
04663   DIVariable DV(Variable);
04664   if (DV.isInlinedFnArgument(MF.getFunction()))
04665     return false;
04666 
04667   Optional<MachineOperand> Op;
04668   // Some arguments' frame index is recorded during argument lowering.
04669   if (int FI = FuncInfo.getArgumentFrameIndex(Arg))
04670     Op = MachineOperand::CreateFI(FI);
04671 
04672   if (!Op && N.getNode()) {
04673     unsigned Reg;
04674     if (N.getOpcode() == ISD::CopyFromReg)
04675       Reg = cast<RegisterSDNode>(N.getOperand(1))->getReg();
04676     else
04677       Reg = getTruncatedArgReg(N);
04678     if (Reg && TargetRegisterInfo::isVirtualRegister(Reg)) {
04679       MachineRegisterInfo &RegInfo = MF.getRegInfo();
04680       unsigned PR = RegInfo.getLiveInPhysReg(Reg);
04681       if (PR)
04682         Reg = PR;
04683     }
04684     if (Reg)
04685       Op = MachineOperand::CreateReg(Reg, false);
04686   }
04687 
04688   if (!Op) {
04689     // Check if ValueMap has reg number.
04690     DenseMap<const Value *, unsigned>::iterator VMI = FuncInfo.ValueMap.find(V);
04691     if (VMI != FuncInfo.ValueMap.end())
04692       Op = MachineOperand::CreateReg(VMI->second, false);
04693   }
04694 
04695   if (!Op && N.getNode())
04696     // Check if frame index is available.
04697     if (LoadSDNode *LNode = dyn_cast<LoadSDNode>(N.getNode()))
04698       if (FrameIndexSDNode *FINode =
04699           dyn_cast<FrameIndexSDNode>(LNode->getBasePtr().getNode()))
04700         Op = MachineOperand::CreateFI(FINode->getIndex());
04701 
04702   if (!Op)
04703     return false;
04704 
04705   if (Op->isReg())
04706     FuncInfo.ArgDbgValues.push_back(
04707         BuildMI(MF, getCurDebugLoc(), TII->get(TargetOpcode::DBG_VALUE),
04708                 IsIndirect, Op->getReg(), Offset, Variable, Expr));
04709   else
04710     FuncInfo.ArgDbgValues.push_back(
04711         BuildMI(MF, getCurDebugLoc(), TII->get(TargetOpcode::DBG_VALUE))
04712             .addOperand(*Op)
04713             .addImm(Offset)
04714             .addMetadata(Variable)
04715             .addMetadata(Expr));
04716 
04717   return true;
04718 }
04719 
04720 // VisualStudio defines setjmp as _setjmp
04721 #if defined(_MSC_VER) && defined(setjmp) && \
04722                          !defined(setjmp_undefined_for_msvc)
04723 #  pragma push_macro("setjmp")
04724 #  undef setjmp
04725 #  define setjmp_undefined_for_msvc
04726 #endif
04727 
04728 /// visitIntrinsicCall - Lower the call to the specified intrinsic function.  If
04729 /// we want to emit this as a call to a named external function, return the name
04730 /// otherwise lower it and return null.
04731 const char *
04732 SelectionDAGBuilder::visitIntrinsicCall(const CallInst &I, unsigned Intrinsic) {
04733   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
04734   SDLoc sdl = getCurSDLoc();
04735   DebugLoc dl = getCurDebugLoc();
04736   SDValue Res;
04737 
04738   switch (Intrinsic) {
04739   default:
04740     // By default, turn this into a target intrinsic node.
04741     visitTargetIntrinsic(I, Intrinsic);
04742     return nullptr;
04743   case Intrinsic::vastart:  visitVAStart(I); return nullptr;
04744   case Intrinsic::vaend:    visitVAEnd(I); return nullptr;
04745   case Intrinsic::vacopy:   visitVACopy(I); return nullptr;
04746   case Intrinsic::returnaddress:
04747     setValue(&I, DAG.getNode(ISD::RETURNADDR, sdl, TLI.getPointerTy(),
04748                              getValue(I.getArgOperand(0))));
04749     return nullptr;
04750   case Intrinsic::frameaddress:
04751     setValue(&I, DAG.getNode(ISD::FRAMEADDR, sdl, TLI.getPointerTy(),
04752                              getValue(I.getArgOperand(0))));
04753     return nullptr;
04754   case Intrinsic::read_register: {
04755     Value *Reg = I.getArgOperand(0);
04756     SDValue RegName =
04757         DAG.getMDNode(cast<MDNode>(cast<MetadataAsValue>(Reg)->getMetadata()));
04758     EVT VT = TLI.getValueType(I.getType());
04759     setValue(&I, DAG.getNode(ISD::READ_REGISTER, sdl, VT, RegName));
04760     return nullptr;
04761   }
04762   case Intrinsic::write_register: {
04763     Value *Reg = I.getArgOperand(0);
04764     Value *RegValue = I.getArgOperand(1);
04765     SDValue Chain = getValue(RegValue).getOperand(0);
04766     SDValue RegName =
04767         DAG.getMDNode(cast<MDNode>(cast<MetadataAsValue>(Reg)->getMetadata()));
04768     DAG.setRoot(DAG.getNode(ISD::WRITE_REGISTER, sdl, MVT::Other, Chain,
04769                             RegName, getValue(RegValue)));
04770     return nullptr;
04771   }
04772   case Intrinsic::setjmp:
04773     return &"_setjmp"[!TLI.usesUnderscoreSetJmp()];
04774   case Intrinsic::longjmp:
04775     return &"_longjmp"[!TLI.usesUnderscoreLongJmp()];
04776   case Intrinsic::memcpy: {
04777     // FIXME: this definition of "user defined address space" is x86-specific
04778     // Assert for address < 256 since we support only user defined address
04779     // spaces.
04780     assert(cast<PointerType>(I.getArgOperand(0)->getType())->getAddressSpace()
04781            < 256 &&
04782            cast<PointerType>(I.getArgOperand(1)->getType())->getAddressSpace()
04783            < 256 &&
04784            "Unknown address space");
04785     SDValue Op1 = getValue(I.getArgOperand(0));
04786     SDValue Op2 = getValue(I.getArgOperand(1));
04787     SDValue Op3 = getValue(I.getArgOperand(2));
04788     unsigned Align = cast<ConstantInt>(I.getArgOperand(3))->getZExtValue();
04789     if (!Align)
04790       Align = 1; // @llvm.memcpy defines 0 and 1 to both mean no alignment.
04791     bool isVol = cast<ConstantInt>(I.getArgOperand(4))->getZExtValue();
04792     DAG.setRoot(DAG.getMemcpy(getRoot(), sdl, Op1, Op2, Op3, Align, isVol, false,
04793                               MachinePointerInfo(I.getArgOperand(0)),
04794                               MachinePointerInfo(I.getArgOperand(1))));
04795     return nullptr;
04796   }
04797   case Intrinsic::memset: {
04798     // FIXME: this definition of "user defined address space" is x86-specific
04799     // Assert for address < 256 since we support only user defined address
04800     // spaces.
04801     assert(cast<PointerType>(I.getArgOperand(0)->getType())->getAddressSpace()
04802            < 256 &&
04803            "Unknown address space");
04804     SDValue Op1 = getValue(I.getArgOperand(0));
04805     SDValue Op2 = getValue(I.getArgOperand(1));
04806     SDValue Op3 = getValue(I.getArgOperand(2));
04807     unsigned Align = cast<ConstantInt>(I.getArgOperand(3))->getZExtValue();
04808     if (!Align)
04809       Align = 1; // @llvm.memset defines 0 and 1 to both mean no alignment.
04810     bool isVol = cast<ConstantInt>(I.getArgOperand(4))->getZExtValue();
04811     DAG.setRoot(DAG.getMemset(getRoot(), sdl, Op1, Op2, Op3, Align, isVol,
04812                               MachinePointerInfo(I.getArgOperand(0))));
04813     return nullptr;
04814   }
04815   case Intrinsic::memmove: {
04816     // FIXME: this definition of "user defined address space" is x86-specific
04817     // Assert for address < 256 since we support only user defined address
04818     // spaces.
04819     assert(cast<PointerType>(I.getArgOperand(0)->getType())->getAddressSpace()
04820            < 256 &&
04821            cast<PointerType>(I.getArgOperand(1)->getType())->getAddressSpace()
04822            < 256 &&
04823            "Unknown address space");
04824     SDValue Op1 = getValue(I.getArgOperand(0));
04825     SDValue Op2 = getValue(I.getArgOperand(1));
04826     SDValue Op3 = getValue(I.getArgOperand(2));
04827     unsigned Align = cast<ConstantInt>(I.getArgOperand(3))->getZExtValue();
04828     if (!Align)
04829       Align = 1; // @llvm.memmove defines 0 and 1 to both mean no alignment.
04830     bool isVol = cast<ConstantInt>(I.getArgOperand(4))->getZExtValue();
04831     DAG.setRoot(DAG.getMemmove(getRoot(), sdl, Op1, Op2, Op3, Align, isVol,
04832                                MachinePointerInfo(I.getArgOperand(0)),
04833                                MachinePointerInfo(I.getArgOperand(1))));
04834     return nullptr;
04835   }
04836   case Intrinsic::dbg_declare: {
04837     const DbgDeclareInst &DI = cast<DbgDeclareInst>(I);
04838     MDNode *Variable = DI.getVariable();
04839     MDNode *Expression = DI.getExpression();
04840     const Value *Address = DI.getAddress();
04841     DIVariable DIVar(Variable);
04842     assert((!DIVar || DIVar.isVariable()) &&
04843       "Variable in DbgDeclareInst should be either null or a DIVariable.");
04844     if (!Address || !DIVar) {
04845       DEBUG(dbgs() << "Dropping debug info for " << DI << "\n");
04846       return nullptr;
04847     }
04848 
04849     // Check if address has undef value.
04850     if (isa<UndefValue>(Address) ||
04851         (Address->use_empty() && !isa<Argument>(Address))) {
04852       DEBUG(dbgs() << "Dropping debug info for " << DI << "\n");
04853       return nullptr;
04854     }
04855 
04856     SDValue &N = NodeMap[Address];
04857     if (!N.getNode() && isa<Argument>(Address))
04858       // Check unused arguments map.
04859       N = UnusedArgNodeMap[Address];
04860     SDDbgValue *SDV;
04861     if (N.getNode()) {
04862       if (const BitCastInst *BCI = dyn_cast<BitCastInst>(Address))
04863         Address = BCI->getOperand(0);
04864       // Parameters are handled specially.
04865       bool isParameter =
04866         (DIVariable(Variable).getTag() == dwarf::DW_TAG_arg_variable ||
04867          isa<Argument>(Address));
04868 
04869       const AllocaInst *AI = dyn_cast<AllocaInst>(Address);
04870 
04871       if (isParameter && !AI) {
04872         FrameIndexSDNode *FINode = dyn_cast<FrameIndexSDNode>(N.getNode());
04873         if (FINode)
04874           // Byval parameter.  We have a frame index at this point.
04875           SDV = DAG.getFrameIndexDbgValue(
04876               Variable, Expression, FINode->getIndex(), 0, dl, SDNodeOrder);
04877         else {
04878           // Address is an argument, so try to emit its dbg value using
04879           // virtual register info from the FuncInfo.ValueMap.
04880           EmitFuncArgumentDbgValue(Address, Variable, Expression, 0, false, N);
04881           return nullptr;
04882         }
04883       } else if (AI)
04884         SDV = DAG.getDbgValue(Variable, Expression, N.getNode(), N.getResNo(),
04885                               true, 0, dl, SDNodeOrder);
04886       else {
04887         // Can't do anything with other non-AI cases yet.
04888         DEBUG(dbgs() << "Dropping debug info for " << DI << "\n");
04889         DEBUG(dbgs() << "non-AllocaInst issue for Address: \n\t");
04890         DEBUG(Address->dump());
04891         return nullptr;
04892       }
04893       DAG.AddDbgValue(SDV, N.getNode(), isParameter);
04894     } else {
04895       // If Address is an argument then try to emit its dbg value using
04896       // virtual register info from the FuncInfo.ValueMap.
04897       if (!EmitFuncArgumentDbgValue(Address, Variable, Expression, 0, false,
04898                                     N)) {
04899         // If variable is pinned by a alloca in dominating bb then
04900         // use StaticAllocaMap.
04901         if (const AllocaInst *AI = dyn_cast<AllocaInst>(Address)) {
04902           if (AI->getParent() != DI.getParent()) {
04903             DenseMap<const AllocaInst*, int>::iterator SI =
04904               FuncInfo.StaticAllocaMap.find(AI);
04905             if (SI != FuncInfo.StaticAllocaMap.end()) {
04906               SDV = DAG.getFrameIndexDbgValue(Variable, Expression, SI->second,
04907                                               0, dl, SDNodeOrder);
04908               DAG.AddDbgValue(SDV, nullptr, false);
04909               return nullptr;
04910             }
04911           }
04912         }
04913         DEBUG(dbgs() << "Dropping debug info for " << DI << "\n");
04914       }
04915     }
04916     return nullptr;
04917   }
04918   case Intrinsic::dbg_value: {
04919     const DbgValueInst &DI = cast<DbgValueInst>(I);
04920     DIVariable DIVar(DI.getVariable());
04921     assert((!DIVar || DIVar.isVariable()) &&
04922       "Variable in DbgValueInst should be either null or a DIVariable.");
04923     if (!DIVar)
04924       return nullptr;
04925 
04926     MDNode *Variable = DI.getVariable();
04927     MDNode *Expression = DI.getExpression();
04928     uint64_t Offset = DI.getOffset();
04929     const Value *V = DI.getValue();
04930     if (!V)
04931       return nullptr;
04932 
04933     SDDbgValue *SDV;
04934     if (isa<ConstantInt>(V) || isa<ConstantFP>(V) || isa<UndefValue>(V)) {
04935       SDV = DAG.getConstantDbgValue(Variable, Expression, V, Offset, dl,
04936                                     SDNodeOrder);
04937       DAG.AddDbgValue(SDV, nullptr, false);
04938     } else {
04939       // Do not use getValue() in here; we don't want to generate code at
04940       // this point if it hasn't been done yet.
04941       SDValue N = NodeMap[V];
04942       if (!N.getNode() && isa<Argument>(V))
04943         // Check unused arguments map.
04944         N = UnusedArgNodeMap[V];
04945       if (N.getNode()) {
04946         // A dbg.value for an alloca is always indirect.
04947         bool IsIndirect = isa<AllocaInst>(V) || Offset != 0;
04948         if (!EmitFuncArgumentDbgValue(V, Variable, Expression, Offset,
04949                                       IsIndirect, N)) {
04950           SDV = DAG.getDbgValue(Variable, Expression, N.getNode(), N.getResNo(),
04951                                 IsIndirect, Offset, dl, SDNodeOrder);
04952           DAG.AddDbgValue(SDV, N.getNode(), false);
04953         }
04954       } else if (!V->use_empty() ) {
04955         // Do not call getValue(V) yet, as we don't want to generate code.
04956         // Remember it for later.
04957         DanglingDebugInfo DDI(&DI, dl, SDNodeOrder);
04958         DanglingDebugInfoMap[V] = DDI;
04959       } else {
04960         // We may expand this to cover more cases.  One case where we have no
04961         // data available is an unreferenced parameter.
04962         DEBUG(dbgs() << "Dropping debug info for " << DI << "\n");
04963       }
04964     }
04965 
04966     // Build a debug info table entry.
04967     if (const BitCastInst *BCI = dyn_cast<BitCastInst>(V))
04968       V = BCI->getOperand(0);
04969     const AllocaInst *AI = dyn_cast<AllocaInst>(V);
04970     // Don't handle byval struct arguments or VLAs, for example.
04971     if (!AI) {
04972       DEBUG(dbgs() << "Dropping debug location info for:\n  " << DI << "\n");
04973       DEBUG(dbgs() << "  Last seen at:\n    " << *V << "\n");
04974       return nullptr;
04975     }
04976     DenseMap<const AllocaInst*, int>::iterator SI =
04977       FuncInfo.StaticAllocaMap.find(AI);
04978     if (SI == FuncInfo.StaticAllocaMap.end())
04979       return nullptr; // VLAs.
04980     return nullptr;
04981   }
04982 
04983   case Intrinsic::eh_typeid_for: {
04984     // Find the type id for the given typeinfo.
04985     GlobalValue *GV = ExtractTypeInfo(I.getArgOperand(0));
04986     unsigned TypeID = DAG.getMachineFunction().getMMI().getTypeIDFor(GV);
04987     Res = DAG.getConstant(TypeID, MVT::i32);
04988     setValue(&I, Res);
04989     return nullptr;
04990   }
04991 
04992   case Intrinsic::eh_return_i32:
04993   case Intrinsic::eh_return_i64:
04994     DAG.getMachineFunction().getMMI().setCallsEHReturn(true);
04995     DAG.setRoot(DAG.getNode(ISD::EH_RETURN, sdl,
04996                             MVT::Other,
04997                             getControlRoot(),
04998                             getValue(I.getArgOperand(0)),
04999                             getValue(I.getArgOperand(1))));
05000     return nullptr;
05001   case Intrinsic::eh_unwind_init:
05002     DAG.getMachineFunction().getMMI().setCallsUnwindInit(true);
05003     return nullptr;
05004   case Intrinsic::eh_dwarf_cfa: {
05005     SDValue CfaArg = DAG.getSExtOrTrunc(getValue(I.getArgOperand(0)), sdl,
05006                                         TLI.getPointerTy());
05007     SDValue Offset = DAG.getNode(ISD::ADD, sdl,
05008                                  CfaArg.getValueType(),
05009                                  DAG.getNode(ISD::FRAME_TO_ARGS_OFFSET, sdl,
05010                                              CfaArg.getValueType()),
05011                                  CfaArg);
05012     SDValue FA = DAG.getNode(ISD::FRAMEADDR, sdl, TLI.getPointerTy(),
05013                              DAG.getConstant(0, TLI.getPointerTy()));
05014     setValue(&I, DAG.getNode(ISD::ADD, sdl, FA.getValueType(),
05015                              FA, Offset));
05016     return nullptr;
05017   }
05018   case Intrinsic::eh_sjlj_callsite: {
05019     MachineModuleInfo &MMI = DAG.getMachineFunction().getMMI();
05020     ConstantInt *CI = dyn_cast<ConstantInt>(I.getArgOperand(0));
05021     assert(CI && "Non-constant call site value in eh.sjlj.callsite!");
05022     assert(MMI.getCurrentCallSite() == 0 && "Overlapping call sites!");
05023 
05024     MMI.setCurrentCallSite(CI->getZExtValue());
05025     return nullptr;
05026   }
05027   case Intrinsic::eh_sjlj_functioncontext: {
05028     // Get and store the index of the function context.
05029     MachineFrameInfo *MFI = DAG.getMachineFunction().getFrameInfo();
05030     AllocaInst *FnCtx =
05031       cast<AllocaInst>(I.getArgOperand(0)->stripPointerCasts());
05032     int FI = FuncInfo.StaticAllocaMap[FnCtx];
05033     MFI->setFunctionContextIndex(FI);
05034     return nullptr;
05035   }
05036   case Intrinsic::eh_sjlj_setjmp: {
05037     SDValue Ops[2];
05038     Ops[0] = getRoot();
05039     Ops[1] = getValue(I.getArgOperand(0));
05040     SDValue Op = DAG.getNode(ISD::EH_SJLJ_SETJMP, sdl,
05041                              DAG.getVTList(MVT::i32, MVT::Other), Ops);
05042     setValue(&I, Op.getValue(0));
05043     DAG.setRoot(Op.getValue(1));
05044     return nullptr;
05045   }
05046   case Intrinsic::eh_sjlj_longjmp: {
05047     DAG.setRoot(DAG.getNode(ISD::EH_SJLJ_LONGJMP, sdl, MVT::Other,
05048                             getRoot(), getValue(I.getArgOperand(0))));
05049     return nullptr;
05050   }
05051 
05052   case Intrinsic::masked_load:
05053     visitMaskedLoad(I);
05054     return nullptr;
05055   case Intrinsic::masked_store:
05056     visitMaskedStore(I);
05057     return nullptr;
05058   case Intrinsic::x86_mmx_pslli_w:
05059   case Intrinsic::x86_mmx_pslli_d:
05060   case Intrinsic::x86_mmx_pslli_q:
05061   case Intrinsic::x86_mmx_psrli_w:
05062   case Intrinsic::x86_mmx_psrli_d:
05063   case Intrinsic::x86_mmx_psrli_q:
05064   case Intrinsic::x86_mmx_psrai_w:
05065   case Intrinsic::x86_mmx_psrai_d: {
05066     SDValue ShAmt = getValue(I.getArgOperand(1));
05067     if (isa<ConstantSDNode>(ShAmt)) {
05068       visitTargetIntrinsic(I, Intrinsic);
05069       return nullptr;
05070     }
05071     unsigned NewIntrinsic = 0;
05072     EVT ShAmtVT = MVT::v2i32;
05073     switch (Intrinsic) {
05074     case Intrinsic::x86_mmx_pslli_w:
05075       NewIntrinsic = Intrinsic::x86_mmx_psll_w;
05076       break;
05077     case Intrinsic::x86_mmx_pslli_d:
05078       NewIntrinsic = Intrinsic::x86_mmx_psll_d;
05079       break;
05080     case Intrinsic::x86_mmx_pslli_q:
05081       NewIntrinsic = Intrinsic::x86_mmx_psll_q;
05082       break;
05083     case Intrinsic::x86_mmx_psrli_w:
05084       NewIntrinsic = Intrinsic::x86_mmx_psrl_w;
05085       break;
05086     case Intrinsic::x86_mmx_psrli_d:
05087       NewIntrinsic = Intrinsic::x86_mmx_psrl_d;
05088       break;
05089     case Intrinsic::x86_mmx_psrli_q:
05090       NewIntrinsic = Intrinsic::x86_mmx_psrl_q;
05091       break;
05092     case Intrinsic::x86_mmx_psrai_w:
05093       NewIntrinsic = Intrinsic::x86_mmx_psra_w;
05094       break;
05095     case Intrinsic::x86_mmx_psrai_d:
05096       NewIntrinsic = Intrinsic::x86_mmx_psra_d;
05097       break;
05098     default: llvm_unreachable("Impossible intrinsic");  // Can't reach here.
05099     }
05100 
05101     // The vector shift intrinsics with scalars uses 32b shift amounts but
05102     // the sse2/mmx shift instructions reads 64 bits. Set the upper 32 bits
05103     // to be zero.
05104     // We must do this early because v2i32 is not a legal type.
05105     SDValue ShOps[2];
05106     ShOps[0] = ShAmt;
05107     ShOps[1] = DAG.getConstant(0, MVT::i32);
05108     ShAmt =  DAG.getNode(ISD::BUILD_VECTOR, sdl, ShAmtVT, ShOps);
05109     EVT DestVT = TLI.getValueType(I.getType());
05110     ShAmt = DAG.getNode(ISD::BITCAST, sdl, DestVT, ShAmt);
05111     Res = DAG.getNode(ISD::INTRINSIC_WO_CHAIN, sdl, DestVT,
05112                        DAG.getConstant(NewIntrinsic, MVT::i32),
05113                        getValue(I.getArgOperand(0)), ShAmt);
05114     setValue(&I, Res);
05115     return nullptr;
05116   }
05117   case Intrinsic::x86_avx_vinsertf128_pd_256:
05118   case Intrinsic::x86_avx_vinsertf128_ps_256:
05119   case Intrinsic::x86_avx_vinsertf128_si_256:
05120   case Intrinsic::x86_avx2_vinserti128: {
05121     EVT DestVT = TLI.getValueType(I.getType());
05122     EVT ElVT = TLI.getValueType(I.getArgOperand(1)->getType());
05123     uint64_t Idx = (cast<ConstantInt>(I.getArgOperand(2))->getZExtValue() & 1) *
05124                    ElVT.getVectorNumElements();
05125     Res =
05126         DAG.getNode(ISD::INSERT_SUBVECTOR, sdl, DestVT,
05127                     getValue(I.getArgOperand(0)), getValue(I.getArgOperand(1)),
05128                     DAG.getConstant(Idx, TLI.getVectorIdxTy()));
05129     setValue(&I, Res);
05130     return nullptr;
05131   }
05132   case Intrinsic::x86_avx_vextractf128_pd_256:
05133   case Intrinsic::x86_avx_vextractf128_ps_256:
05134   case Intrinsic::x86_avx_vextractf128_si_256:
05135   case Intrinsic::x86_avx2_vextracti128: {
05136     EVT DestVT = TLI.getValueType(I.getType());
05137     uint64_t Idx = (cast<ConstantInt>(I.getArgOperand(1))->getZExtValue() & 1) *
05138                    DestVT.getVectorNumElements();
05139     Res = DAG.getNode(ISD::EXTRACT_SUBVECTOR, sdl, DestVT,
05140                       getValue(I.getArgOperand(0)),
05141                       DAG.getConstant(Idx, TLI.getVectorIdxTy()));
05142     setValue(&I, Res);
05143     return nullptr;
05144   }
05145   case Intrinsic::convertff:
05146   case Intrinsic::convertfsi:
05147   case Intrinsic::convertfui:
05148   case Intrinsic::convertsif:
05149   case Intrinsic::convertuif:
05150   case Intrinsic::convertss:
05151   case Intrinsic::convertsu:
05152   case Intrinsic::convertus:
05153   case Intrinsic::convertuu: {
05154     ISD::CvtCode Code = ISD::CVT_INVALID;
05155     switch (Intrinsic) {
05156     default: llvm_unreachable("Impossible intrinsic");  // Can't reach here.
05157     case Intrinsic::convertff:  Code = ISD::CVT_FF; break;
05158     case Intrinsic::convertfsi: Code = ISD::CVT_FS; break;
05159     case Intrinsic::convertfui: Code = ISD::CVT_FU; break;
05160     case Intrinsic::convertsif: Code = ISD::CVT_SF; break;
05161     case Intrinsic::convertuif: Code = ISD::CVT_UF; break;
05162     case Intrinsic::convertss:  Code = ISD::CVT_SS; break;
05163     case Intrinsic::convertsu:  Code = ISD::CVT_SU; break;
05164     case Intrinsic::convertus:  Code = ISD::CVT_US; break;
05165     case Intrinsic::convertuu:  Code = ISD::CVT_UU; break;
05166     }
05167     EVT DestVT = TLI.getValueType(I.getType());
05168     const Value *Op1 = I.getArgOperand(0);
05169     Res = DAG.getConvertRndSat(DestVT, sdl, getValue(Op1),
05170                                DAG.getValueType(DestVT),
05171                                DAG.getValueType(getValue(Op1).getValueType()),
05172                                getValue(I.getArgOperand(1)),
05173                                getValue(I.getArgOperand(2)),
05174                                Code);
05175     setValue(&I, Res);
05176     return nullptr;
05177   }
05178   case Intrinsic::powi:
05179     setValue(&I, ExpandPowI(sdl, getValue(I.getArgOperand(0)),
05180                             getValue(I.getArgOperand(1)), DAG));
05181     return nullptr;
05182   case Intrinsic::log:
05183     setValue(&I, expandLog(sdl, getValue(I.getArgOperand(0)), DAG, TLI));
05184     return nullptr;
05185   case Intrinsic::log2:
05186     setValue(&I, expandLog2(sdl, getValue(I.getArgOperand(0)), DAG, TLI));
05187     return nullptr;
05188   case Intrinsic::log10:
05189     setValue(&I, expandLog10(sdl, getValue(I.getArgOperand(0)), DAG, TLI));
05190     return nullptr;
05191   case Intrinsic::exp:
05192     setValue(&I, expandExp(sdl, getValue(I.getArgOperand(0)), DAG, TLI));
05193     return nullptr;
05194   case Intrinsic::exp2:
05195     setValue(&I, expandExp2(sdl, getValue(I.getArgOperand(0)), DAG, TLI));
05196     return nullptr;
05197   case Intrinsic::pow:
05198     setValue(&I, expandPow(sdl, getValue(I.getArgOperand(0)),
05199                            getValue(I.getArgOperand(1)), DAG, TLI));
05200     return nullptr;
05201   case Intrinsic::sqrt:
05202   case Intrinsic::fabs:
05203   case Intrinsic::sin:
05204   case Intrinsic::cos:
05205   case Intrinsic::floor:
05206   case Intrinsic::ceil:
05207   case Intrinsic::trunc:
05208   case Intrinsic::rint:
05209   case Intrinsic::nearbyint:
05210   case Intrinsic::round: {
05211     unsigned Opcode;
05212     switch (Intrinsic) {
05213     default: llvm_unreachable("Impossible intrinsic");  // Can't reach here.
05214     case Intrinsic::sqrt:      Opcode = ISD::FSQRT;      break;
05215     case Intrinsic::fabs:      Opcode = ISD::FABS;       break;
05216     case Intrinsic::sin:       Opcode = ISD::FSIN;       break;
05217     case Intrinsic::cos:       Opcode = ISD::FCOS;       break;
05218     case Intrinsic::floor:     Opcode = ISD::FFLOOR;     break;
05219     case Intrinsic::ceil:      Opcode = ISD::FCEIL;      break;
05220     case Intrinsic::trunc:     Opcode = ISD::FTRUNC;     break;
05221     case Intrinsic::rint:      Opcode = ISD::FRINT;      break;
05222     case Intrinsic::nearbyint: Opcode = ISD::FNEARBYINT; break;
05223     case Intrinsic::round:     Opcode = ISD::FROUND;     break;
05224     }
05225 
05226     setValue(&I, DAG.getNode(Opcode, sdl,
05227                              getValue(I.getArgOperand(0)).getValueType(),
05228                              getValue(I.getArgOperand(0))));
05229     return nullptr;
05230   }
05231   case Intrinsic::minnum:
05232     setValue(&I, DAG.getNode(ISD::FMINNUM, sdl,
05233                              getValue(I.getArgOperand(0)).getValueType(),
05234                              getValue(I.getArgOperand(0)),
05235                              getValue(I.getArgOperand(1))));
05236     return nullptr;
05237   case Intrinsic::maxnum:
05238     setValue(&I, DAG.getNode(ISD::FMAXNUM, sdl,
05239                              getValue(I.getArgOperand(0)).getValueType(),
05240                              getValue(I.getArgOperand(0)),
05241                              getValue(I.getArgOperand(1))));
05242     return nullptr;
05243   case Intrinsic::copysign:
05244     setValue(&I, DAG.getNode(ISD::FCOPYSIGN, sdl,
05245                              getValue(I.getArgOperand(0)).getValueType(),
05246                              getValue(I.getArgOperand(0)),
05247                              getValue(I.getArgOperand(1))));
05248     return nullptr;
05249   case Intrinsic::fma:
05250     setValue(&I, DAG.getNode(ISD::FMA, sdl,
05251                              getValue(I.getArgOperand(0)).getValueType(),
05252                              getValue(I.getArgOperand(0)),
05253                              getValue(I.getArgOperand(1)),
05254                              getValue(I.getArgOperand(2))));
05255     return nullptr;
05256   case Intrinsic::fmuladd: {
05257     EVT VT = TLI.getValueType(I.getType());
05258     if (TM.Options.AllowFPOpFusion != FPOpFusion::Strict &&
05259         TLI.isFMAFasterThanFMulAndFAdd(VT)) {
05260       setValue(&I, DAG.getNode(ISD::FMA, sdl,
05261                                getValue(I.getArgOperand(0)).getValueType(),
05262                                getValue(I.getArgOperand(0)),
05263                                getValue(I.getArgOperand(1)),
05264                                getValue(I.getArgOperand(2))));
05265     } else {
05266       SDValue Mul = DAG.getNode(ISD::FMUL, sdl,
05267                                 getValue(I.getArgOperand(0)).getValueType(),
05268                                 getValue(I.getArgOperand(0)),
05269                                 getValue(I.getArgOperand(1)));
05270       SDValue Add = DAG.getNode(ISD::FADD, sdl,
05271                                 getValue(I.getArgOperand(0)).getValueType(),
05272                                 Mul,
05273                                 getValue(I.getArgOperand(2)));
05274       setValue(&I, Add);
05275     }
05276     return nullptr;
05277   }
05278   case Intrinsic::convert_to_fp16:
05279     setValue(&I, DAG.getNode(ISD::BITCAST, sdl, MVT::i16,
05280                              DAG.getNode(ISD::FP_ROUND, sdl, MVT::f16,
05281                                          getValue(I.getArgOperand(0)),
05282                                          DAG.getTargetConstant(0, MVT::i32))));
05283     return nullptr;
05284   case Intrinsic::convert_from_fp16:
05285     setValue(&I,
05286              DAG.getNode(ISD::FP_EXTEND, sdl, TLI.getValueType(I.getType()),
05287                          DAG.getNode(ISD::BITCAST, sdl, MVT::f16,
05288                                      getValue(I.getArgOperand(0)))));
05289     return nullptr;
05290   case Intrinsic::pcmarker: {
05291     SDValue Tmp = getValue(I.getArgOperand(0));
05292     DAG.setRoot(DAG.getNode(ISD::PCMARKER, sdl, MVT::Other, getRoot(), Tmp));
05293     return nullptr;
05294   }
05295   case Intrinsic::readcyclecounter: {
05296     SDValue Op = getRoot();
05297     Res = DAG.getNode(ISD::READCYCLECOUNTER, sdl,
05298                       DAG.getVTList(MVT::i64, MVT::Other), Op);
05299     setValue(&I, Res);
05300     DAG.setRoot(Res.getValue(1));
05301     return nullptr;
05302   }
05303   case Intrinsic::bswap:
05304     setValue(&I, DAG.getNode(ISD::BSWAP, sdl,
05305                              getValue(I.getArgOperand(0)).getValueType(),
05306                              getValue(I.getArgOperand(0))));
05307     return nullptr;
05308   case Intrinsic::cttz: {
05309     SDValue Arg = getValue(I.getArgOperand(0));
05310     ConstantInt *CI = cast<ConstantInt>(I.getArgOperand(1));
05311     EVT Ty = Arg.getValueType();
05312     setValue(&I, DAG.getNode(CI->isZero() ? ISD::CTTZ : ISD::CTTZ_ZERO_UNDEF,
05313                              sdl, Ty, Arg));
05314     return nullptr;
05315   }
05316   case Intrinsic::ctlz: {
05317     SDValue Arg = getValue(I.getArgOperand(0));
05318     ConstantInt *CI = cast<ConstantInt>(I.getArgOperand(1));
05319     EVT Ty = Arg.getValueType();
05320     setValue(&I, DAG.getNode(CI->isZero() ? ISD::CTLZ : ISD::CTLZ_ZERO_UNDEF,
05321                              sdl, Ty, Arg));
05322     return nullptr;
05323   }
05324   case Intrinsic::ctpop: {
05325     SDValue Arg = getValue(I.getArgOperand(0));
05326     EVT Ty = Arg.getValueType();
05327     setValue(&I, DAG.getNode(ISD::CTPOP, sdl, Ty, Arg));
05328     return nullptr;
05329   }
05330   case Intrinsic::stacksave: {
05331     SDValue Op = getRoot();
05332     Res = DAG.getNode(ISD::STACKSAVE, sdl,
05333                       DAG.getVTList(TLI.getPointerTy(), MVT::Other), Op);
05334     setValue(&I, Res);
05335     DAG.setRoot(Res.getValue(1));
05336     return nullptr;
05337   }
05338   case Intrinsic::stackrestore: {
05339     Res = getValue(I.getArgOperand(0));
05340     DAG.setRoot(DAG.getNode(ISD::STACKRESTORE, sdl, MVT::Other, getRoot(), Res));
05341     return nullptr;
05342   }
05343   case Intrinsic::stackprotector: {
05344     // Emit code into the DAG to store the stack guard onto the stack.
05345     MachineFunction &MF = DAG.getMachineFunction();
05346     MachineFrameInfo *MFI = MF.getFrameInfo();
05347     EVT PtrTy = TLI.getPointerTy();
05348     SDValue Src, Chain = getRoot();
05349     const Value *Ptr = cast<LoadInst>(I.getArgOperand(0))->getPointerOperand();
05350     const GlobalVariable *GV = dyn_cast<GlobalVariable>(Ptr);
05351 
05352     // See if Ptr is a bitcast. If it is, look through it and see if we can get
05353     // global variable __stack_chk_guard.
05354     if (!GV)
05355       if (const Operator *BC = dyn_cast<Operator>(Ptr))
05356         if (BC->getOpcode() == Instruction::BitCast)
05357           GV = dyn_cast<GlobalVariable>(BC->getOperand(0));
05358 
05359     if (GV && TLI.useLoadStackGuardNode()) {
05360       // Emit a LOAD_STACK_GUARD node.
05361       MachineSDNode *Node = DAG.getMachineNode(TargetOpcode::LOAD_STACK_GUARD,
05362                                                sdl, PtrTy, Chain);
05363       MachinePointerInfo MPInfo(GV);
05364       MachineInstr::mmo_iterator MemRefs = MF.allocateMemRefsArray(1);
05365       unsigned Flags = MachineMemOperand::MOLoad |
05366                        MachineMemOperand::MOInvariant;
05367       *MemRefs = MF.getMachineMemOperand(MPInfo, Flags,
05368                                          PtrTy.getSizeInBits() / 8,
05369                                          DAG.getEVTAlignment(PtrTy));
05370       Node->setMemRefs(MemRefs, MemRefs + 1);
05371 
05372       // Copy the guard value to a virtual register so that it can be
05373       // retrieved in the epilogue.
05374       Src = SDValue(Node, 0);
05375       const TargetRegisterClass *RC =
05376           TLI.getRegClassFor(Src.getSimpleValueType());
05377       unsigned Reg = MF.getRegInfo().createVirtualRegister(RC);
05378 
05379       SPDescriptor.setGuardReg(Reg);
05380       Chain = DAG.getCopyToReg(Chain, sdl, Reg, Src);
05381     } else {
05382       Src = getValue(I.getArgOperand(0));   // The guard's value.
05383     }
05384 
05385     AllocaInst *Slot = cast<AllocaInst>(I.getArgOperand(1));
05386 
05387     int FI = FuncInfo.StaticAllocaMap[Slot];
05388     MFI->setStackProtectorIndex(FI);
05389 
05390     SDValue FIN = DAG.getFrameIndex(FI, PtrTy);
05391 
05392     // Store the stack protector onto the stack.
05393     Res = DAG.getStore(Chain, sdl, Src, FIN,
05394                        MachinePointerInfo::getFixedStack(FI),
05395                        true, false, 0);
05396     setValue(&I, Res);
05397     DAG.setRoot(Res);
05398     return nullptr;
05399   }
05400   case Intrinsic::objectsize: {
05401     // If we don't know by now, we're never going to know.
05402     ConstantInt *CI = dyn_cast<ConstantInt>(I.getArgOperand(1));
05403 
05404     assert(CI && "Non-constant type in __builtin_object_size?");
05405 
05406     SDValue Arg = getValue(I.getCalledValue());
05407     EVT Ty = Arg.getValueType();
05408 
05409     if (CI->isZero())
05410       Res = DAG.getConstant(-1ULL, Ty);
05411     else
05412       Res = DAG.getConstant(0, Ty);
05413 
05414     setValue(&I, Res);
05415     return nullptr;
05416   }
05417   case Intrinsic::annotation:
05418   case Intrinsic::ptr_annotation:
05419     // Drop the intrinsic, but forward the value
05420     setValue(&I, getValue(I.getOperand(0)));
05421     return nullptr;
05422   case Intrinsic::assume:
05423   case Intrinsic::var_annotation:
05424     // Discard annotate attributes and assumptions
05425     return nullptr;
05426 
05427   case Intrinsic::init_trampoline: {
05428     const Function *F = cast<Function>(I.getArgOperand(1)->stripPointerCasts());
05429 
05430     SDValue Ops[6];
05431     Ops[0] = getRoot();
05432     Ops[1] = getValue(I.getArgOperand(0));
05433     Ops[2] = getValue(I.getArgOperand(1));
05434     Ops[3] = getValue(I.getArgOperand(2));
05435     Ops[4] = DAG.getSrcValue(I.getArgOperand(0));
05436     Ops[5] = DAG.getSrcValue(F);
05437 
05438     Res = DAG.getNode(ISD::INIT_TRAMPOLINE, sdl, MVT::Other, Ops);
05439 
05440     DAG.setRoot(Res);
05441     return nullptr;
05442   }
05443   case Intrinsic::adjust_trampoline: {
05444     setValue(&I, DAG.getNode(ISD::ADJUST_TRAMPOLINE, sdl,
05445                              TLI.getPointerTy(),
05446                              getValue(I.getArgOperand(0))));
05447     return nullptr;
05448   }
05449   case Intrinsic::gcroot:
05450     if (GFI) {
05451       const Value *Alloca = I.getArgOperand(0)->stripPointerCasts();
05452       const Constant *TypeMap = cast<Constant>(I.getArgOperand(1));
05453 
05454       FrameIndexSDNode *FI = cast<FrameIndexSDNode>(getValue(Alloca).getNode());
05455       GFI->addStackRoot(FI->getIndex(), TypeMap);
05456     }
05457     return nullptr;
05458   case Intrinsic::gcread:
05459   case Intrinsic::gcwrite:
05460     llvm_unreachable("GC failed to lower gcread/gcwrite intrinsics!");
05461   case Intrinsic::flt_rounds:
05462     setValue(&I, DAG.getNode(ISD::FLT_ROUNDS_, sdl, MVT::i32));
05463     return nullptr;
05464 
05465   case Intrinsic::expect: {
05466     // Just replace __builtin_expect(exp, c) with EXP.
05467     setValue(&I, getValue(I.getArgOperand(0)));
05468     return nullptr;
05469   }
05470 
05471   case Intrinsic::debugtrap:
05472   case Intrinsic::trap: {
05473     StringRef TrapFuncName = TM.Options.getTrapFunctionName();
05474     if (TrapFuncName.empty()) {
05475       ISD::NodeType Op = (Intrinsic == Intrinsic::trap) ?
05476         ISD::TRAP : ISD::DEBUGTRAP;
05477       DAG.setRoot(DAG.getNode(Op, sdl,MVT::Other, getRoot()));
05478       return nullptr;
05479     }
05480     TargetLowering::ArgListTy Args;
05481 
05482     TargetLowering::CallLoweringInfo CLI(DAG);
05483     CLI.setDebugLoc(sdl).setChain(getRoot())
05484       .setCallee(CallingConv::C, I.getType(),
05485                  DAG.getExternalSymbol(TrapFuncName.data(), TLI.getPointerTy()),
05486                  std::move(Args), 0);
05487 
05488     std::pair<SDValue, SDValue> Result = TLI.LowerCallTo(CLI);
05489     DAG.setRoot(Result.second);
05490     return nullptr;
05491   }
05492 
05493   case Intrinsic::uadd_with_overflow:
05494   case Intrinsic::sadd_with_overflow:
05495   case Intrinsic::usub_with_overflow:
05496   case Intrinsic::ssub_with_overflow:
05497   case Intrinsic::umul_with_overflow:
05498   case Intrinsic::smul_with_overflow: {
05499     ISD::NodeType Op;
05500     switch (Intrinsic) {
05501     default: llvm_unreachable("Impossible intrinsic");  // Can't reach here.
05502     case Intrinsic::uadd_with_overflow: Op = ISD::UADDO; break;
05503     case Intrinsic::sadd_with_overflow: Op = ISD::SADDO; break;
05504     case Intrinsic::usub_with_overflow: Op = ISD::USUBO; break;
05505     case Intrinsic::ssub_with_overflow: Op = ISD::SSUBO; break;
05506     case Intrinsic::umul_with_overflow: Op = ISD::UMULO; break;
05507     case Intrinsic::smul_with_overflow: Op = ISD::SMULO; break;
05508     }
05509     SDValue Op1 = getValue(I.getArgOperand(0));
05510     SDValue Op2 = getValue(I.getArgOperand(1));
05511 
05512     SDVTList VTs = DAG.getVTList(Op1.getValueType(), MVT::i1);
05513     setValue(&I, DAG.getNode(Op, sdl, VTs, Op1, Op2));
05514     return nullptr;
05515   }
05516   case Intrinsic::prefetch: {
05517     SDValue Ops[5];
05518     unsigned rw = cast<ConstantInt>(I.getArgOperand(1))->getZExtValue();
05519     Ops[0] = getRoot();
05520     Ops[1] = getValue(I.getArgOperand(0));
05521     Ops[2] = getValue(I.getArgOperand(1));
05522     Ops[3] = getValue(I.getArgOperand(2));
05523     Ops[4] = getValue(I.getArgOperand(3));
05524     DAG.setRoot(DAG.getMemIntrinsicNode(ISD::PREFETCH, sdl,
05525                                         DAG.getVTList(MVT::Other), Ops,
05526                                         EVT::getIntegerVT(*Context, 8),
05527                                         MachinePointerInfo(I.getArgOperand(0)),
05528                                         0, /* align */
05529                                         false, /* volatile */
05530                                         rw==0, /* read */
05531                                         rw==1)); /* write */
05532     return nullptr;
05533   }
05534   case Intrinsic::lifetime_start:
05535   case Intrinsic::lifetime_end: {
05536     bool IsStart = (Intrinsic == Intrinsic::lifetime_start);
05537     // Stack coloring is not enabled in O0, discard region information.
05538     if (TM.getOptLevel() == CodeGenOpt::None)
05539       return nullptr;
05540 
05541     SmallVector<Value *, 4> Allocas;
05542     GetUnderlyingObjects(I.getArgOperand(1), Allocas, DL);
05543 
05544     for (SmallVectorImpl<Value*>::iterator Object = Allocas.begin(),
05545            E = Allocas.end(); Object != E; ++Object) {
05546       AllocaInst *LifetimeObject = dyn_cast_or_null<AllocaInst>(*Object);
05547 
05548       // Could not find an Alloca.
05549       if (!LifetimeObject)
05550         continue;
05551 
05552       // First check that the Alloca is static, otherwise it won't have a
05553       // valid frame index.
05554       auto SI = FuncInfo.StaticAllocaMap.find(LifetimeObject);
05555       if (SI == FuncInfo.StaticAllocaMap.end())
05556         return nullptr;
05557 
05558       int FI = SI->second;
05559 
05560       SDValue Ops[2];
05561       Ops[0] = getRoot();
05562       Ops[1] = DAG.getFrameIndex(FI, TLI.getPointerTy(), true);
05563       unsigned Opcode = (IsStart ? ISD::LIFETIME_START : ISD::LIFETIME_END);
05564 
05565       Res = DAG.getNode(Opcode, sdl, MVT::Other, Ops);
05566       DAG.setRoot(Res);
05567     }
05568     return nullptr;
05569   }
05570   case Intrinsic::invariant_start:
05571     // Discard region information.
05572     setValue(&I, DAG.getUNDEF(TLI.getPointerTy()));
05573     return nullptr;
05574   case Intrinsic::invariant_end:
05575     // Discard region information.
05576     return nullptr;
05577   case Intrinsic::stackprotectorcheck: {
05578     // Do not actually emit anything for this basic block. Instead we initialize
05579     // the stack protector descriptor and export the guard variable so we can
05580     // access it in FinishBasicBlock.
05581     const BasicBlock *BB = I.getParent();
05582     SPDescriptor.initialize(BB, FuncInfo.MBBMap[BB], I);
05583     ExportFromCurrentBlock(SPDescriptor.getGuard());
05584 
05585     // Flush our exports since we are going to process a terminator.
05586     (void)getControlRoot();
05587     return nullptr;
05588   }
05589   case Intrinsic::clear_cache:
05590     return TLI.getClearCacheBuiltinName();
05591   case Intrinsic::donothing:
05592     // ignore
05593     return nullptr;
05594   case Intrinsic::experimental_stackmap: {
05595     visitStackmap(I);
05596     return nullptr;
05597   }
05598   case Intrinsic::experimental_patchpoint_void:
05599   case Intrinsic::experimental_patchpoint_i64: {
05600     visitPatchpoint(&I);
05601     return nullptr;
05602   }
05603   case Intrinsic::experimental_gc_statepoint: {
05604     visitStatepoint(I);
05605     return nullptr;
05606   }
05607   case Intrinsic::experimental_gc_result_int:
05608   case Intrinsic::experimental_gc_result_float:
05609   case Intrinsic::experimental_gc_result_ptr:
05610   case Intrinsic::experimental_gc_result: {
05611     visitGCResult(I);
05612     return nullptr;
05613   }
05614   case Intrinsic::experimental_gc_relocate: {
05615     visitGCRelocate(I);
05616     return nullptr;
05617   }
05618   case Intrinsic::instrprof_increment:
05619     llvm_unreachable("instrprof failed to lower an increment");
05620 
05621   case Intrinsic::frameallocate: {
05622     MachineFunction &MF = DAG.getMachineFunction();
05623     const TargetInstrInfo *TII = DAG.getSubtarget().getInstrInfo();
05624 
05625     // Do the allocation and map it as a normal value.
05626     // FIXME: Maybe we should add this to the alloca map so that we don't have
05627     // to register allocate it?
05628     uint64_t Size = cast<ConstantInt>(I.getArgOperand(0))->getZExtValue();
05629     int Alloc = MF.getFrameInfo()->CreateFrameAllocation(Size);
05630     MVT PtrVT = TLI.getPointerTy(0);
05631     SDValue FIVal = DAG.getFrameIndex(Alloc, PtrVT);
05632     setValue(&I, FIVal);
05633 
05634     // Directly emit a FRAME_ALLOC machine instr. Label assignment emission is
05635     // the same on all targets.
05636     MCSymbol *FrameAllocSym =
05637         MF.getMMI().getContext().getOrCreateFrameAllocSymbol(MF.getName());
05638     BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, dl,
05639             TII->get(TargetOpcode::FRAME_ALLOC))
05640         .addSym(FrameAllocSym)
05641         .addFrameIndex(Alloc);
05642 
05643     return nullptr;
05644   }
05645 
05646   case Intrinsic::framerecover: {
05647     // i8* @llvm.framerecover(i8* %fn, i8* %fp)
05648     MachineFunction &MF = DAG.getMachineFunction();
05649     MVT PtrVT = TLI.getPointerTy(0);
05650 
05651     // Get the symbol that defines the frame offset.
05652     Function *Fn = cast<Function>(I.getArgOperand(0)->stripPointerCasts());
05653     MCSymbol *FrameAllocSym =
05654         MF.getMMI().getContext().getOrCreateFrameAllocSymbol(Fn->getName());
05655 
05656     // Create a TargetExternalSymbol for the label to avoid any target lowering
05657     // that would make this PC relative.
05658     StringRef Name = FrameAllocSym->getName();
05659     assert(Name.size() == strlen(Name.data()) && "not null terminated");
05660     SDValue OffsetSym = DAG.getTargetExternalSymbol(Name.data(), PtrVT);
05661     SDValue OffsetVal =
05662         DAG.getNode(ISD::FRAME_ALLOC_RECOVER, sdl, PtrVT, OffsetSym);
05663 
05664     // Add the offset to the FP.
05665     Value *FP = I.getArgOperand(1);
05666     SDValue FPVal = getValue(FP);
05667     SDValue Add = DAG.getNode(ISD::ADD, sdl, PtrVT, FPVal, OffsetVal);
05668     setValue(&I, Add);
05669 
05670     return nullptr;
05671   }
05672   case Intrinsic::eh_begincatch:
05673   case Intrinsic::eh_endcatch:
05674     llvm_unreachable("begin/end catch intrinsics not lowered in codegen");
05675   }
05676 }
05677 
05678 std::pair<SDValue, SDValue>
05679 SelectionDAGBuilder::lowerInvokable(TargetLowering::CallLoweringInfo &CLI,
05680                                     MachineBasicBlock *LandingPad) {
05681   MachineModuleInfo &MMI = DAG.getMachineFunction().getMMI();
05682   MCSymbol *BeginLabel = nullptr;
05683 
05684   if (LandingPad) {
05685     // Insert a label before the invoke call to mark the try range.  This can be
05686     // used to detect deletion of the invoke via the MachineModuleInfo.
05687     BeginLabel = MMI.getContext().CreateTempSymbol();
05688 
05689     // For SjLj, keep track of which landing pads go with which invokes
05690     // so as to maintain the ordering of pads in the LSDA.
05691     unsigned CallSiteIndex = MMI.getCurrentCallSite();
05692     if (CallSiteIndex) {
05693       MMI.setCallSiteBeginLabel(BeginLabel, CallSiteIndex);
05694       LPadToCallSiteMap[LandingPad].push_back(CallSiteIndex);
05695 
05696       // Now that the call site is handled, stop tracking it.
05697       MMI.setCurrentCallSite(0);
05698     }
05699 
05700     // Both PendingLoads and PendingExports must be flushed here;
05701     // this call might not return.
05702     (void)getRoot();
05703     DAG.setRoot(DAG.getEHLabel(getCurSDLoc(), getControlRoot(), BeginLabel));
05704 
05705     CLI.setChain(getRoot());
05706   }
05707   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
05708   std::pair<SDValue, SDValue> Result = TLI.LowerCallTo(CLI);
05709 
05710   assert((CLI.IsTailCall || Result.second.getNode()) &&
05711          "Non-null chain expected with non-tail call!");
05712   assert((Result.second.getNode() || !Result.first.getNode()) &&
05713          "Null value expected with tail call!");
05714 
05715   if (!Result.second.getNode()) {
05716     // As a special case, a null chain means that a tail call has been emitted
05717     // and the DAG root is already updated.
05718     HasTailCall = true;
05719 
05720     // Since there's no actual continuation from this block, nothing can be
05721     // relying on us setting vregs for them.
05722     PendingExports.clear();
05723   } else {
05724     DAG.setRoot(Result.second);
05725   }
05726 
05727   if (LandingPad) {
05728     // Insert a label at the end of the invoke call to mark the try range.  This
05729     // can be used to detect deletion of the invoke via the MachineModuleInfo.
05730     MCSymbol *EndLabel = MMI.getContext().CreateTempSymbol();
05731     DAG.setRoot(DAG.getEHLabel(getCurSDLoc(), getRoot(), EndLabel));
05732 
05733     // Inform MachineModuleInfo of range.
05734     MMI.addInvoke(LandingPad, BeginLabel, EndLabel);
05735   }
05736 
05737   return Result;
05738 }
05739 
05740 void SelectionDAGBuilder::LowerCallTo(ImmutableCallSite CS, SDValue Callee,
05741                                       bool isTailCall,
05742                                       MachineBasicBlock *LandingPad) {
05743   PointerType *PT = cast<PointerType>(CS.getCalledValue()->getType());
05744   FunctionType *FTy = cast<FunctionType>(PT->getElementType());
05745   Type *RetTy = FTy->getReturnType();
05746 
05747   TargetLowering::ArgListTy Args;
05748   TargetLowering::ArgListEntry Entry;
05749   Args.reserve(CS.arg_size());
05750 
05751   for (ImmutableCallSite::arg_iterator i = CS.arg_begin(), e = CS.arg_end();
05752        i != e; ++i) {
05753     const Value *V = *i;
05754 
05755     // Skip empty types
05756     if (V->getType()->isEmptyTy())
05757       continue;
05758 
05759     SDValue ArgNode = getValue(V);
05760     Entry.Node = ArgNode; Entry.Ty = V->getType();
05761 
05762     // Skip the first return-type Attribute to get to params.
05763     Entry.setAttributes(&CS, i - CS.arg_begin() + 1);
05764     Args.push_back(Entry);
05765   }
05766 
05767   // Check if target-independent constraints permit a tail call here.
05768   // Target-dependent constraints are checked within TLI->LowerCallTo.
05769   if (isTailCall && !isInTailCallPosition(CS, DAG.getTarget()))
05770     isTailCall = false;
05771 
05772   TargetLowering::CallLoweringInfo CLI(DAG);
05773   CLI.setDebugLoc(getCurSDLoc()).setChain(getRoot())
05774     .setCallee(RetTy, FTy, Callee, std::move(Args), CS)
05775     .setTailCall(isTailCall);
05776   std::pair<SDValue,SDValue> Result = lowerInvokable(CLI, LandingPad);
05777 
05778   if (Result.first.getNode())
05779     setValue(CS.getInstruction(), Result.first);
05780 }
05781 
05782 /// IsOnlyUsedInZeroEqualityComparison - Return true if it only matters that the
05783 /// value is equal or not-equal to zero.
05784 static bool IsOnlyUsedInZeroEqualityComparison(const Value *V) {
05785   for (const User *U : V->users()) {
05786     if (const ICmpInst *IC = dyn_cast<ICmpInst>(U))
05787       if (IC->isEquality())
05788         if (const Constant *C = dyn_cast<Constant>(IC->getOperand(1)))
05789           if (C->isNullValue())
05790             continue;
05791     // Unknown instruction.
05792     return false;
05793   }
05794   return true;
05795 }
05796 
05797 static SDValue getMemCmpLoad(const Value *PtrVal, MVT LoadVT,
05798                              Type *LoadTy,
05799                              SelectionDAGBuilder &Builder) {
05800 
05801   // Check to see if this load can be trivially constant folded, e.g. if the
05802   // input is from a string literal.
05803   if (const Constant *LoadInput = dyn_cast<Constant>(PtrVal)) {
05804     // Cast pointer to the type we really want to load.
05805     LoadInput = ConstantExpr::getBitCast(const_cast<Constant *>(LoadInput),
05806                                          PointerType::getUnqual(LoadTy));
05807 
05808     if (const Constant *LoadCst =
05809           ConstantFoldLoadFromConstPtr(const_cast<Constant *>(LoadInput),
05810                                        Builder.DL))
05811       return Builder.getValue(LoadCst);
05812   }
05813 
05814   // Otherwise, we have to emit the load.  If the pointer is to unfoldable but
05815   // still constant memory, the input chain can be the entry node.
05816   SDValue Root;
05817   bool ConstantMemory = false;
05818 
05819   // Do not serialize (non-volatile) loads of constant memory with anything.
05820   if (Builder.AA->pointsToConstantMemory(PtrVal)) {
05821     Root = Builder.DAG.getEntryNode();
05822     ConstantMemory = true;
05823   } else {
05824     // Do not serialize non-volatile loads against each other.
05825     Root = Builder.DAG.getRoot();
05826   }
05827 
05828   SDValue Ptr = Builder.getValue(PtrVal);
05829   SDValue LoadVal = Builder.DAG.getLoad(LoadVT, Builder.getCurSDLoc(), Root,
05830                                         Ptr, MachinePointerInfo(PtrVal),
05831                                         false /*volatile*/,
05832                                         false /*nontemporal*/,
05833                                         false /*isinvariant*/, 1 /* align=1 */);
05834 
05835   if (!ConstantMemory)
05836     Builder.PendingLoads.push_back(LoadVal.getValue(1));
05837   return LoadVal;
05838 }
05839 
05840 /// processIntegerCallValue - Record the value for an instruction that
05841 /// produces an integer result, converting the type where necessary.
05842 void SelectionDAGBuilder::processIntegerCallValue(const Instruction &I,
05843                                                   SDValue Value,
05844                                                   bool IsSigned) {
05845   EVT VT = DAG.getTargetLoweringInfo().getValueType(I.getType(), true);
05846   if (IsSigned)
05847     Value = DAG.getSExtOrTrunc(Value, getCurSDLoc(), VT);
05848   else
05849     Value = DAG.getZExtOrTrunc(Value, getCurSDLoc(), VT);
05850   setValue(&I, Value);
05851 }
05852 
05853 /// visitMemCmpCall - See if we can lower a call to memcmp in an optimized form.
05854 /// If so, return true and lower it, otherwise return false and it will be
05855 /// lowered like a normal call.
05856 bool SelectionDAGBuilder::visitMemCmpCall(const CallInst &I) {
05857   // Verify that the prototype makes sense.  int memcmp(void*,void*,size_t)
05858   if (I.getNumArgOperands() != 3)
05859     return false;
05860 
05861   const Value *LHS = I.getArgOperand(0), *RHS = I.getArgOperand(1);
05862   if (!LHS->getType()->isPointerTy() || !RHS->getType()->isPointerTy() ||
05863       !I.getArgOperand(2)->getType()->isIntegerTy() ||
05864       !I.getType()->isIntegerTy())
05865     return false;
05866 
05867   const Value *Size = I.getArgOperand(2);
05868   const ConstantInt *CSize = dyn_cast<ConstantInt>(Size);
05869   if (CSize && CSize->getZExtValue() == 0) {
05870     EVT CallVT = DAG.getTargetLoweringInfo().getValueType(I.getType(), true);
05871     setValue(&I, DAG.getConstant(0, CallVT));
05872     return true;
05873   }
05874 
05875   const TargetSelectionDAGInfo &TSI = DAG.getSelectionDAGInfo();
05876   std::pair<SDValue, SDValue> Res =
05877     TSI.EmitTargetCodeForMemcmp(DAG, getCurSDLoc(), DAG.getRoot(),
05878                                 getValue(LHS), getValue(RHS), getValue(Size),
05879                                 MachinePointerInfo(LHS),
05880                                 MachinePointerInfo(RHS));
05881   if (Res.first.getNode()) {
05882     processIntegerCallValue(I, Res.first, true);
05883     PendingLoads.push_back(Res.second);
05884     return true;
05885   }
05886 
05887   // memcmp(S1,S2,2) != 0 -> (*(short*)LHS != *(short*)RHS)  != 0
05888   // memcmp(S1,S2,4) != 0 -> (*(int*)LHS != *(int*)RHS)  != 0
05889   if (CSize && IsOnlyUsedInZeroEqualityComparison(&I)) {
05890     bool ActuallyDoIt = true;
05891     MVT LoadVT;
05892     Type *LoadTy;
05893     switch (CSize->getZExtValue()) {
05894     default:
05895       LoadVT = MVT::Other;
05896       LoadTy = nullptr;
05897       ActuallyDoIt = false;
05898       break;
05899     case 2:
05900       LoadVT = MVT::i16;
05901       LoadTy = Type::getInt16Ty(CSize->getContext());
05902       break;
05903     case 4:
05904       LoadVT = MVT::i32;
05905       LoadTy = Type::getInt32Ty(CSize->getContext());
05906       break;
05907     case 8:
05908       LoadVT = MVT::i64;
05909       LoadTy = Type::getInt64Ty(CSize->getContext());
05910       break;
05911         /*
05912     case 16:
05913       LoadVT = MVT::v4i32;
05914       LoadTy = Type::getInt32Ty(CSize->getContext());
05915       LoadTy = VectorType::get(LoadTy, 4);
05916       break;
05917          */
05918     }
05919 
05920     // This turns into unaligned loads.  We only do this if the target natively
05921     // supports the MVT we'll be loading or if it is small enough (<= 4) that
05922     // we'll only produce a small number of byte loads.
05923 
05924     // Require that we can find a legal MVT, and only do this if the target
05925     // supports unaligned loads of that type.  Expanding into byte loads would
05926     // bloat the code.
05927     const TargetLowering &TLI = DAG.getTargetLoweringInfo();
05928     if (ActuallyDoIt && CSize->getZExtValue() > 4) {
05929       unsigned DstAS = LHS->getType()->getPointerAddressSpace();
05930       unsigned SrcAS = RHS->getType()->getPointerAddressSpace();
05931       // TODO: Handle 5 byte compare as 4-byte + 1 byte.
05932       // TODO: Handle 8 byte compare on x86-32 as two 32-bit loads.
05933       // TODO: Check alignment of src and dest ptrs.
05934       if (!TLI.isTypeLegal(LoadVT) ||
05935           !TLI.allowsMisalignedMemoryAccesses(LoadVT, SrcAS) ||
05936           !TLI.allowsMisalignedMemoryAccesses(LoadVT, DstAS))
05937         ActuallyDoIt = false;
05938     }
05939 
05940     if (ActuallyDoIt) {
05941       SDValue LHSVal = getMemCmpLoad(LHS, LoadVT, LoadTy, *this);
05942       SDValue RHSVal = getMemCmpLoad(RHS, LoadVT, LoadTy, *this);
05943 
05944       SDValue Res = DAG.getSetCC(getCurSDLoc(), MVT::i1, LHSVal, RHSVal,
05945                                  ISD::SETNE);
05946       processIntegerCallValue(I, Res, false);
05947       return true;
05948     }
05949   }
05950 
05951 
05952   return false;
05953 }
05954 
05955 /// visitMemChrCall -- See if we can lower a memchr call into an optimized
05956 /// form.  If so, return true and lower it, otherwise return false and it
05957 /// will be lowered like a normal call.
05958 bool SelectionDAGBuilder::visitMemChrCall(const CallInst &I) {
05959   // Verify that the prototype makes sense.  void *memchr(void *, int, size_t)
05960   if (I.getNumArgOperands() != 3)
05961     return false;
05962 
05963   const Value *Src = I.getArgOperand(0);
05964   const Value *Char = I.getArgOperand(1);
05965   const Value *Length = I.getArgOperand(2);
05966   if (!Src->getType()->isPointerTy() ||
05967       !Char->getType()->isIntegerTy() ||
05968       !Length->getType()->isIntegerTy() ||
05969       !I.getType()->isPointerTy())
05970     return false;
05971 
05972   const TargetSelectionDAGInfo &TSI = DAG.getSelectionDAGInfo();
05973   std::pair<SDValue, SDValue> Res =
05974     TSI.EmitTargetCodeForMemchr(DAG, getCurSDLoc(), DAG.getRoot(),
05975                                 getValue(Src), getValue(Char), getValue(Length),
05976                                 MachinePointerInfo(Src));
05977   if (Res.first.getNode()) {
05978     setValue(&I, Res.first);
05979     PendingLoads.push_back(Res.second);
05980     return true;
05981   }
05982 
05983   return false;
05984 }
05985 
05986 /// visitStrCpyCall -- See if we can lower a strcpy or stpcpy call into an
05987 /// optimized form.  If so, return true and lower it, otherwise return false
05988 /// and it will be lowered like a normal call.
05989 bool SelectionDAGBuilder::visitStrCpyCall(const CallInst &I, bool isStpcpy) {
05990   // Verify that the prototype makes sense.  char *strcpy(char *, char *)
05991   if (I.getNumArgOperands() != 2)
05992     return false;
05993 
05994   const Value *Arg0 = I.getArgOperand(0), *Arg1 = I.getArgOperand(1);
05995   if (!Arg0->getType()->isPointerTy() ||
05996       !Arg1->getType()->isPointerTy() ||
05997       !I.getType()->isPointerTy())
05998     return false;
05999 
06000   const TargetSelectionDAGInfo &TSI = DAG.getSelectionDAGInfo();
06001   std::pair<SDValue, SDValue> Res =
06002     TSI.EmitTargetCodeForStrcpy(DAG, getCurSDLoc(), getRoot(),
06003                                 getValue(Arg0), getValue(Arg1),
06004                                 MachinePointerInfo(Arg0),
06005                                 MachinePointerInfo(Arg1), isStpcpy);
06006   if (Res.first.getNode()) {
06007     setValue(&I, Res.first);
06008     DAG.setRoot(Res.second);
06009     return true;
06010   }
06011 
06012   return false;
06013 }
06014 
06015 /// visitStrCmpCall - See if we can lower a call to strcmp in an optimized form.
06016 /// If so, return true and lower it, otherwise return false and it will be
06017 /// lowered like a normal call.
06018 bool SelectionDAGBuilder::visitStrCmpCall(const CallInst &I) {
06019   // Verify that the prototype makes sense.  int strcmp(void*,void*)
06020   if (I.getNumArgOperands() != 2)
06021     return false;
06022 
06023   const Value *Arg0 = I.getArgOperand(0), *Arg1 = I.getArgOperand(1);
06024   if (!Arg0->getType()->isPointerTy() ||
06025       !Arg1->getType()->isPointerTy() ||
06026       !I.getType()->isIntegerTy())
06027     return false;
06028 
06029   const TargetSelectionDAGInfo &TSI = DAG.getSelectionDAGInfo();
06030   std::pair<SDValue, SDValue> Res =
06031     TSI.EmitTargetCodeForStrcmp(DAG, getCurSDLoc(), DAG.getRoot(),
06032                                 getValue(Arg0), getValue(Arg1),
06033                                 MachinePointerInfo(Arg0),
06034                                 MachinePointerInfo(Arg1));
06035   if (Res.first.getNode()) {
06036     processIntegerCallValue(I, Res.first, true);
06037     PendingLoads.push_back(Res.second);
06038     return true;
06039   }
06040 
06041   return false;
06042 }
06043 
06044 /// visitStrLenCall -- See if we can lower a strlen call into an optimized
06045 /// form.  If so, return true and lower it, otherwise return false and it
06046 /// will be lowered like a normal call.
06047 bool SelectionDAGBuilder::visitStrLenCall(const CallInst &I) {
06048   // Verify that the prototype makes sense.  size_t strlen(char *)
06049   if (I.getNumArgOperands() != 1)
06050     return false;
06051 
06052   const Value *Arg0 = I.getArgOperand(0);
06053   if (!Arg0->getType()->isPointerTy() || !I.getType()->isIntegerTy())
06054     return false;
06055 
06056   const TargetSelectionDAGInfo &TSI = DAG.getSelectionDAGInfo();
06057   std::pair<SDValue, SDValue> Res =
06058     TSI.EmitTargetCodeForStrlen(DAG, getCurSDLoc(), DAG.getRoot(),
06059                                 getValue(Arg0), MachinePointerInfo(Arg0));
06060   if (Res.first.getNode()) {
06061     processIntegerCallValue(I, Res.first, false);
06062     PendingLoads.push_back(Res.second);
06063     return true;
06064   }
06065 
06066   return false;
06067 }
06068 
06069 /// visitStrNLenCall -- See if we can lower a strnlen call into an optimized
06070 /// form.  If so, return true and lower it, otherwise return false and it
06071 /// will be lowered like a normal call.
06072 bool SelectionDAGBuilder::visitStrNLenCall(const CallInst &I) {
06073   // Verify that the prototype makes sense.  size_t strnlen(char *, size_t)
06074   if (I.getNumArgOperands() != 2)
06075     return false;
06076 
06077   const Value *Arg0 = I.getArgOperand(0), *Arg1 = I.getArgOperand(1);
06078   if (!Arg0->getType()->isPointerTy() ||
06079       !Arg1->getType()->isIntegerTy() ||
06080       !I.getType()->isIntegerTy())
06081     return false;
06082 
06083   const TargetSelectionDAGInfo &TSI = DAG.getSelectionDAGInfo();
06084   std::pair<SDValue, SDValue> Res =
06085     TSI.EmitTargetCodeForStrnlen(DAG, getCurSDLoc(), DAG.getRoot(),
06086                                  getValue(Arg0), getValue(Arg1),
06087                                  MachinePointerInfo(Arg0));
06088   if (Res.first.getNode()) {
06089     processIntegerCallValue(I, Res.first, false);
06090     PendingLoads.push_back(Res.second);
06091     return true;
06092   }
06093 
06094   return false;
06095 }
06096 
06097 /// visitUnaryFloatCall - If a call instruction is a unary floating-point
06098 /// operation (as expected), translate it to an SDNode with the specified opcode
06099 /// and return true.
06100 bool SelectionDAGBuilder::visitUnaryFloatCall(const CallInst &I,
06101                                               unsigned Opcode) {
06102   // Sanity check that it really is a unary floating-point call.
06103   if (I.getNumArgOperands() != 1 ||
06104       !I.getArgOperand(0)->getType()->isFloatingPointTy() ||
06105       I.getType() != I.getArgOperand(0)->getType() ||
06106       !I.onlyReadsMemory())
06107     return false;
06108 
06109   SDValue Tmp = getValue(I.getArgOperand(0));
06110   setValue(&I, DAG.getNode(Opcode, getCurSDLoc(), Tmp.getValueType(), Tmp));
06111   return true;
06112 }
06113 
06114 /// visitBinaryFloatCall - If a call instruction is a binary floating-point
06115 /// operation (as expected), translate it to an SDNode with the specified opcode
06116 /// and return true.
06117 bool SelectionDAGBuilder::visitBinaryFloatCall(const CallInst &I,
06118                                                unsigned Opcode) {
06119   // Sanity check that it really is a binary floating-point call.
06120   if (I.getNumArgOperands() != 2 ||
06121       !I.getArgOperand(0)->getType()->isFloatingPointTy() ||
06122       I.getType() != I.getArgOperand(0)->getType() ||
06123       I.getType() != I.getArgOperand(1)->getType() ||
06124       !I.onlyReadsMemory())
06125     return false;
06126 
06127   SDValue Tmp0 = getValue(I.getArgOperand(0));
06128   SDValue Tmp1 = getValue(I.getArgOperand(1));
06129   EVT VT = Tmp0.getValueType();
06130   setValue(&I, DAG.getNode(Opcode, getCurSDLoc(), VT, Tmp0, Tmp1));
06131   return true;
06132 }
06133 
06134 void SelectionDAGBuilder::visitCall(const CallInst &I) {
06135   // Handle inline assembly differently.
06136   if (isa<InlineAsm>(I.getCalledValue())) {
06137     visitInlineAsm(&I);
06138     return;
06139   }
06140 
06141   MachineModuleInfo &MMI = DAG.