LLVM  mainline
InstCombineCalls.cpp
Go to the documentation of this file.
00001 //===- InstCombineCalls.cpp -----------------------------------------------===//
00002 //
00003 //                     The LLVM Compiler Infrastructure
00004 //
00005 // This file is distributed under the University of Illinois Open Source
00006 // License. See LICENSE.TXT for details.
00007 //
00008 //===----------------------------------------------------------------------===//
00009 //
00010 // This file implements the visitCall and visitInvoke functions.
00011 //
00012 //===----------------------------------------------------------------------===//
00013 
00014 #include "InstCombineInternal.h"
00015 #include "llvm/ADT/Statistic.h"
00016 #include "llvm/Analysis/InstructionSimplify.h"
00017 #include "llvm/Analysis/MemoryBuiltins.h"
00018 #include "llvm/IR/CallSite.h"
00019 #include "llvm/IR/Dominators.h"
00020 #include "llvm/IR/PatternMatch.h"
00021 #include "llvm/IR/Statepoint.h"
00022 #include "llvm/Transforms/Utils/BuildLibCalls.h"
00023 #include "llvm/Transforms/Utils/Local.h"
00024 #include "llvm/Transforms/Utils/SimplifyLibCalls.h"
00025 using namespace llvm;
00026 using namespace PatternMatch;
00027 
00028 #define DEBUG_TYPE "instcombine"
00029 
00030 STATISTIC(NumSimplified, "Number of library calls simplified");
00031 
00032 /// Return the specified type promoted as it would be to pass though a va_arg
00033 /// area.
00034 static Type *getPromotedType(Type *Ty) {
00035   if (IntegerType* ITy = dyn_cast<IntegerType>(Ty)) {
00036     if (ITy->getBitWidth() < 32)
00037       return Type::getInt32Ty(Ty->getContext());
00038   }
00039   return Ty;
00040 }
00041 
00042 /// Given an aggregate type which ultimately holds a single scalar element,
00043 /// like {{{type}}} or [1 x type], return type.
00044 static Type *reduceToSingleValueType(Type *T) {
00045   while (!T->isSingleValueType()) {
00046     if (StructType *STy = dyn_cast<StructType>(T)) {
00047       if (STy->getNumElements() == 1)
00048         T = STy->getElementType(0);
00049       else
00050         break;
00051     } else if (ArrayType *ATy = dyn_cast<ArrayType>(T)) {
00052       if (ATy->getNumElements() == 1)
00053         T = ATy->getElementType();
00054       else
00055         break;
00056     } else
00057       break;
00058   }
00059 
00060   return T;
00061 }
00062 
00063 Instruction *InstCombiner::SimplifyMemTransfer(MemIntrinsic *MI) {
00064   unsigned DstAlign = getKnownAlignment(MI->getArgOperand(0), DL, MI, AC, DT);
00065   unsigned SrcAlign = getKnownAlignment(MI->getArgOperand(1), DL, MI, AC, DT);
00066   unsigned MinAlign = std::min(DstAlign, SrcAlign);
00067   unsigned CopyAlign = MI->getAlignment();
00068 
00069   if (CopyAlign < MinAlign) {
00070     MI->setAlignment(ConstantInt::get(MI->getAlignmentType(), MinAlign, false));
00071     return MI;
00072   }
00073 
00074   // If MemCpyInst length is 1/2/4/8 bytes then replace memcpy with
00075   // load/store.
00076   ConstantInt *MemOpLength = dyn_cast<ConstantInt>(MI->getArgOperand(2));
00077   if (!MemOpLength) return nullptr;
00078 
00079   // Source and destination pointer types are always "i8*" for intrinsic.  See
00080   // if the size is something we can handle with a single primitive load/store.
00081   // A single load+store correctly handles overlapping memory in the memmove
00082   // case.
00083   uint64_t Size = MemOpLength->getLimitedValue();
00084   assert(Size && "0-sized memory transferring should be removed already.");
00085 
00086   if (Size > 8 || (Size&(Size-1)))
00087     return nullptr;  // If not 1/2/4/8 bytes, exit.
00088 
00089   // Use an integer load+store unless we can find something better.
00090   unsigned SrcAddrSp =
00091     cast<PointerType>(MI->getArgOperand(1)->getType())->getAddressSpace();
00092   unsigned DstAddrSp =
00093     cast<PointerType>(MI->getArgOperand(0)->getType())->getAddressSpace();
00094 
00095   IntegerType* IntType = IntegerType::get(MI->getContext(), Size<<3);
00096   Type *NewSrcPtrTy = PointerType::get(IntType, SrcAddrSp);
00097   Type *NewDstPtrTy = PointerType::get(IntType, DstAddrSp);
00098 
00099   // Memcpy forces the use of i8* for the source and destination.  That means
00100   // that if you're using memcpy to move one double around, you'll get a cast
00101   // from double* to i8*.  We'd much rather use a double load+store rather than
00102   // an i64 load+store, here because this improves the odds that the source or
00103   // dest address will be promotable.  See if we can find a better type than the
00104   // integer datatype.
00105   Value *StrippedDest = MI->getArgOperand(0)->stripPointerCasts();
00106   MDNode *CopyMD = nullptr;
00107   if (StrippedDest != MI->getArgOperand(0)) {
00108     Type *SrcETy = cast<PointerType>(StrippedDest->getType())
00109                                     ->getElementType();
00110     if (SrcETy->isSized() && DL.getTypeStoreSize(SrcETy) == Size) {
00111       // The SrcETy might be something like {{{double}}} or [1 x double].  Rip
00112       // down through these levels if so.
00113       SrcETy = reduceToSingleValueType(SrcETy);
00114 
00115       if (SrcETy->isSingleValueType()) {
00116         NewSrcPtrTy = PointerType::get(SrcETy, SrcAddrSp);
00117         NewDstPtrTy = PointerType::get(SrcETy, DstAddrSp);
00118 
00119         // If the memcpy has metadata describing the members, see if we can
00120         // get the TBAA tag describing our copy.
00121         if (MDNode *M = MI->getMetadata(LLVMContext::MD_tbaa_struct)) {
00122           if (M->getNumOperands() == 3 && M->getOperand(0) &&
00123               mdconst::hasa<ConstantInt>(M->getOperand(0)) &&
00124               mdconst::extract<ConstantInt>(M->getOperand(0))->isNullValue() &&
00125               M->getOperand(1) &&
00126               mdconst::hasa<ConstantInt>(M->getOperand(1)) &&
00127               mdconst::extract<ConstantInt>(M->getOperand(1))->getValue() ==
00128                   Size &&
00129               M->getOperand(2) && isa<MDNode>(M->getOperand(2)))
00130             CopyMD = cast<MDNode>(M->getOperand(2));
00131         }
00132       }
00133     }
00134   }
00135 
00136   // If the memcpy/memmove provides better alignment info than we can
00137   // infer, use it.
00138   SrcAlign = std::max(SrcAlign, CopyAlign);
00139   DstAlign = std::max(DstAlign, CopyAlign);
00140 
00141   Value *Src = Builder->CreateBitCast(MI->getArgOperand(1), NewSrcPtrTy);
00142   Value *Dest = Builder->CreateBitCast(MI->getArgOperand(0), NewDstPtrTy);
00143   LoadInst *L = Builder->CreateLoad(Src, MI->isVolatile());
00144   L->setAlignment(SrcAlign);
00145   if (CopyMD)
00146     L->setMetadata(LLVMContext::MD_tbaa, CopyMD);
00147   StoreInst *S = Builder->CreateStore(L, Dest, MI->isVolatile());
00148   S->setAlignment(DstAlign);
00149   if (CopyMD)
00150     S->setMetadata(LLVMContext::MD_tbaa, CopyMD);
00151 
00152   // Set the size of the copy to 0, it will be deleted on the next iteration.
00153   MI->setArgOperand(2, Constant::getNullValue(MemOpLength->getType()));
00154   return MI;
00155 }
00156 
00157 Instruction *InstCombiner::SimplifyMemSet(MemSetInst *MI) {
00158   unsigned Alignment = getKnownAlignment(MI->getDest(), DL, MI, AC, DT);
00159   if (MI->getAlignment() < Alignment) {
00160     MI->setAlignment(ConstantInt::get(MI->getAlignmentType(),
00161                                              Alignment, false));
00162     return MI;
00163   }
00164 
00165   // Extract the length and alignment and fill if they are constant.
00166   ConstantInt *LenC = dyn_cast<ConstantInt>(MI->getLength());
00167   ConstantInt *FillC = dyn_cast<ConstantInt>(MI->getValue());
00168   if (!LenC || !FillC || !FillC->getType()->isIntegerTy(8))
00169     return nullptr;
00170   uint64_t Len = LenC->getLimitedValue();
00171   Alignment = MI->getAlignment();
00172   assert(Len && "0-sized memory setting should be removed already.");
00173 
00174   // memset(s,c,n) -> store s, c (for n=1,2,4,8)
00175   if (Len <= 8 && isPowerOf2_32((uint32_t)Len)) {
00176     Type *ITy = IntegerType::get(MI->getContext(), Len*8);  // n=1 -> i8.
00177 
00178     Value *Dest = MI->getDest();
00179     unsigned DstAddrSp = cast<PointerType>(Dest->getType())->getAddressSpace();
00180     Type *NewDstPtrTy = PointerType::get(ITy, DstAddrSp);
00181     Dest = Builder->CreateBitCast(Dest, NewDstPtrTy);
00182 
00183     // Alignment 0 is identity for alignment 1 for memset, but not store.
00184     if (Alignment == 0) Alignment = 1;
00185 
00186     // Extract the fill value and store.
00187     uint64_t Fill = FillC->getZExtValue()*0x0101010101010101ULL;
00188     StoreInst *S = Builder->CreateStore(ConstantInt::get(ITy, Fill), Dest,
00189                                         MI->isVolatile());
00190     S->setAlignment(Alignment);
00191 
00192     // Set the size of the copy to 0, it will be deleted on the next iteration.
00193     MI->setLength(Constant::getNullValue(LenC->getType()));
00194     return MI;
00195   }
00196 
00197   return nullptr;
00198 }
00199 
00200 static Value *SimplifyX86immshift(const IntrinsicInst &II,
00201                                   InstCombiner::BuilderTy &Builder) {
00202   bool LogicalShift = false;
00203   bool ShiftLeft = false;
00204 
00205   switch (II.getIntrinsicID()) {
00206   default:
00207     return nullptr;
00208   case Intrinsic::x86_sse2_psra_d:
00209   case Intrinsic::x86_sse2_psra_w:
00210   case Intrinsic::x86_sse2_psrai_d:
00211   case Intrinsic::x86_sse2_psrai_w:
00212   case Intrinsic::x86_avx2_psra_d:
00213   case Intrinsic::x86_avx2_psra_w:
00214   case Intrinsic::x86_avx2_psrai_d:
00215   case Intrinsic::x86_avx2_psrai_w:
00216     LogicalShift = false; ShiftLeft = false;
00217     break;
00218   case Intrinsic::x86_sse2_psrl_d:
00219   case Intrinsic::x86_sse2_psrl_q:
00220   case Intrinsic::x86_sse2_psrl_w:
00221   case Intrinsic::x86_sse2_psrli_d:
00222   case Intrinsic::x86_sse2_psrli_q:
00223   case Intrinsic::x86_sse2_psrli_w:
00224   case Intrinsic::x86_avx2_psrl_d:
00225   case Intrinsic::x86_avx2_psrl_q:
00226   case Intrinsic::x86_avx2_psrl_w:
00227   case Intrinsic::x86_avx2_psrli_d:
00228   case Intrinsic::x86_avx2_psrli_q:
00229   case Intrinsic::x86_avx2_psrli_w:
00230     LogicalShift = true; ShiftLeft = false;
00231     break;
00232   case Intrinsic::x86_sse2_psll_d:
00233   case Intrinsic::x86_sse2_psll_q:
00234   case Intrinsic::x86_sse2_psll_w:
00235   case Intrinsic::x86_sse2_pslli_d:
00236   case Intrinsic::x86_sse2_pslli_q:
00237   case Intrinsic::x86_sse2_pslli_w:
00238   case Intrinsic::x86_avx2_psll_d:
00239   case Intrinsic::x86_avx2_psll_q:
00240   case Intrinsic::x86_avx2_psll_w:
00241   case Intrinsic::x86_avx2_pslli_d:
00242   case Intrinsic::x86_avx2_pslli_q:
00243   case Intrinsic::x86_avx2_pslli_w:
00244     LogicalShift = true; ShiftLeft = true;
00245     break;
00246   }
00247   assert((LogicalShift || !ShiftLeft) && "Only logical shifts can shift left");
00248 
00249   // Simplify if count is constant.
00250   auto Arg1 = II.getArgOperand(1);
00251   auto CAZ = dyn_cast<ConstantAggregateZero>(Arg1);
00252   auto CDV = dyn_cast<ConstantDataVector>(Arg1);
00253   auto CInt = dyn_cast<ConstantInt>(Arg1);
00254   if (!CAZ && !CDV && !CInt)
00255     return nullptr;
00256 
00257   APInt Count(64, 0);
00258   if (CDV) {
00259     // SSE2/AVX2 uses all the first 64-bits of the 128-bit vector
00260     // operand to compute the shift amount.
00261     auto VT = cast<VectorType>(CDV->getType());
00262     unsigned BitWidth = VT->getElementType()->getPrimitiveSizeInBits();
00263     assert((64 % BitWidth) == 0 && "Unexpected packed shift size");
00264     unsigned NumSubElts = 64 / BitWidth;
00265 
00266     // Concatenate the sub-elements to create the 64-bit value.
00267     for (unsigned i = 0; i != NumSubElts; ++i) {
00268       unsigned SubEltIdx = (NumSubElts - 1) - i;
00269       auto SubElt = cast<ConstantInt>(CDV->getElementAsConstant(SubEltIdx));
00270       Count = Count.shl(BitWidth);
00271       Count |= SubElt->getValue().zextOrTrunc(64);
00272     }
00273   }
00274   else if (CInt)
00275     Count = CInt->getValue();
00276 
00277   auto Vec = II.getArgOperand(0);
00278   auto VT = cast<VectorType>(Vec->getType());
00279   auto SVT = VT->getElementType();
00280   unsigned VWidth = VT->getNumElements();
00281   unsigned BitWidth = SVT->getPrimitiveSizeInBits();
00282 
00283   // If shift-by-zero then just return the original value.
00284   if (Count == 0)
00285     return Vec;
00286 
00287   // Handle cases when Shift >= BitWidth.
00288   if (Count.uge(BitWidth)) {
00289     // If LogicalShift - just return zero.
00290     if (LogicalShift)
00291       return ConstantAggregateZero::get(VT);
00292 
00293     // If ArithmeticShift - clamp Shift to (BitWidth - 1).
00294     Count = APInt(64, BitWidth - 1);
00295   }
00296 
00297   // Get a constant vector of the same type as the first operand.
00298   auto ShiftAmt = ConstantInt::get(SVT, Count.zextOrTrunc(BitWidth));
00299   auto ShiftVec = Builder.CreateVectorSplat(VWidth, ShiftAmt);
00300 
00301   if (ShiftLeft)
00302     return Builder.CreateShl(Vec, ShiftVec);
00303 
00304   if (LogicalShift)
00305     return Builder.CreateLShr(Vec, ShiftVec);
00306 
00307   return Builder.CreateAShr(Vec, ShiftVec);
00308 }
00309 
00310 static Value *SimplifyX86extend(const IntrinsicInst &II,
00311                                 InstCombiner::BuilderTy &Builder,
00312                                 bool SignExtend) {
00313   VectorType *SrcTy = cast<VectorType>(II.getArgOperand(0)->getType());
00314   VectorType *DstTy = cast<VectorType>(II.getType());
00315   unsigned NumDstElts = DstTy->getNumElements();
00316 
00317   // Extract a subvector of the first NumDstElts lanes and sign/zero extend.
00318   SmallVector<int, 8> ShuffleMask;
00319   for (int i = 0; i != (int)NumDstElts; ++i)
00320     ShuffleMask.push_back(i);
00321 
00322   Value *SV = Builder.CreateShuffleVector(II.getArgOperand(0),
00323                                           UndefValue::get(SrcTy), ShuffleMask);
00324   return SignExtend ? Builder.CreateSExt(SV, DstTy)
00325                     : Builder.CreateZExt(SV, DstTy);
00326 }
00327 
00328 static Value *SimplifyX86insertps(const IntrinsicInst &II,
00329                                   InstCombiner::BuilderTy &Builder) {
00330   if (auto *CInt = dyn_cast<ConstantInt>(II.getArgOperand(2))) {
00331     VectorType *VecTy = cast<VectorType>(II.getType());
00332     assert(VecTy->getNumElements() == 4 && "insertps with wrong vector type");
00333 
00334     // The immediate permute control byte looks like this:
00335     //    [3:0] - zero mask for each 32-bit lane
00336     //    [5:4] - select one 32-bit destination lane
00337     //    [7:6] - select one 32-bit source lane
00338 
00339     uint8_t Imm = CInt->getZExtValue();
00340     uint8_t ZMask = Imm & 0xf;
00341     uint8_t DestLane = (Imm >> 4) & 0x3;
00342     uint8_t SourceLane = (Imm >> 6) & 0x3;
00343 
00344     ConstantAggregateZero *ZeroVector = ConstantAggregateZero::get(VecTy);
00345 
00346     // If all zero mask bits are set, this was just a weird way to
00347     // generate a zero vector.
00348     if (ZMask == 0xf)
00349       return ZeroVector;
00350 
00351     // Initialize by passing all of the first source bits through.
00352     int ShuffleMask[4] = { 0, 1, 2, 3 };
00353 
00354     // We may replace the second operand with the zero vector.
00355     Value *V1 = II.getArgOperand(1);
00356 
00357     if (ZMask) {
00358       // If the zero mask is being used with a single input or the zero mask
00359       // overrides the destination lane, this is a shuffle with the zero vector.
00360       if ((II.getArgOperand(0) == II.getArgOperand(1)) ||
00361           (ZMask & (1 << DestLane))) {
00362         V1 = ZeroVector;
00363         // We may still move 32-bits of the first source vector from one lane
00364         // to another.
00365         ShuffleMask[DestLane] = SourceLane;
00366         // The zero mask may override the previous insert operation.
00367         for (unsigned i = 0; i < 4; ++i)
00368           if ((ZMask >> i) & 0x1)
00369             ShuffleMask[i] = i + 4;
00370       } else {
00371         // TODO: Model this case as 2 shuffles or a 'logical and' plus shuffle?
00372         return nullptr;
00373       }
00374     } else {
00375       // Replace the selected destination lane with the selected source lane.
00376       ShuffleMask[DestLane] = SourceLane + 4;
00377     }
00378 
00379     return Builder.CreateShuffleVector(II.getArgOperand(0), V1, ShuffleMask);
00380   }
00381   return nullptr;
00382 }
00383 
00384 /// Attempt to simplify SSE4A EXTRQ/EXTRQI instructions using constant folding
00385 /// or conversion to a shuffle vector.
00386 static Value *SimplifyX86extrq(IntrinsicInst &II, Value *Op0,
00387                                ConstantInt *CILength, ConstantInt *CIIndex,
00388                                InstCombiner::BuilderTy &Builder) {
00389   auto LowConstantHighUndef = [&](uint64_t Val) {
00390     Type *IntTy64 = Type::getInt64Ty(II.getContext());
00391     Constant *Args[] = {ConstantInt::get(IntTy64, Val),
00392                         UndefValue::get(IntTy64)};
00393     return ConstantVector::get(Args);
00394   };
00395 
00396   // See if we're dealing with constant values.
00397   Constant *C0 = dyn_cast<Constant>(Op0);
00398   ConstantInt *CI0 =
00399       C0 ? dyn_cast<ConstantInt>(C0->getAggregateElement((unsigned)0))
00400          : nullptr;
00401 
00402   // Attempt to constant fold.
00403   if (CILength && CIIndex) {
00404     // From AMD documentation: "The bit index and field length are each six
00405     // bits in length other bits of the field are ignored."
00406     APInt APIndex = CIIndex->getValue().zextOrTrunc(6);
00407     APInt APLength = CILength->getValue().zextOrTrunc(6);
00408 
00409     unsigned Index = APIndex.getZExtValue();
00410 
00411     // From AMD documentation: "a value of zero in the field length is
00412     // defined as length of 64".
00413     unsigned Length = APLength == 0 ? 64 : APLength.getZExtValue();
00414 
00415     // From AMD documentation: "If the sum of the bit index + length field
00416     // is greater than 64, the results are undefined".
00417     unsigned End = Index + Length;
00418 
00419     // Note that both field index and field length are 8-bit quantities.
00420     // Since variables 'Index' and 'Length' are unsigned values
00421     // obtained from zero-extending field index and field length
00422     // respectively, their sum should never wrap around.
00423     if (End > 64)
00424       return UndefValue::get(II.getType());
00425 
00426     // If we are inserting whole bytes, we can convert this to a shuffle.
00427     // Lowering can recognize EXTRQI shuffle masks.
00428     if ((Length % 8) == 0 && (Index % 8) == 0) {
00429       // Convert bit indices to byte indices.
00430       Length /= 8;
00431       Index /= 8;
00432 
00433       Type *IntTy8 = Type::getInt8Ty(II.getContext());
00434       Type *IntTy32 = Type::getInt32Ty(II.getContext());
00435       VectorType *ShufTy = VectorType::get(IntTy8, 16);
00436 
00437       SmallVector<Constant *, 16> ShuffleMask;
00438       for (int i = 0; i != (int)Length; ++i)
00439         ShuffleMask.push_back(
00440             Constant::getIntegerValue(IntTy32, APInt(32, i + Index)));
00441       for (int i = Length; i != 8; ++i)
00442         ShuffleMask.push_back(
00443             Constant::getIntegerValue(IntTy32, APInt(32, i + 16)));
00444       for (int i = 8; i != 16; ++i)
00445         ShuffleMask.push_back(UndefValue::get(IntTy32));
00446 
00447       Value *SV = Builder.CreateShuffleVector(
00448           Builder.CreateBitCast(Op0, ShufTy),
00449           ConstantAggregateZero::get(ShufTy), ConstantVector::get(ShuffleMask));
00450       return Builder.CreateBitCast(SV, II.getType());
00451     }
00452 
00453     // Constant Fold - shift Index'th bit to lowest position and mask off
00454     // Length bits.
00455     if (CI0) {
00456       APInt Elt = CI0->getValue();
00457       Elt = Elt.lshr(Index).zextOrTrunc(Length);
00458       return LowConstantHighUndef(Elt.getZExtValue());
00459     }
00460 
00461     // If we were an EXTRQ call, we'll save registers if we convert to EXTRQI.
00462     if (II.getIntrinsicID() == Intrinsic::x86_sse4a_extrq) {
00463       Value *Args[] = {Op0, CILength, CIIndex};
00464       Module *M = II.getModule();
00465       Value *F = Intrinsic::getDeclaration(M, Intrinsic::x86_sse4a_extrqi);
00466       return Builder.CreateCall(F, Args);
00467     }
00468   }
00469 
00470   // Constant Fold - extraction from zero is always {zero, undef}.
00471   if (CI0 && CI0->equalsInt(0))
00472     return LowConstantHighUndef(0);
00473 
00474   return nullptr;
00475 }
00476 
00477 /// Attempt to simplify SSE4A INSERTQ/INSERTQI instructions using constant
00478 /// folding or conversion to a shuffle vector.
00479 static Value *SimplifyX86insertq(IntrinsicInst &II, Value *Op0, Value *Op1,
00480                                  APInt APLength, APInt APIndex,
00481                                  InstCombiner::BuilderTy &Builder) {
00482 
00483   // From AMD documentation: "The bit index and field length are each six bits
00484   // in length other bits of the field are ignored."
00485   APIndex = APIndex.zextOrTrunc(6);
00486   APLength = APLength.zextOrTrunc(6);
00487 
00488   // Attempt to constant fold.
00489   unsigned Index = APIndex.getZExtValue();
00490 
00491   // From AMD documentation: "a value of zero in the field length is
00492   // defined as length of 64".
00493   unsigned Length = APLength == 0 ? 64 : APLength.getZExtValue();
00494 
00495   // From AMD documentation: "If the sum of the bit index + length field
00496   // is greater than 64, the results are undefined".
00497   unsigned End = Index + Length;
00498 
00499   // Note that both field index and field length are 8-bit quantities.
00500   // Since variables 'Index' and 'Length' are unsigned values
00501   // obtained from zero-extending field index and field length
00502   // respectively, their sum should never wrap around.
00503   if (End > 64)
00504     return UndefValue::get(II.getType());
00505 
00506   // If we are inserting whole bytes, we can convert this to a shuffle.
00507   // Lowering can recognize INSERTQI shuffle masks.
00508   if ((Length % 8) == 0 && (Index % 8) == 0) {
00509     // Convert bit indices to byte indices.
00510     Length /= 8;
00511     Index /= 8;
00512 
00513     Type *IntTy8 = Type::getInt8Ty(II.getContext());
00514     Type *IntTy32 = Type::getInt32Ty(II.getContext());
00515     VectorType *ShufTy = VectorType::get(IntTy8, 16);
00516 
00517     SmallVector<Constant *, 16> ShuffleMask;
00518     for (int i = 0; i != (int)Index; ++i)
00519       ShuffleMask.push_back(Constant::getIntegerValue(IntTy32, APInt(32, i)));
00520     for (int i = 0; i != (int)Length; ++i)
00521       ShuffleMask.push_back(
00522           Constant::getIntegerValue(IntTy32, APInt(32, i + 16)));
00523     for (int i = Index + Length; i != 8; ++i)
00524       ShuffleMask.push_back(Constant::getIntegerValue(IntTy32, APInt(32, i)));
00525     for (int i = 8; i != 16; ++i)
00526       ShuffleMask.push_back(UndefValue::get(IntTy32));
00527 
00528     Value *SV = Builder.CreateShuffleVector(Builder.CreateBitCast(Op0, ShufTy),
00529                                             Builder.CreateBitCast(Op1, ShufTy),
00530                                             ConstantVector::get(ShuffleMask));
00531     return Builder.CreateBitCast(SV, II.getType());
00532   }
00533 
00534   // See if we're dealing with constant values.
00535   Constant *C0 = dyn_cast<Constant>(Op0);
00536   Constant *C1 = dyn_cast<Constant>(Op1);
00537   ConstantInt *CI00 =
00538       C0 ? dyn_cast<ConstantInt>(C0->getAggregateElement((unsigned)0))
00539          : nullptr;
00540   ConstantInt *CI10 =
00541       C1 ? dyn_cast<ConstantInt>(C1->getAggregateElement((unsigned)0))
00542          : nullptr;
00543 
00544   // Constant Fold - insert bottom Length bits starting at the Index'th bit.
00545   if (CI00 && CI10) {
00546     APInt V00 = CI00->getValue();
00547     APInt V10 = CI10->getValue();
00548     APInt Mask = APInt::getLowBitsSet(64, Length).shl(Index);
00549     V00 = V00 & ~Mask;
00550     V10 = V10.zextOrTrunc(Length).zextOrTrunc(64).shl(Index);
00551     APInt Val = V00 | V10;
00552     Type *IntTy64 = Type::getInt64Ty(II.getContext());
00553     Constant *Args[] = {ConstantInt::get(IntTy64, Val.getZExtValue()),
00554                         UndefValue::get(IntTy64)};
00555     return ConstantVector::get(Args);
00556   }
00557 
00558   // If we were an INSERTQ call, we'll save demanded elements if we convert to
00559   // INSERTQI.
00560   if (II.getIntrinsicID() == Intrinsic::x86_sse4a_insertq) {
00561     Type *IntTy8 = Type::getInt8Ty(II.getContext());
00562     Constant *CILength = ConstantInt::get(IntTy8, Length, false);
00563     Constant *CIIndex = ConstantInt::get(IntTy8, Index, false);
00564 
00565     Value *Args[] = {Op0, Op1, CILength, CIIndex};
00566     Module *M = II.getModule();
00567     Value *F = Intrinsic::getDeclaration(M, Intrinsic::x86_sse4a_insertqi);
00568     return Builder.CreateCall(F, Args);
00569   }
00570 
00571   return nullptr;
00572 }
00573 
00574 /// The shuffle mask for a perm2*128 selects any two halves of two 256-bit
00575 /// source vectors, unless a zero bit is set. If a zero bit is set,
00576 /// then ignore that half of the mask and clear that half of the vector.
00577 static Value *SimplifyX86vperm2(const IntrinsicInst &II,
00578                                 InstCombiner::BuilderTy &Builder) {
00579   if (auto *CInt = dyn_cast<ConstantInt>(II.getArgOperand(2))) {
00580     VectorType *VecTy = cast<VectorType>(II.getType());
00581     ConstantAggregateZero *ZeroVector = ConstantAggregateZero::get(VecTy);
00582 
00583     // The immediate permute control byte looks like this:
00584     //    [1:0] - select 128 bits from sources for low half of destination
00585     //    [2]   - ignore
00586     //    [3]   - zero low half of destination
00587     //    [5:4] - select 128 bits from sources for high half of destination
00588     //    [6]   - ignore
00589     //    [7]   - zero high half of destination
00590 
00591     uint8_t Imm = CInt->getZExtValue();
00592 
00593     bool LowHalfZero = Imm & 0x08;
00594     bool HighHalfZero = Imm & 0x80;
00595 
00596     // If both zero mask bits are set, this was just a weird way to
00597     // generate a zero vector.
00598     if (LowHalfZero && HighHalfZero)
00599       return ZeroVector;
00600 
00601     // If 0 or 1 zero mask bits are set, this is a simple shuffle.
00602     unsigned NumElts = VecTy->getNumElements();
00603     unsigned HalfSize = NumElts / 2;
00604     SmallVector<int, 8> ShuffleMask(NumElts);
00605 
00606     // The high bit of the selection field chooses the 1st or 2nd operand.
00607     bool LowInputSelect = Imm & 0x02;
00608     bool HighInputSelect = Imm & 0x20;
00609 
00610     // The low bit of the selection field chooses the low or high half
00611     // of the selected operand.
00612     bool LowHalfSelect = Imm & 0x01;
00613     bool HighHalfSelect = Imm & 0x10;
00614 
00615     // Determine which operand(s) are actually in use for this instruction.
00616     Value *V0 = LowInputSelect ? II.getArgOperand(1) : II.getArgOperand(0);
00617     Value *V1 = HighInputSelect ? II.getArgOperand(1) : II.getArgOperand(0);
00618 
00619     // If needed, replace operands based on zero mask.
00620     V0 = LowHalfZero ? ZeroVector : V0;
00621     V1 = HighHalfZero ? ZeroVector : V1;
00622 
00623     // Permute low half of result.
00624     unsigned StartIndex = LowHalfSelect ? HalfSize : 0;
00625     for (unsigned i = 0; i < HalfSize; ++i)
00626       ShuffleMask[i] = StartIndex + i;
00627 
00628     // Permute high half of result.
00629     StartIndex = HighHalfSelect ? HalfSize : 0;
00630     StartIndex += NumElts;
00631     for (unsigned i = 0; i < HalfSize; ++i)
00632       ShuffleMask[i + HalfSize] = StartIndex + i;
00633 
00634     return Builder.CreateShuffleVector(V0, V1, ShuffleMask);
00635   }
00636   return nullptr;
00637 }
00638 
00639 /// Decode XOP integer vector comparison intrinsics.
00640 static Value *SimplifyX86vpcom(const IntrinsicInst &II,
00641                                InstCombiner::BuilderTy &Builder, bool IsSigned) {
00642   if (auto *CInt = dyn_cast<ConstantInt>(II.getArgOperand(2))) {
00643     uint64_t Imm = CInt->getZExtValue() & 0x7;
00644     VectorType *VecTy = cast<VectorType>(II.getType());
00645     CmpInst::Predicate Pred = ICmpInst::BAD_ICMP_PREDICATE;
00646 
00647     switch (Imm) {
00648     case 0x0:
00649       Pred = IsSigned ? ICmpInst::ICMP_SLT : ICmpInst::ICMP_ULT;
00650       break;
00651     case 0x1:
00652       Pred = IsSigned ? ICmpInst::ICMP_SLE : ICmpInst::ICMP_ULE;
00653       break;
00654     case 0x2:
00655       Pred = IsSigned ? ICmpInst::ICMP_SGT : ICmpInst::ICMP_UGT;
00656       break;
00657     case 0x3:
00658       Pred = IsSigned ? ICmpInst::ICMP_SGE : ICmpInst::ICMP_UGE;
00659       break;
00660     case 0x4:
00661       Pred = ICmpInst::ICMP_EQ; break;
00662     case 0x5:
00663       Pred = ICmpInst::ICMP_NE; break;
00664     case 0x6:
00665       return ConstantInt::getSigned(VecTy, 0); // FALSE
00666     case 0x7:
00667       return ConstantInt::getSigned(VecTy, -1); // TRUE
00668     }
00669 
00670     if (Value *Cmp = Builder.CreateICmp(Pred, II.getArgOperand(0), II.getArgOperand(1)))
00671       return Builder.CreateSExtOrTrunc(Cmp, VecTy);
00672   }
00673   return nullptr;
00674 }
00675 
00676 /// CallInst simplification. This mostly only handles folding of intrinsic
00677 /// instructions. For normal calls, it allows visitCallSite to do the heavy
00678 /// lifting.
00679 Instruction *InstCombiner::visitCallInst(CallInst &CI) {
00680   auto Args = CI.arg_operands();
00681   if (Value *V = SimplifyCall(CI.getCalledValue(), Args.begin(), Args.end(), DL,
00682                               TLI, DT, AC))
00683     return ReplaceInstUsesWith(CI, V);
00684 
00685   if (isFreeCall(&CI, TLI))
00686     return visitFree(CI);
00687 
00688   // If the caller function is nounwind, mark the call as nounwind, even if the
00689   // callee isn't.
00690   if (CI.getParent()->getParent()->doesNotThrow() &&
00691       !CI.doesNotThrow()) {
00692     CI.setDoesNotThrow();
00693     return &CI;
00694   }
00695 
00696   IntrinsicInst *II = dyn_cast<IntrinsicInst>(&CI);
00697   if (!II) return visitCallSite(&CI);
00698 
00699   // Intrinsics cannot occur in an invoke, so handle them here instead of in
00700   // visitCallSite.
00701   if (MemIntrinsic *MI = dyn_cast<MemIntrinsic>(II)) {
00702     bool Changed = false;
00703 
00704     // memmove/cpy/set of zero bytes is a noop.
00705     if (Constant *NumBytes = dyn_cast<Constant>(MI->getLength())) {
00706       if (NumBytes->isNullValue())
00707         return EraseInstFromFunction(CI);
00708 
00709       if (ConstantInt *CI = dyn_cast<ConstantInt>(NumBytes))
00710         if (CI->getZExtValue() == 1) {
00711           // Replace the instruction with just byte operations.  We would
00712           // transform other cases to loads/stores, but we don't know if
00713           // alignment is sufficient.
00714         }
00715     }
00716 
00717     // No other transformations apply to volatile transfers.
00718     if (MI->isVolatile())
00719       return nullptr;
00720 
00721     // If we have a memmove and the source operation is a constant global,
00722     // then the source and dest pointers can't alias, so we can change this
00723     // into a call to memcpy.
00724     if (MemMoveInst *MMI = dyn_cast<MemMoveInst>(MI)) {
00725       if (GlobalVariable *GVSrc = dyn_cast<GlobalVariable>(MMI->getSource()))
00726         if (GVSrc->isConstant()) {
00727           Module *M = CI.getModule();
00728           Intrinsic::ID MemCpyID = Intrinsic::memcpy;
00729           Type *Tys[3] = { CI.getArgOperand(0)->getType(),
00730                            CI.getArgOperand(1)->getType(),
00731                            CI.getArgOperand(2)->getType() };
00732           CI.setCalledFunction(Intrinsic::getDeclaration(M, MemCpyID, Tys));
00733           Changed = true;
00734         }
00735     }
00736 
00737     if (MemTransferInst *MTI = dyn_cast<MemTransferInst>(MI)) {
00738       // memmove(x,x,size) -> noop.
00739       if (MTI->getSource() == MTI->getDest())
00740         return EraseInstFromFunction(CI);
00741     }
00742 
00743     // If we can determine a pointer alignment that is bigger than currently
00744     // set, update the alignment.
00745     if (isa<MemTransferInst>(MI)) {
00746       if (Instruction *I = SimplifyMemTransfer(MI))
00747         return I;
00748     } else if (MemSetInst *MSI = dyn_cast<MemSetInst>(MI)) {
00749       if (Instruction *I = SimplifyMemSet(MSI))
00750         return I;
00751     }
00752 
00753     if (Changed) return II;
00754   }
00755 
00756   auto SimplifyDemandedVectorEltsLow = [this](Value *Op, unsigned Width,
00757                                               unsigned DemandedWidth) {
00758     APInt UndefElts(Width, 0);
00759     APInt DemandedElts = APInt::getLowBitsSet(Width, DemandedWidth);
00760     return SimplifyDemandedVectorElts(Op, DemandedElts, UndefElts);
00761   };
00762 
00763   switch (II->getIntrinsicID()) {
00764   default: break;
00765   case Intrinsic::objectsize: {
00766     uint64_t Size;
00767     if (getObjectSize(II->getArgOperand(0), Size, DL, TLI))
00768       return ReplaceInstUsesWith(CI, ConstantInt::get(CI.getType(), Size));
00769     return nullptr;
00770   }
00771   case Intrinsic::bswap: {
00772     Value *IIOperand = II->getArgOperand(0);
00773     Value *X = nullptr;
00774 
00775     // bswap(bswap(x)) -> x
00776     if (match(IIOperand, m_BSwap(m_Value(X))))
00777         return ReplaceInstUsesWith(CI, X);
00778 
00779     // bswap(trunc(bswap(x))) -> trunc(lshr(x, c))
00780     if (match(IIOperand, m_Trunc(m_BSwap(m_Value(X))))) {
00781       unsigned C = X->getType()->getPrimitiveSizeInBits() -
00782         IIOperand->getType()->getPrimitiveSizeInBits();
00783       Value *CV = ConstantInt::get(X->getType(), C);
00784       Value *V = Builder->CreateLShr(X, CV);
00785       return new TruncInst(V, IIOperand->getType());
00786     }
00787     break;
00788   }
00789 
00790   case Intrinsic::bitreverse: {
00791     Value *IIOperand = II->getArgOperand(0);
00792     Value *X = nullptr;
00793 
00794     // bitreverse(bitreverse(x)) -> x
00795     if (match(IIOperand, m_Intrinsic<Intrinsic::bitreverse>(m_Value(X))))
00796       return ReplaceInstUsesWith(CI, X);
00797     break;
00798   }
00799 
00800   case Intrinsic::powi:
00801     if (ConstantInt *Power = dyn_cast<ConstantInt>(II->getArgOperand(1))) {
00802       // powi(x, 0) -> 1.0
00803       if (Power->isZero())
00804         return ReplaceInstUsesWith(CI, ConstantFP::get(CI.getType(), 1.0));
00805       // powi(x, 1) -> x
00806       if (Power->isOne())
00807         return ReplaceInstUsesWith(CI, II->getArgOperand(0));
00808       // powi(x, -1) -> 1/x
00809       if (Power->isAllOnesValue())
00810         return BinaryOperator::CreateFDiv(ConstantFP::get(CI.getType(), 1.0),
00811                                           II->getArgOperand(0));
00812     }
00813     break;
00814   case Intrinsic::cttz: {
00815     // If all bits below the first known one are known zero,
00816     // this value is constant.
00817     IntegerType *IT = dyn_cast<IntegerType>(II->getArgOperand(0)->getType());
00818     // FIXME: Try to simplify vectors of integers.
00819     if (!IT) break;
00820     uint32_t BitWidth = IT->getBitWidth();
00821     APInt KnownZero(BitWidth, 0);
00822     APInt KnownOne(BitWidth, 0);
00823     computeKnownBits(II->getArgOperand(0), KnownZero, KnownOne, 0, II);
00824     unsigned TrailingZeros = KnownOne.countTrailingZeros();
00825     APInt Mask(APInt::getLowBitsSet(BitWidth, TrailingZeros));
00826     if ((Mask & KnownZero) == Mask)
00827       return ReplaceInstUsesWith(CI, ConstantInt::get(IT,
00828                                  APInt(BitWidth, TrailingZeros)));
00829 
00830     }
00831     break;
00832   case Intrinsic::ctlz: {
00833     // If all bits above the first known one are known zero,
00834     // this value is constant.
00835     IntegerType *IT = dyn_cast<IntegerType>(II->getArgOperand(0)->getType());
00836     // FIXME: Try to simplify vectors of integers.
00837     if (!IT) break;
00838     uint32_t BitWidth = IT->getBitWidth();
00839     APInt KnownZero(BitWidth, 0);
00840     APInt KnownOne(BitWidth, 0);
00841     computeKnownBits(II->getArgOperand(0), KnownZero, KnownOne, 0, II);
00842     unsigned LeadingZeros = KnownOne.countLeadingZeros();
00843     APInt Mask(APInt::getHighBitsSet(BitWidth, LeadingZeros));
00844     if ((Mask & KnownZero) == Mask)
00845       return ReplaceInstUsesWith(CI, ConstantInt::get(IT,
00846                                  APInt(BitWidth, LeadingZeros)));
00847 
00848     }
00849     break;
00850 
00851   case Intrinsic::uadd_with_overflow:
00852   case Intrinsic::sadd_with_overflow:
00853   case Intrinsic::umul_with_overflow:
00854   case Intrinsic::smul_with_overflow:
00855     if (isa<Constant>(II->getArgOperand(0)) &&
00856         !isa<Constant>(II->getArgOperand(1))) {
00857       // Canonicalize constants into the RHS.
00858       Value *LHS = II->getArgOperand(0);
00859       II->setArgOperand(0, II->getArgOperand(1));
00860       II->setArgOperand(1, LHS);
00861       return II;
00862     }
00863     // fall through
00864 
00865   case Intrinsic::usub_with_overflow:
00866   case Intrinsic::ssub_with_overflow: {
00867     OverflowCheckFlavor OCF =
00868         IntrinsicIDToOverflowCheckFlavor(II->getIntrinsicID());
00869     assert(OCF != OCF_INVALID && "unexpected!");
00870 
00871     Value *OperationResult = nullptr;
00872     Constant *OverflowResult = nullptr;
00873     if (OptimizeOverflowCheck(OCF, II->getArgOperand(0), II->getArgOperand(1),
00874                               *II, OperationResult, OverflowResult))
00875       return CreateOverflowTuple(II, OperationResult, OverflowResult);
00876 
00877     break;
00878   }
00879 
00880   case Intrinsic::minnum:
00881   case Intrinsic::maxnum: {
00882     Value *Arg0 = II->getArgOperand(0);
00883     Value *Arg1 = II->getArgOperand(1);
00884 
00885     // fmin(x, x) -> x
00886     if (Arg0 == Arg1)
00887       return ReplaceInstUsesWith(CI, Arg0);
00888 
00889     const ConstantFP *C0 = dyn_cast<ConstantFP>(Arg0);
00890     const ConstantFP *C1 = dyn_cast<ConstantFP>(Arg1);
00891 
00892     // Canonicalize constants into the RHS.
00893     if (C0 && !C1) {
00894       II->setArgOperand(0, Arg1);
00895       II->setArgOperand(1, Arg0);
00896       return II;
00897     }
00898 
00899     // fmin(x, nan) -> x
00900     if (C1 && C1->isNaN())
00901       return ReplaceInstUsesWith(CI, Arg0);
00902 
00903     // This is the value because if undef were NaN, we would return the other
00904     // value and cannot return a NaN unless both operands are.
00905     //
00906     // fmin(undef, x) -> x
00907     if (isa<UndefValue>(Arg0))
00908       return ReplaceInstUsesWith(CI, Arg1);
00909 
00910     // fmin(x, undef) -> x
00911     if (isa<UndefValue>(Arg1))
00912       return ReplaceInstUsesWith(CI, Arg0);
00913 
00914     Value *X = nullptr;
00915     Value *Y = nullptr;
00916     if (II->getIntrinsicID() == Intrinsic::minnum) {
00917       // fmin(x, fmin(x, y)) -> fmin(x, y)
00918       // fmin(y, fmin(x, y)) -> fmin(x, y)
00919       if (match(Arg1, m_FMin(m_Value(X), m_Value(Y)))) {
00920         if (Arg0 == X || Arg0 == Y)
00921           return ReplaceInstUsesWith(CI, Arg1);
00922       }
00923 
00924       // fmin(fmin(x, y), x) -> fmin(x, y)
00925       // fmin(fmin(x, y), y) -> fmin(x, y)
00926       if (match(Arg0, m_FMin(m_Value(X), m_Value(Y)))) {
00927         if (Arg1 == X || Arg1 == Y)
00928           return ReplaceInstUsesWith(CI, Arg0);
00929       }
00930 
00931       // TODO: fmin(nnan x, inf) -> x
00932       // TODO: fmin(nnan ninf x, flt_max) -> x
00933       if (C1 && C1->isInfinity()) {
00934         // fmin(x, -inf) -> -inf
00935         if (C1->isNegative())
00936           return ReplaceInstUsesWith(CI, Arg1);
00937       }
00938     } else {
00939       assert(II->getIntrinsicID() == Intrinsic::maxnum);
00940       // fmax(x, fmax(x, y)) -> fmax(x, y)
00941       // fmax(y, fmax(x, y)) -> fmax(x, y)
00942       if (match(Arg1, m_FMax(m_Value(X), m_Value(Y)))) {
00943         if (Arg0 == X || Arg0 == Y)
00944           return ReplaceInstUsesWith(CI, Arg1);
00945       }
00946 
00947       // fmax(fmax(x, y), x) -> fmax(x, y)
00948       // fmax(fmax(x, y), y) -> fmax(x, y)
00949       if (match(Arg0, m_FMax(m_Value(X), m_Value(Y)))) {
00950         if (Arg1 == X || Arg1 == Y)
00951           return ReplaceInstUsesWith(CI, Arg0);
00952       }
00953 
00954       // TODO: fmax(nnan x, -inf) -> x
00955       // TODO: fmax(nnan ninf x, -flt_max) -> x
00956       if (C1 && C1->isInfinity()) {
00957         // fmax(x, inf) -> inf
00958         if (!C1->isNegative())
00959           return ReplaceInstUsesWith(CI, Arg1);
00960       }
00961     }
00962     break;
00963   }
00964   case Intrinsic::ppc_altivec_lvx:
00965   case Intrinsic::ppc_altivec_lvxl:
00966     // Turn PPC lvx -> load if the pointer is known aligned.
00967     if (getOrEnforceKnownAlignment(II->getArgOperand(0), 16, DL, II, AC, DT) >=
00968         16) {
00969       Value *Ptr = Builder->CreateBitCast(II->getArgOperand(0),
00970                                          PointerType::getUnqual(II->getType()));
00971       return new LoadInst(Ptr);
00972     }
00973     break;
00974   case Intrinsic::ppc_vsx_lxvw4x:
00975   case Intrinsic::ppc_vsx_lxvd2x: {
00976     // Turn PPC VSX loads into normal loads.
00977     Value *Ptr = Builder->CreateBitCast(II->getArgOperand(0),
00978                                         PointerType::getUnqual(II->getType()));
00979     return new LoadInst(Ptr, Twine(""), false, 1);
00980   }
00981   case Intrinsic::ppc_altivec_stvx:
00982   case Intrinsic::ppc_altivec_stvxl:
00983     // Turn stvx -> store if the pointer is known aligned.
00984     if (getOrEnforceKnownAlignment(II->getArgOperand(1), 16, DL, II, AC, DT) >=
00985         16) {
00986       Type *OpPtrTy =
00987         PointerType::getUnqual(II->getArgOperand(0)->getType());
00988       Value *Ptr = Builder->CreateBitCast(II->getArgOperand(1), OpPtrTy);
00989       return new StoreInst(II->getArgOperand(0), Ptr);
00990     }
00991     break;
00992   case Intrinsic::ppc_vsx_stxvw4x:
00993   case Intrinsic::ppc_vsx_stxvd2x: {
00994     // Turn PPC VSX stores into normal stores.
00995     Type *OpPtrTy = PointerType::getUnqual(II->getArgOperand(0)->getType());
00996     Value *Ptr = Builder->CreateBitCast(II->getArgOperand(1), OpPtrTy);
00997     return new StoreInst(II->getArgOperand(0), Ptr, false, 1);
00998   }
00999   case Intrinsic::ppc_qpx_qvlfs:
01000     // Turn PPC QPX qvlfs -> load if the pointer is known aligned.
01001     if (getOrEnforceKnownAlignment(II->getArgOperand(0), 16, DL, II, AC, DT) >=
01002         16) {
01003       Type *VTy = VectorType::get(Builder->getFloatTy(),
01004                                   II->getType()->getVectorNumElements());
01005       Value *Ptr = Builder->CreateBitCast(II->getArgOperand(0),
01006                                          PointerType::getUnqual(VTy));
01007       Value *Load = Builder->CreateLoad(Ptr);
01008       return new FPExtInst(Load, II->getType());
01009     }
01010     break;
01011   case Intrinsic::ppc_qpx_qvlfd:
01012     // Turn PPC QPX qvlfd -> load if the pointer is known aligned.
01013     if (getOrEnforceKnownAlignment(II->getArgOperand(0), 32, DL, II, AC, DT) >=
01014         32) {
01015       Value *Ptr = Builder->CreateBitCast(II->getArgOperand(0),
01016                                          PointerType::getUnqual(II->getType()));
01017       return new LoadInst(Ptr);
01018     }
01019     break;
01020   case Intrinsic::ppc_qpx_qvstfs:
01021     // Turn PPC QPX qvstfs -> store if the pointer is known aligned.
01022     if (getOrEnforceKnownAlignment(II->getArgOperand(1), 16, DL, II, AC, DT) >=
01023         16) {
01024       Type *VTy = VectorType::get(Builder->getFloatTy(),
01025           II->getArgOperand(0)->getType()->getVectorNumElements());
01026       Value *TOp = Builder->CreateFPTrunc(II->getArgOperand(0), VTy);
01027       Type *OpPtrTy = PointerType::getUnqual(VTy);
01028       Value *Ptr = Builder->CreateBitCast(II->getArgOperand(1), OpPtrTy);
01029       return new StoreInst(TOp, Ptr);
01030     }
01031     break;
01032   case Intrinsic::ppc_qpx_qvstfd:
01033     // Turn PPC QPX qvstfd -> store if the pointer is known aligned.
01034     if (getOrEnforceKnownAlignment(II->getArgOperand(1), 32, DL, II, AC, DT) >=
01035         32) {
01036       Type *OpPtrTy =
01037         PointerType::getUnqual(II->getArgOperand(0)->getType());
01038       Value *Ptr = Builder->CreateBitCast(II->getArgOperand(1), OpPtrTy);
01039       return new StoreInst(II->getArgOperand(0), Ptr);
01040     }
01041     break;
01042 
01043   case Intrinsic::x86_sse_storeu_ps:
01044   case Intrinsic::x86_sse2_storeu_pd:
01045   case Intrinsic::x86_sse2_storeu_dq:
01046     // Turn X86 storeu -> store if the pointer is known aligned.
01047     if (getOrEnforceKnownAlignment(II->getArgOperand(0), 16, DL, II, AC, DT) >=
01048         16) {
01049       Type *OpPtrTy =
01050         PointerType::getUnqual(II->getArgOperand(1)->getType());
01051       Value *Ptr = Builder->CreateBitCast(II->getArgOperand(0), OpPtrTy);
01052       return new StoreInst(II->getArgOperand(1), Ptr);
01053     }
01054     break;
01055 
01056   case Intrinsic::x86_vcvtph2ps_128:
01057   case Intrinsic::x86_vcvtph2ps_256: {
01058     auto Arg = II->getArgOperand(0);
01059     auto ArgType = cast<VectorType>(Arg->getType());
01060     auto RetType = cast<VectorType>(II->getType());
01061     unsigned ArgWidth = ArgType->getNumElements();
01062     unsigned RetWidth = RetType->getNumElements();
01063     assert(RetWidth <= ArgWidth && "Unexpected input/return vector widths");
01064     assert(ArgType->isIntOrIntVectorTy() &&
01065            ArgType->getScalarSizeInBits() == 16 &&
01066            "CVTPH2PS input type should be 16-bit integer vector");
01067     assert(RetType->getScalarType()->isFloatTy() &&
01068            "CVTPH2PS output type should be 32-bit float vector");
01069 
01070     // Constant folding: Convert to generic half to single conversion.
01071     if (isa<ConstantAggregateZero>(Arg))
01072       return ReplaceInstUsesWith(*II, ConstantAggregateZero::get(RetType));
01073 
01074     if (isa<ConstantDataVector>(Arg)) {
01075       auto VectorHalfAsShorts = Arg;
01076       if (RetWidth < ArgWidth) {
01077         SmallVector<int, 8> SubVecMask;
01078         for (unsigned i = 0; i != RetWidth; ++i)
01079           SubVecMask.push_back((int)i);
01080         VectorHalfAsShorts = Builder->CreateShuffleVector(
01081             Arg, UndefValue::get(ArgType), SubVecMask);
01082       }
01083 
01084       auto VectorHalfType =
01085           VectorType::get(Type::getHalfTy(II->getContext()), RetWidth);
01086       auto VectorHalfs =
01087           Builder->CreateBitCast(VectorHalfAsShorts, VectorHalfType);
01088       auto VectorFloats = Builder->CreateFPExt(VectorHalfs, RetType);
01089       return ReplaceInstUsesWith(*II, VectorFloats);
01090     }
01091 
01092     // We only use the lowest lanes of the argument.
01093     if (Value *V = SimplifyDemandedVectorEltsLow(Arg, ArgWidth, RetWidth)) {
01094       II->setArgOperand(0, V);
01095       return II;
01096     }
01097     break;
01098   }
01099 
01100   case Intrinsic::x86_sse_cvtss2si:
01101   case Intrinsic::x86_sse_cvtss2si64:
01102   case Intrinsic::x86_sse_cvttss2si:
01103   case Intrinsic::x86_sse_cvttss2si64:
01104   case Intrinsic::x86_sse2_cvtsd2si:
01105   case Intrinsic::x86_sse2_cvtsd2si64:
01106   case Intrinsic::x86_sse2_cvttsd2si:
01107   case Intrinsic::x86_sse2_cvttsd2si64: {
01108     // These intrinsics only demand the 0th element of their input vectors. If
01109     // we can simplify the input based on that, do so now.
01110     Value *Arg = II->getArgOperand(0);
01111     unsigned VWidth = Arg->getType()->getVectorNumElements();
01112     if (Value *V = SimplifyDemandedVectorEltsLow(Arg, VWidth, 1)) {
01113       II->setArgOperand(0, V);
01114       return II;
01115     }
01116     break;
01117   }
01118 
01119   // Constant fold ashr( <A x Bi>, Ci ).
01120   // Constant fold lshr( <A x Bi>, Ci ).
01121   // Constant fold shl( <A x Bi>, Ci ).
01122   case Intrinsic::x86_sse2_psrai_d:
01123   case Intrinsic::x86_sse2_psrai_w:
01124   case Intrinsic::x86_avx2_psrai_d:
01125   case Intrinsic::x86_avx2_psrai_w:
01126   case Intrinsic::x86_sse2_psrli_d:
01127   case Intrinsic::x86_sse2_psrli_q:
01128   case Intrinsic::x86_sse2_psrli_w:
01129   case Intrinsic::x86_avx2_psrli_d:
01130   case Intrinsic::x86_avx2_psrli_q:
01131   case Intrinsic::x86_avx2_psrli_w:
01132   case Intrinsic::x86_sse2_pslli_d:
01133   case Intrinsic::x86_sse2_pslli_q:
01134   case Intrinsic::x86_sse2_pslli_w:
01135   case Intrinsic::x86_avx2_pslli_d:
01136   case Intrinsic::x86_avx2_pslli_q:
01137   case Intrinsic::x86_avx2_pslli_w:
01138     if (Value *V = SimplifyX86immshift(*II, *Builder))
01139       return ReplaceInstUsesWith(*II, V);
01140     break;
01141 
01142   case Intrinsic::x86_sse2_psra_d:
01143   case Intrinsic::x86_sse2_psra_w:
01144   case Intrinsic::x86_avx2_psra_d:
01145   case Intrinsic::x86_avx2_psra_w:
01146   case Intrinsic::x86_sse2_psrl_d:
01147   case Intrinsic::x86_sse2_psrl_q:
01148   case Intrinsic::x86_sse2_psrl_w:
01149   case Intrinsic::x86_avx2_psrl_d:
01150   case Intrinsic::x86_avx2_psrl_q:
01151   case Intrinsic::x86_avx2_psrl_w:
01152   case Intrinsic::x86_sse2_psll_d:
01153   case Intrinsic::x86_sse2_psll_q:
01154   case Intrinsic::x86_sse2_psll_w:
01155   case Intrinsic::x86_avx2_psll_d:
01156   case Intrinsic::x86_avx2_psll_q:
01157   case Intrinsic::x86_avx2_psll_w: {
01158     if (Value *V = SimplifyX86immshift(*II, *Builder))
01159       return ReplaceInstUsesWith(*II, V);
01160 
01161     // SSE2/AVX2 uses only the first 64-bits of the 128-bit vector
01162     // operand to compute the shift amount.
01163     Value *Arg1 = II->getArgOperand(1);
01164     assert(Arg1->getType()->getPrimitiveSizeInBits() == 128 &&
01165            "Unexpected packed shift size");
01166     unsigned VWidth = Arg1->getType()->getVectorNumElements();
01167 
01168     if (Value *V = SimplifyDemandedVectorEltsLow(Arg1, VWidth, VWidth / 2)) {
01169       II->setArgOperand(1, V);
01170       return II;
01171     }
01172     break;
01173   }
01174 
01175   case Intrinsic::x86_avx2_pmovsxbd:
01176   case Intrinsic::x86_avx2_pmovsxbq:
01177   case Intrinsic::x86_avx2_pmovsxbw:
01178   case Intrinsic::x86_avx2_pmovsxdq:
01179   case Intrinsic::x86_avx2_pmovsxwd:
01180   case Intrinsic::x86_avx2_pmovsxwq:
01181     if (Value *V = SimplifyX86extend(*II, *Builder, true))
01182       return ReplaceInstUsesWith(*II, V);
01183     break;
01184 
01185   case Intrinsic::x86_sse41_pmovzxbd:
01186   case Intrinsic::x86_sse41_pmovzxbq:
01187   case Intrinsic::x86_sse41_pmovzxbw:
01188   case Intrinsic::x86_sse41_pmovzxdq:
01189   case Intrinsic::x86_sse41_pmovzxwd:
01190   case Intrinsic::x86_sse41_pmovzxwq:
01191   case Intrinsic::x86_avx2_pmovzxbd:
01192   case Intrinsic::x86_avx2_pmovzxbq:
01193   case Intrinsic::x86_avx2_pmovzxbw:
01194   case Intrinsic::x86_avx2_pmovzxdq:
01195   case Intrinsic::x86_avx2_pmovzxwd:
01196   case Intrinsic::x86_avx2_pmovzxwq:
01197     if (Value *V = SimplifyX86extend(*II, *Builder, false))
01198       return ReplaceInstUsesWith(*II, V);
01199     break;
01200 
01201   case Intrinsic::x86_sse41_insertps:
01202     if (Value *V = SimplifyX86insertps(*II, *Builder))
01203       return ReplaceInstUsesWith(*II, V);
01204     break;
01205 
01206   case Intrinsic::x86_sse4a_extrq: {
01207     Value *Op0 = II->getArgOperand(0);
01208     Value *Op1 = II->getArgOperand(1);
01209     unsigned VWidth0 = Op0->getType()->getVectorNumElements();
01210     unsigned VWidth1 = Op1->getType()->getVectorNumElements();
01211     assert(Op0->getType()->getPrimitiveSizeInBits() == 128 &&
01212            Op1->getType()->getPrimitiveSizeInBits() == 128 && VWidth0 == 2 &&
01213            VWidth1 == 16 && "Unexpected operand sizes");
01214 
01215     // See if we're dealing with constant values.
01216     Constant *C1 = dyn_cast<Constant>(Op1);
01217     ConstantInt *CILength =
01218         C1 ? dyn_cast<ConstantInt>(C1->getAggregateElement((unsigned)0))
01219            : nullptr;
01220     ConstantInt *CIIndex =
01221         C1 ? dyn_cast<ConstantInt>(C1->getAggregateElement((unsigned)1))
01222            : nullptr;
01223 
01224     // Attempt to simplify to a constant, shuffle vector or EXTRQI call.
01225     if (Value *V = SimplifyX86extrq(*II, Op0, CILength, CIIndex, *Builder))
01226       return ReplaceInstUsesWith(*II, V);
01227 
01228     // EXTRQ only uses the lowest 64-bits of the first 128-bit vector
01229     // operands and the lowest 16-bits of the second.
01230     if (Value *V = SimplifyDemandedVectorEltsLow(Op0, VWidth0, 1)) {
01231       II->setArgOperand(0, V);
01232       return II;
01233     }
01234     if (Value *V = SimplifyDemandedVectorEltsLow(Op1, VWidth1, 2)) {
01235       II->setArgOperand(1, V);
01236       return II;
01237     }
01238     break;
01239   }
01240 
01241   case Intrinsic::x86_sse4a_extrqi: {
01242     // EXTRQI: Extract Length bits starting from Index. Zero pad the remaining
01243     // bits of the lower 64-bits. The upper 64-bits are undefined.
01244     Value *Op0 = II->getArgOperand(0);
01245     unsigned VWidth = Op0->getType()->getVectorNumElements();
01246     assert(Op0->getType()->getPrimitiveSizeInBits() == 128 && VWidth == 2 &&
01247            "Unexpected operand size");
01248 
01249     // See if we're dealing with constant values.
01250     ConstantInt *CILength = dyn_cast<ConstantInt>(II->getArgOperand(1));
01251     ConstantInt *CIIndex = dyn_cast<ConstantInt>(II->getArgOperand(2));
01252 
01253     // Attempt to simplify to a constant or shuffle vector.
01254     if (Value *V = SimplifyX86extrq(*II, Op0, CILength, CIIndex, *Builder))
01255       return ReplaceInstUsesWith(*II, V);
01256 
01257     // EXTRQI only uses the lowest 64-bits of the first 128-bit vector
01258     // operand.
01259     if (Value *V = SimplifyDemandedVectorEltsLow(Op0, VWidth, 1)) {
01260       II->setArgOperand(0, V);
01261       return II;
01262     }
01263     break;
01264   }
01265 
01266   case Intrinsic::x86_sse4a_insertq: {
01267     Value *Op0 = II->getArgOperand(0);
01268     Value *Op1 = II->getArgOperand(1);
01269     unsigned VWidth = Op0->getType()->getVectorNumElements();
01270     assert(Op0->getType()->getPrimitiveSizeInBits() == 128 &&
01271            Op1->getType()->getPrimitiveSizeInBits() == 128 && VWidth == 2 &&
01272            Op1->getType()->getVectorNumElements() == 2 &&
01273            "Unexpected operand size");
01274 
01275     // See if we're dealing with constant values.
01276     Constant *C1 = dyn_cast<Constant>(Op1);
01277     ConstantInt *CI11 =
01278         C1 ? dyn_cast<ConstantInt>(C1->getAggregateElement((unsigned)1))
01279            : nullptr;
01280 
01281     // Attempt to simplify to a constant, shuffle vector or INSERTQI call.
01282     if (CI11) {
01283       APInt V11 = CI11->getValue();
01284       APInt Len = V11.zextOrTrunc(6);
01285       APInt Idx = V11.lshr(8).zextOrTrunc(6);
01286       if (Value *V = SimplifyX86insertq(*II, Op0, Op1, Len, Idx, *Builder))
01287         return ReplaceInstUsesWith(*II, V);
01288     }
01289 
01290     // INSERTQ only uses the lowest 64-bits of the first 128-bit vector
01291     // operand.
01292     if (Value *V = SimplifyDemandedVectorEltsLow(Op0, VWidth, 1)) {
01293       II->setArgOperand(0, V);
01294       return II;
01295     }
01296     break;
01297   }
01298 
01299   case Intrinsic::x86_sse4a_insertqi: {
01300     // INSERTQI: Extract lowest Length bits from lower half of second source and
01301     // insert over first source starting at Index bit. The upper 64-bits are
01302     // undefined.
01303     Value *Op0 = II->getArgOperand(0);
01304     Value *Op1 = II->getArgOperand(1);
01305     unsigned VWidth0 = Op0->getType()->getVectorNumElements();
01306     unsigned VWidth1 = Op1->getType()->getVectorNumElements();
01307     assert(Op0->getType()->getPrimitiveSizeInBits() == 128 &&
01308            Op1->getType()->getPrimitiveSizeInBits() == 128 && VWidth0 == 2 &&
01309            VWidth1 == 2 && "Unexpected operand sizes");
01310 
01311     // See if we're dealing with constant values.
01312     ConstantInt *CILength = dyn_cast<ConstantInt>(II->getArgOperand(2));
01313     ConstantInt *CIIndex = dyn_cast<ConstantInt>(II->getArgOperand(3));
01314 
01315     // Attempt to simplify to a constant or shuffle vector.
01316     if (CILength && CIIndex) {
01317       APInt Len = CILength->getValue().zextOrTrunc(6);
01318       APInt Idx = CIIndex->getValue().zextOrTrunc(6);
01319       if (Value *V = SimplifyX86insertq(*II, Op0, Op1, Len, Idx, *Builder))
01320         return ReplaceInstUsesWith(*II, V);
01321     }
01322 
01323     // INSERTQI only uses the lowest 64-bits of the first two 128-bit vector
01324     // operands.
01325     if (Value *V = SimplifyDemandedVectorEltsLow(Op0, VWidth0, 1)) {
01326       II->setArgOperand(0, V);
01327       return II;
01328     }
01329 
01330     if (Value *V = SimplifyDemandedVectorEltsLow(Op1, VWidth1, 1)) {
01331       II->setArgOperand(1, V);
01332       return II;
01333     }
01334     break;
01335   }
01336 
01337   case Intrinsic::x86_sse41_pblendvb:
01338   case Intrinsic::x86_sse41_blendvps:
01339   case Intrinsic::x86_sse41_blendvpd:
01340   case Intrinsic::x86_avx_blendv_ps_256:
01341   case Intrinsic::x86_avx_blendv_pd_256:
01342   case Intrinsic::x86_avx2_pblendvb: {
01343     // Convert blendv* to vector selects if the mask is constant.
01344     // This optimization is convoluted because the intrinsic is defined as
01345     // getting a vector of floats or doubles for the ps and pd versions.
01346     // FIXME: That should be changed.
01347 
01348     Value *Op0 = II->getArgOperand(0);
01349     Value *Op1 = II->getArgOperand(1);
01350     Value *Mask = II->getArgOperand(2);
01351 
01352     // fold (blend A, A, Mask) -> A
01353     if (Op0 == Op1)
01354       return ReplaceInstUsesWith(CI, Op0);
01355 
01356     // Zero Mask - select 1st argument.
01357     if (isa<ConstantAggregateZero>(Mask))
01358       return ReplaceInstUsesWith(CI, Op0);
01359 
01360     // Constant Mask - select 1st/2nd argument lane based on top bit of mask.
01361     if (auto C = dyn_cast<ConstantDataVector>(Mask)) {
01362       auto Tyi1 = Builder->getInt1Ty();
01363       auto SelectorType = cast<VectorType>(Mask->getType());
01364       auto EltTy = SelectorType->getElementType();
01365       unsigned Size = SelectorType->getNumElements();
01366       unsigned BitWidth =
01367           EltTy->isFloatTy()
01368               ? 32
01369               : (EltTy->isDoubleTy() ? 64 : EltTy->getIntegerBitWidth());
01370       assert((BitWidth == 64 || BitWidth == 32 || BitWidth == 8) &&
01371              "Wrong arguments for variable blend intrinsic");
01372       SmallVector<Constant *, 32> Selectors;
01373       for (unsigned I = 0; I < Size; ++I) {
01374         // The intrinsics only read the top bit
01375         uint64_t Selector;
01376         if (BitWidth == 8)
01377           Selector = C->getElementAsInteger(I);
01378         else
01379           Selector = C->getElementAsAPFloat(I).bitcastToAPInt().getZExtValue();
01380         Selectors.push_back(ConstantInt::get(Tyi1, Selector >> (BitWidth - 1)));
01381       }
01382       auto NewSelector = ConstantVector::get(Selectors);
01383       return SelectInst::Create(NewSelector, Op1, Op0, "blendv");
01384     }
01385     break;
01386   }
01387 
01388   case Intrinsic::x86_ssse3_pshuf_b_128:
01389   case Intrinsic::x86_avx2_pshuf_b: {
01390     // Turn pshufb(V1,mask) -> shuffle(V1,Zero,mask) if mask is a constant.
01391     auto *V = II->getArgOperand(1);
01392     auto *VTy = cast<VectorType>(V->getType());
01393     unsigned NumElts = VTy->getNumElements();
01394     assert((NumElts == 16 || NumElts == 32) &&
01395            "Unexpected number of elements in shuffle mask!");
01396     // Initialize the resulting shuffle mask to all zeroes.
01397     uint32_t Indexes[32] = {0};
01398 
01399     if (auto *Mask = dyn_cast<ConstantDataVector>(V)) {
01400       // Each byte in the shuffle control mask forms an index to permute the
01401       // corresponding byte in the destination operand.
01402       for (unsigned I = 0; I < NumElts; ++I) {
01403         int8_t Index = Mask->getElementAsInteger(I);
01404         // If the most significant bit (bit[7]) of each byte of the shuffle
01405         // control mask is set, then zero is written in the result byte.
01406         // The zero vector is in the right-hand side of the resulting
01407         // shufflevector.
01408 
01409         // The value of each index is the least significant 4 bits of the
01410         // shuffle control byte.
01411         Indexes[I] = (Index < 0) ? NumElts : Index & 0xF;
01412       }
01413     } else if (!isa<ConstantAggregateZero>(V))
01414       break;
01415 
01416     // The value of each index for the high 128-bit lane is the least
01417     // significant 4 bits of the respective shuffle control byte.
01418     for (unsigned I = 16; I < NumElts; ++I)
01419       Indexes[I] += I & 0xF0;
01420 
01421     auto NewC = ConstantDataVector::get(V->getContext(),
01422                                         makeArrayRef(Indexes, NumElts));
01423     auto V1 = II->getArgOperand(0);
01424     auto V2 = Constant::getNullValue(II->getType());
01425     auto Shuffle = Builder->CreateShuffleVector(V1, V2, NewC);
01426     return ReplaceInstUsesWith(CI, Shuffle);
01427   }
01428 
01429   case Intrinsic::x86_avx_vpermilvar_ps:
01430   case Intrinsic::x86_avx_vpermilvar_ps_256:
01431   case Intrinsic::x86_avx_vpermilvar_pd:
01432   case Intrinsic::x86_avx_vpermilvar_pd_256: {
01433     // Convert vpermil* to shufflevector if the mask is constant.
01434     Value *V = II->getArgOperand(1);
01435     unsigned Size = cast<VectorType>(V->getType())->getNumElements();
01436     assert(Size == 8 || Size == 4 || Size == 2);
01437     uint32_t Indexes[8];
01438     if (auto C = dyn_cast<ConstantDataVector>(V)) {
01439       // The intrinsics only read one or two bits, clear the rest.
01440       for (unsigned I = 0; I < Size; ++I) {
01441         uint32_t Index = C->getElementAsInteger(I) & 0x3;
01442         if (II->getIntrinsicID() == Intrinsic::x86_avx_vpermilvar_pd ||
01443             II->getIntrinsicID() == Intrinsic::x86_avx_vpermilvar_pd_256)
01444           Index >>= 1;
01445         Indexes[I] = Index;
01446       }
01447     } else if (isa<ConstantAggregateZero>(V)) {
01448       for (unsigned I = 0; I < Size; ++I)
01449         Indexes[I] = 0;
01450     } else {
01451       break;
01452     }
01453     // The _256 variants are a bit trickier since the mask bits always index
01454     // into the corresponding 128 half. In order to convert to a generic
01455     // shuffle, we have to make that explicit.
01456     if (II->getIntrinsicID() == Intrinsic::x86_avx_vpermilvar_ps_256 ||
01457         II->getIntrinsicID() == Intrinsic::x86_avx_vpermilvar_pd_256) {
01458       for (unsigned I = Size / 2; I < Size; ++I)
01459         Indexes[I] += Size / 2;
01460     }
01461     auto NewC =
01462         ConstantDataVector::get(V->getContext(), makeArrayRef(Indexes, Size));
01463     auto V1 = II->getArgOperand(0);
01464     auto V2 = UndefValue::get(V1->getType());
01465     auto Shuffle = Builder->CreateShuffleVector(V1, V2, NewC);
01466     return ReplaceInstUsesWith(CI, Shuffle);
01467   }
01468 
01469   case Intrinsic::x86_avx_vperm2f128_pd_256:
01470   case Intrinsic::x86_avx_vperm2f128_ps_256:
01471   case Intrinsic::x86_avx_vperm2f128_si_256:
01472   case Intrinsic::x86_avx2_vperm2i128:
01473     if (Value *V = SimplifyX86vperm2(*II, *Builder))
01474       return ReplaceInstUsesWith(*II, V);
01475     break;
01476 
01477   case Intrinsic::x86_xop_vpcomb:
01478   case Intrinsic::x86_xop_vpcomd:
01479   case Intrinsic::x86_xop_vpcomq:
01480   case Intrinsic::x86_xop_vpcomw:
01481     if (Value *V = SimplifyX86vpcom(*II, *Builder, true))
01482       return ReplaceInstUsesWith(*II, V);
01483     break;
01484 
01485   case Intrinsic::x86_xop_vpcomub:
01486   case Intrinsic::x86_xop_vpcomud:
01487   case Intrinsic::x86_xop_vpcomuq:
01488   case Intrinsic::x86_xop_vpcomuw:
01489     if (Value *V = SimplifyX86vpcom(*II, *Builder, false))
01490       return ReplaceInstUsesWith(*II, V);
01491     break;
01492 
01493   case Intrinsic::ppc_altivec_vperm:
01494     // Turn vperm(V1,V2,mask) -> shuffle(V1,V2,mask) if mask is a constant.
01495     // Note that ppc_altivec_vperm has a big-endian bias, so when creating
01496     // a vectorshuffle for little endian, we must undo the transformation
01497     // performed on vec_perm in altivec.h.  That is, we must complement
01498     // the permutation mask with respect to 31 and reverse the order of
01499     // V1 and V2.
01500     if (Constant *Mask = dyn_cast<Constant>(II->getArgOperand(2))) {
01501       assert(Mask->getType()->getVectorNumElements() == 16 &&
01502              "Bad type for intrinsic!");
01503 
01504       // Check that all of the elements are integer constants or undefs.
01505       bool AllEltsOk = true;
01506       for (unsigned i = 0; i != 16; ++i) {
01507         Constant *Elt = Mask->getAggregateElement(i);
01508         if (!Elt || !(isa<ConstantInt>(Elt) || isa<UndefValue>(Elt))) {
01509           AllEltsOk = false;
01510           break;
01511         }
01512       }
01513 
01514       if (AllEltsOk) {
01515         // Cast the input vectors to byte vectors.
01516         Value *Op0 = Builder->CreateBitCast(II->getArgOperand(0),
01517                                             Mask->getType());
01518         Value *Op1 = Builder->CreateBitCast(II->getArgOperand(1),
01519                                             Mask->getType());
01520         Value *Result = UndefValue::get(Op0->getType());
01521 
01522         // Only extract each element once.
01523         Value *ExtractedElts[32];
01524         memset(ExtractedElts, 0, sizeof(ExtractedElts));
01525 
01526         for (unsigned i = 0; i != 16; ++i) {
01527           if (isa<UndefValue>(Mask->getAggregateElement(i)))
01528             continue;
01529           unsigned Idx =
01530             cast<ConstantInt>(Mask->getAggregateElement(i))->getZExtValue();
01531           Idx &= 31;  // Match the hardware behavior.
01532           if (DL.isLittleEndian())
01533             Idx = 31 - Idx;
01534 
01535           if (!ExtractedElts[Idx]) {
01536             Value *Op0ToUse = (DL.isLittleEndian()) ? Op1 : Op0;
01537             Value *Op1ToUse = (DL.isLittleEndian()) ? Op0 : Op1;
01538             ExtractedElts[Idx] =
01539               Builder->CreateExtractElement(Idx < 16 ? Op0ToUse : Op1ToUse,
01540                                             Builder->getInt32(Idx&15));
01541           }
01542 
01543           // Insert this value into the result vector.
01544           Result = Builder->CreateInsertElement(Result, ExtractedElts[Idx],
01545                                                 Builder->getInt32(i));
01546         }
01547         return CastInst::Create(Instruction::BitCast, Result, CI.getType());
01548       }
01549     }
01550     break;
01551 
01552   case Intrinsic::arm_neon_vld1:
01553   case Intrinsic::arm_neon_vld2:
01554   case Intrinsic::arm_neon_vld3:
01555   case Intrinsic::arm_neon_vld4:
01556   case Intrinsic::arm_neon_vld2lane:
01557   case Intrinsic::arm_neon_vld3lane:
01558   case Intrinsic::arm_neon_vld4lane:
01559   case Intrinsic::arm_neon_vst1:
01560   case Intrinsic::arm_neon_vst2:
01561   case Intrinsic::arm_neon_vst3:
01562   case Intrinsic::arm_neon_vst4:
01563   case Intrinsic::arm_neon_vst2lane:
01564   case Intrinsic::arm_neon_vst3lane:
01565   case Intrinsic::arm_neon_vst4lane: {
01566     unsigned MemAlign = getKnownAlignment(II->getArgOperand(0), DL, II, AC, DT);
01567     unsigned AlignArg = II->getNumArgOperands() - 1;
01568     ConstantInt *IntrAlign = dyn_cast<ConstantInt>(II->getArgOperand(AlignArg));
01569     if (IntrAlign && IntrAlign->getZExtValue() < MemAlign) {
01570       II->setArgOperand(AlignArg,
01571                         ConstantInt::get(Type::getInt32Ty(II->getContext()),
01572                                          MemAlign, false));
01573       return II;
01574     }
01575     break;
01576   }
01577 
01578   case Intrinsic::arm_neon_vmulls:
01579   case Intrinsic::arm_neon_vmullu:
01580   case Intrinsic::aarch64_neon_smull:
01581   case Intrinsic::aarch64_neon_umull: {
01582     Value *Arg0 = II->getArgOperand(0);
01583     Value *Arg1 = II->getArgOperand(1);
01584 
01585     // Handle mul by zero first:
01586     if (isa<ConstantAggregateZero>(Arg0) || isa<ConstantAggregateZero>(Arg1)) {
01587       return ReplaceInstUsesWith(CI, ConstantAggregateZero::get(II->getType()));
01588     }
01589 
01590     // Check for constant LHS & RHS - in this case we just simplify.
01591     bool Zext = (II->getIntrinsicID() == Intrinsic::arm_neon_vmullu ||
01592                  II->getIntrinsicID() == Intrinsic::aarch64_neon_umull);
01593     VectorType *NewVT = cast<VectorType>(II->getType());
01594     if (Constant *CV0 = dyn_cast<Constant>(Arg0)) {
01595       if (Constant *CV1 = dyn_cast<Constant>(Arg1)) {
01596         CV0 = ConstantExpr::getIntegerCast(CV0, NewVT, /*isSigned=*/!Zext);
01597         CV1 = ConstantExpr::getIntegerCast(CV1, NewVT, /*isSigned=*/!Zext);
01598 
01599         return ReplaceInstUsesWith(CI, ConstantExpr::getMul(CV0, CV1));
01600       }
01601 
01602       // Couldn't simplify - canonicalize constant to the RHS.
01603       std::swap(Arg0, Arg1);
01604     }
01605 
01606     // Handle mul by one:
01607     if (Constant *CV1 = dyn_cast<Constant>(Arg1))
01608       if (ConstantInt *Splat =
01609               dyn_cast_or_null<ConstantInt>(CV1->getSplatValue()))
01610         if (Splat->isOne())
01611           return CastInst::CreateIntegerCast(Arg0, II->getType(),
01612                                              /*isSigned=*/!Zext);
01613 
01614     break;
01615   }
01616 
01617   case Intrinsic::amdgcn_rcp: {
01618     if (const ConstantFP *C = dyn_cast<ConstantFP>(II->getArgOperand(0))) {
01619       const APFloat &ArgVal = C->getValueAPF();
01620       APFloat Val(ArgVal.getSemantics(), 1.0);
01621       APFloat::opStatus Status = Val.divide(ArgVal,
01622                                             APFloat::rmNearestTiesToEven);
01623       // Only do this if it was exact and therefore not dependent on the
01624       // rounding mode.
01625       if (Status == APFloat::opOK)
01626         return ReplaceInstUsesWith(CI, ConstantFP::get(II->getContext(), Val));
01627     }
01628 
01629     break;
01630   }
01631   case Intrinsic::stackrestore: {
01632     // If the save is right next to the restore, remove the restore.  This can
01633     // happen when variable allocas are DCE'd.
01634     if (IntrinsicInst *SS = dyn_cast<IntrinsicInst>(II->getArgOperand(0))) {
01635       if (SS->getIntrinsicID() == Intrinsic::stacksave) {
01636         if (&*++SS->getIterator() == II)
01637           return EraseInstFromFunction(CI);
01638       }
01639     }
01640 
01641     // Scan down this block to see if there is another stack restore in the
01642     // same block without an intervening call/alloca.
01643     BasicBlock::iterator BI(II);
01644     TerminatorInst *TI = II->getParent()->getTerminator();
01645     bool CannotRemove = false;
01646     for (++BI; &*BI != TI; ++BI) {
01647       if (isa<AllocaInst>(BI)) {
01648         CannotRemove = true;
01649         break;
01650       }
01651       if (CallInst *BCI = dyn_cast<CallInst>(BI)) {
01652         if (IntrinsicInst *II = dyn_cast<IntrinsicInst>(BCI)) {
01653           // If there is a stackrestore below this one, remove this one.
01654           if (II->getIntrinsicID() == Intrinsic::stackrestore)
01655             return EraseInstFromFunction(CI);
01656           // Otherwise, ignore the intrinsic.
01657         } else {
01658           // If we found a non-intrinsic call, we can't remove the stack
01659           // restore.
01660           CannotRemove = true;
01661           break;
01662         }
01663       }
01664     }
01665 
01666     // If the stack restore is in a return, resume, or unwind block and if there
01667     // are no allocas or calls between the restore and the return, nuke the
01668     // restore.
01669     if (!CannotRemove && (isa<ReturnInst>(TI) || isa<ResumeInst>(TI)))
01670       return EraseInstFromFunction(CI);
01671     break;
01672   }
01673   case Intrinsic::lifetime_start: {
01674     // Remove trivially empty lifetime_start/end ranges, i.e. a start
01675     // immediately followed by an end (ignoring debuginfo or other
01676     // lifetime markers in between).
01677     BasicBlock::iterator BI = II->getIterator(), BE = II->getParent()->end();
01678     for (++BI; BI != BE; ++BI) {
01679       if (IntrinsicInst *LTE = dyn_cast<IntrinsicInst>(BI)) {
01680         if (isa<DbgInfoIntrinsic>(LTE) ||
01681             LTE->getIntrinsicID() == Intrinsic::lifetime_start)
01682           continue;
01683         if (LTE->getIntrinsicID() == Intrinsic::lifetime_end) {
01684           if (II->getOperand(0) == LTE->getOperand(0) &&
01685               II->getOperand(1) == LTE->getOperand(1)) {
01686             EraseInstFromFunction(*LTE);
01687             return EraseInstFromFunction(*II);
01688           }
01689           continue;
01690         }
01691       }
01692       break;
01693     }
01694     break;
01695   }
01696   case Intrinsic::assume: {
01697     // Canonicalize assume(a && b) -> assume(a); assume(b);
01698     // Note: New assumption intrinsics created here are registered by
01699     // the InstCombineIRInserter object.
01700     Value *IIOperand = II->getArgOperand(0), *A, *B,
01701           *AssumeIntrinsic = II->getCalledValue();
01702     if (match(IIOperand, m_And(m_Value(A), m_Value(B)))) {
01703       Builder->CreateCall(AssumeIntrinsic, A, II->getName());
01704       Builder->CreateCall(AssumeIntrinsic, B, II->getName());
01705       return EraseInstFromFunction(*II);
01706     }
01707     // assume(!(a || b)) -> assume(!a); assume(!b);
01708     if (match(IIOperand, m_Not(m_Or(m_Value(A), m_Value(B))))) {
01709       Builder->CreateCall(AssumeIntrinsic, Builder->CreateNot(A),
01710                           II->getName());
01711       Builder->CreateCall(AssumeIntrinsic, Builder->CreateNot(B),
01712                           II->getName());
01713       return EraseInstFromFunction(*II);
01714     }
01715 
01716     // assume( (load addr) != null ) -> add 'nonnull' metadata to load
01717     // (if assume is valid at the load)
01718     if (ICmpInst* ICmp = dyn_cast<ICmpInst>(IIOperand)) {
01719       Value *LHS = ICmp->getOperand(0);
01720       Value *RHS = ICmp->getOperand(1);
01721       if (ICmpInst::ICMP_NE == ICmp->getPredicate() &&
01722           isa<LoadInst>(LHS) &&
01723           isa<Constant>(RHS) &&
01724           RHS->getType()->isPointerTy() &&
01725           cast<Constant>(RHS)->isNullValue()) {
01726         LoadInst* LI = cast<LoadInst>(LHS);
01727         if (isValidAssumeForContext(II, LI, DT)) {
01728           MDNode *MD = MDNode::get(II->getContext(), None);
01729           LI->setMetadata(LLVMContext::MD_nonnull, MD);
01730           return EraseInstFromFunction(*II);
01731         }
01732       }
01733       // TODO: apply nonnull return attributes to calls and invokes
01734       // TODO: apply range metadata for range check patterns?
01735     }
01736     // If there is a dominating assume with the same condition as this one,
01737     // then this one is redundant, and should be removed.
01738     APInt KnownZero(1, 0), KnownOne(1, 0);
01739     computeKnownBits(IIOperand, KnownZero, KnownOne, 0, II);
01740     if (KnownOne.isAllOnesValue())
01741       return EraseInstFromFunction(*II);
01742 
01743     break;
01744   }
01745   case Intrinsic::experimental_gc_relocate: {
01746     // Translate facts known about a pointer before relocating into
01747     // facts about the relocate value, while being careful to
01748     // preserve relocation semantics.
01749     Value *DerivedPtr = cast<GCRelocateInst>(II)->getDerivedPtr();
01750     auto *GCRelocateType = cast<PointerType>(II->getType());
01751 
01752     // Remove the relocation if unused, note that this check is required
01753     // to prevent the cases below from looping forever.
01754     if (II->use_empty())
01755       return EraseInstFromFunction(*II);
01756 
01757     // Undef is undef, even after relocation.
01758     // TODO: provide a hook for this in GCStrategy.  This is clearly legal for
01759     // most practical collectors, but there was discussion in the review thread
01760     // about whether it was legal for all possible collectors.
01761     if (isa<UndefValue>(DerivedPtr)) {
01762       // gc_relocate is uncasted. Use undef of gc_relocate's type to replace it.
01763       return ReplaceInstUsesWith(*II, UndefValue::get(GCRelocateType));
01764     }
01765 
01766     // The relocation of null will be null for most any collector.
01767     // TODO: provide a hook for this in GCStrategy.  There might be some weird
01768     // collector this property does not hold for.
01769     if (isa<ConstantPointerNull>(DerivedPtr)) {
01770       // gc_relocate is uncasted. Use null-pointer of gc_relocate's type to replace it.
01771       return ReplaceInstUsesWith(*II, ConstantPointerNull::get(GCRelocateType));
01772     }
01773 
01774     // isKnownNonNull -> nonnull attribute
01775     if (isKnownNonNullAt(DerivedPtr, II, DT, TLI))
01776       II->addAttribute(AttributeSet::ReturnIndex, Attribute::NonNull);
01777 
01778     // isDereferenceablePointer -> deref attribute
01779     if (isDereferenceablePointer(DerivedPtr, DL)) {
01780       if (Argument *A = dyn_cast<Argument>(DerivedPtr)) {
01781         uint64_t Bytes = A->getDereferenceableBytes();
01782         II->addDereferenceableAttr(AttributeSet::ReturnIndex, Bytes);
01783       }
01784     }
01785 
01786     // TODO: bitcast(relocate(p)) -> relocate(bitcast(p))
01787     // Canonicalize on the type from the uses to the defs
01788 
01789     // TODO: relocate((gep p, C, C2, ...)) -> gep(relocate(p), C, C2, ...)
01790   }
01791   }
01792 
01793   return visitCallSite(II);
01794 }
01795 
01796 // InvokeInst simplification
01797 //
01798 Instruction *InstCombiner::visitInvokeInst(InvokeInst &II) {
01799   return visitCallSite(&II);
01800 }
01801 
01802 /// If this cast does not affect the value passed through the varargs area, we
01803 /// can eliminate the use of the cast.
01804 static bool isSafeToEliminateVarargsCast(const CallSite CS,
01805                                          const DataLayout &DL,
01806                                          const CastInst *const CI,
01807                                          const int ix) {
01808   if (!CI->isLosslessCast())
01809     return false;
01810 
01811   // If this is a GC intrinsic, avoid munging types.  We need types for
01812   // statepoint reconstruction in SelectionDAG.
01813   // TODO: This is probably something which should be expanded to all
01814   // intrinsics since the entire point of intrinsics is that
01815   // they are understandable by the optimizer.
01816   if (isStatepoint(CS) || isGCRelocate(CS) || isGCResult(CS))
01817     return false;
01818 
01819   // The size of ByVal or InAlloca arguments is derived from the type, so we
01820   // can't change to a type with a different size.  If the size were
01821   // passed explicitly we could avoid this check.
01822   if (!CS.isByValOrInAllocaArgument(ix))
01823     return true;
01824 
01825   Type* SrcTy =
01826             cast<PointerType>(CI->getOperand(0)->getType())->getElementType();
01827   Type* DstTy = cast<PointerType>(CI->getType())->getElementType();
01828   if (!SrcTy->isSized() || !DstTy->isSized())
01829     return false;
01830   if (DL.getTypeAllocSize(SrcTy) != DL.getTypeAllocSize(DstTy))
01831     return false;
01832   return true;
01833 }
01834 
01835 Instruction *InstCombiner::tryOptimizeCall(CallInst *CI) {
01836   if (!CI->getCalledFunction()) return nullptr;
01837 
01838   auto InstCombineRAUW = [this](Instruction *From, Value *With) {
01839     ReplaceInstUsesWith(*From, With);
01840   };
01841   LibCallSimplifier Simplifier(DL, TLI, InstCombineRAUW);
01842   if (Value *With = Simplifier.optimizeCall(CI)) {
01843     ++NumSimplified;
01844     return CI->use_empty() ? CI : ReplaceInstUsesWith(*CI, With);
01845   }
01846 
01847   return nullptr;
01848 }
01849 
01850 static IntrinsicInst *FindInitTrampolineFromAlloca(Value *TrampMem) {
01851   // Strip off at most one level of pointer casts, looking for an alloca.  This
01852   // is good enough in practice and simpler than handling any number of casts.
01853   Value *Underlying = TrampMem->stripPointerCasts();
01854   if (Underlying != TrampMem &&
01855       (!Underlying->hasOneUse() || Underlying->user_back() != TrampMem))
01856     return nullptr;
01857   if (!isa<AllocaInst>(Underlying))
01858     return nullptr;
01859 
01860   IntrinsicInst *InitTrampoline = nullptr;
01861   for (User *U : TrampMem->users()) {
01862     IntrinsicInst *II = dyn_cast<IntrinsicInst>(U);
01863     if (!II)
01864       return nullptr;
01865     if (II->getIntrinsicID() == Intrinsic::init_trampoline) {
01866       if (InitTrampoline)
01867         // More than one init_trampoline writes to this value.  Give up.
01868         return nullptr;
01869       InitTrampoline = II;
01870       continue;
01871     }
01872     if (II->getIntrinsicID() == Intrinsic::adjust_trampoline)
01873       // Allow any number of calls to adjust.trampoline.
01874       continue;
01875     return nullptr;
01876   }
01877 
01878   // No call to init.trampoline found.
01879   if (!InitTrampoline)
01880     return nullptr;
01881 
01882   // Check that the alloca is being used in the expected way.
01883   if (InitTrampoline->getOperand(0) != TrampMem)
01884     return nullptr;
01885 
01886   return InitTrampoline;
01887 }
01888 
01889 static IntrinsicInst *FindInitTrampolineFromBB(IntrinsicInst *AdjustTramp,
01890                                                Value *TrampMem) {
01891   // Visit all the previous instructions in the basic block, and try to find a
01892   // init.trampoline which has a direct path to the adjust.trampoline.
01893   for (BasicBlock::iterator I = AdjustTramp->getIterator(),
01894                             E = AdjustTramp->getParent()->begin();
01895        I != E;) {
01896     Instruction *Inst = &*--I;
01897     if (IntrinsicInst *II = dyn_cast<IntrinsicInst>(I))
01898       if (II->getIntrinsicID() == Intrinsic::init_trampoline &&
01899           II->getOperand(0) == TrampMem)
01900         return II;
01901     if (Inst->mayWriteToMemory())
01902       return nullptr;
01903   }
01904   return nullptr;
01905 }
01906 
01907 // Given a call to llvm.adjust.trampoline, find and return the corresponding
01908 // call to llvm.init.trampoline if the call to the trampoline can be optimized
01909 // to a direct call to a function.  Otherwise return NULL.
01910 //
01911 static IntrinsicInst *FindInitTrampoline(Value *Callee) {
01912   Callee = Callee->stripPointerCasts();
01913   IntrinsicInst *AdjustTramp = dyn_cast<IntrinsicInst>(Callee);
01914   if (!AdjustTramp ||
01915       AdjustTramp->getIntrinsicID() != Intrinsic::adjust_trampoline)
01916     return nullptr;
01917 
01918   Value *TrampMem = AdjustTramp->getOperand(0);
01919 
01920   if (IntrinsicInst *IT = FindInitTrampolineFromAlloca(TrampMem))
01921     return IT;
01922   if (IntrinsicInst *IT = FindInitTrampolineFromBB(AdjustTramp, TrampMem))
01923     return IT;
01924   return nullptr;
01925 }
01926 
01927 /// Improvements for call and invoke instructions.
01928 Instruction *InstCombiner::visitCallSite(CallSite CS) {
01929 
01930   if (isAllocLikeFn(CS.getInstruction(), TLI))
01931     return visitAllocSite(*CS.getInstruction());
01932 
01933   bool Changed = false;
01934 
01935   // Mark any parameters that are known to be non-null with the nonnull
01936   // attribute.  This is helpful for inlining calls to functions with null
01937   // checks on their arguments.
01938   SmallVector<unsigned, 4> Indices;
01939   unsigned ArgNo = 0;
01940 
01941   for (Value *V : CS.args()) {
01942     if (V->getType()->isPointerTy() && !CS.paramHasAttr(ArgNo+1, Attribute::NonNull) &&
01943         isKnownNonNullAt(V, CS.getInstruction(), DT, TLI))
01944       Indices.push_back(ArgNo + 1);
01945     ArgNo++;
01946   }
01947 
01948   assert(ArgNo == CS.arg_size() && "sanity check");
01949 
01950   if (!Indices.empty()) {
01951     AttributeSet AS = CS.getAttributes();
01952     LLVMContext &Ctx = CS.getInstruction()->getContext();
01953     AS = AS.addAttribute(Ctx, Indices,
01954                          Attribute::get(Ctx, Attribute::NonNull));
01955     CS.setAttributes(AS);
01956     Changed = true;
01957   }
01958 
01959   // If the callee is a pointer to a function, attempt to move any casts to the
01960   // arguments of the call/invoke.
01961   Value *Callee = CS.getCalledValue();
01962   if (!isa<Function>(Callee) && transformConstExprCastCall(CS))
01963     return nullptr;
01964 
01965   if (Function *CalleeF = dyn_cast<Function>(Callee))
01966     // If the call and callee calling conventions don't match, this call must
01967     // be unreachable, as the call is undefined.
01968     if (CalleeF->getCallingConv() != CS.getCallingConv() &&
01969         // Only do this for calls to a function with a body.  A prototype may
01970         // not actually end up matching the implementation's calling conv for a
01971         // variety of reasons (e.g. it may be written in assembly).
01972         !CalleeF->isDeclaration()) {
01973       Instruction *OldCall = CS.getInstruction();
01974       new StoreInst(ConstantInt::getTrue(Callee->getContext()),
01975                 UndefValue::get(Type::getInt1PtrTy(Callee->getContext())),
01976                                   OldCall);
01977       // If OldCall does not return void then replaceAllUsesWith undef.
01978       // This allows ValueHandlers and custom metadata to adjust itself.
01979       if (!OldCall->getType()->isVoidTy())
01980         ReplaceInstUsesWith(*OldCall, UndefValue::get(OldCall->getType()));
01981       if (isa<CallInst>(OldCall))
01982         return EraseInstFromFunction(*OldCall);
01983 
01984       // We cannot remove an invoke, because it would change the CFG, just
01985       // change the callee to a null pointer.
01986       cast<InvokeInst>(OldCall)->setCalledFunction(
01987                                     Constant::getNullValue(CalleeF->getType()));
01988       return nullptr;
01989     }
01990 
01991   if (isa<ConstantPointerNull>(Callee) || isa<UndefValue>(Callee)) {
01992     // If CS does not return void then replaceAllUsesWith undef.
01993     // This allows ValueHandlers and custom metadata to adjust itself.
01994     if (!CS.getInstruction()->getType()->isVoidTy())
01995       ReplaceInstUsesWith(*CS.getInstruction(),
01996                           UndefValue::get(CS.getInstruction()->getType()));
01997 
01998     if (isa<InvokeInst>(CS.getInstruction())) {
01999       // Can't remove an invoke because we cannot change the CFG.
02000       return nullptr;
02001     }
02002 
02003     // This instruction is not reachable, just remove it.  We insert a store to
02004     // undef so that we know that this code is not reachable, despite the fact
02005     // that we can't modify the CFG here.
02006     new StoreInst(ConstantInt::getTrue(Callee->getContext()),
02007                   UndefValue::get(Type::getInt1PtrTy(Callee->getContext())),
02008                   CS.getInstruction());
02009 
02010     return EraseInstFromFunction(*CS.getInstruction());
02011   }
02012 
02013   if (IntrinsicInst *II = FindInitTrampoline(Callee))
02014     return transformCallThroughTrampoline(CS, II);
02015 
02016   PointerType *PTy = cast<PointerType>(Callee->getType());
02017   FunctionType *FTy = cast<FunctionType>(PTy->getElementType());
02018   if (FTy->isVarArg()) {
02019     int ix = FTy->getNumParams();
02020     // See if we can optimize any arguments passed through the varargs area of
02021     // the call.
02022     for (CallSite::arg_iterator I = CS.arg_begin() + FTy->getNumParams(),
02023            E = CS.arg_end(); I != E; ++I, ++ix) {
02024       CastInst *CI = dyn_cast<CastInst>(*I);
02025       if (CI && isSafeToEliminateVarargsCast(CS, DL, CI, ix)) {
02026         *I = CI->getOperand(0);
02027         Changed = true;
02028       }
02029     }
02030   }
02031 
02032   if (isa<InlineAsm>(Callee) && !CS.doesNotThrow()) {
02033     // Inline asm calls cannot throw - mark them 'nounwind'.
02034     CS.setDoesNotThrow();
02035     Changed = true;
02036   }
02037 
02038   // Try to optimize the call if possible, we require DataLayout for most of
02039   // this.  None of these calls are seen as possibly dead so go ahead and
02040   // delete the instruction now.
02041   if (CallInst *CI = dyn_cast<CallInst>(CS.getInstruction())) {
02042     Instruction *I = tryOptimizeCall(CI);
02043     // If we changed something return the result, etc. Otherwise let
02044     // the fallthrough check.
02045     if (I) return EraseInstFromFunction(*I);
02046   }
02047 
02048   return Changed ? CS.getInstruction() : nullptr;
02049 }
02050 
02051 /// If the callee is a constexpr cast of a function, attempt to move the cast to
02052 /// the arguments of the call/invoke.
02053 bool InstCombiner::transformConstExprCastCall(CallSite CS) {
02054   Function *Callee =
02055     dyn_cast<Function>(CS.getCalledValue()->stripPointerCasts());
02056   if (!Callee)
02057     return false;
02058   // The prototype of thunks are a lie, don't try to directly call such
02059   // functions.
02060   if (Callee->hasFnAttribute("thunk"))
02061     return false;
02062   Instruction *Caller = CS.getInstruction();
02063   const AttributeSet &CallerPAL = CS.getAttributes();
02064 
02065   // Okay, this is a cast from a function to a different type.  Unless doing so
02066   // would cause a type conversion of one of our arguments, change this call to
02067   // be a direct call with arguments casted to the appropriate types.
02068   //
02069   FunctionType *FT = Callee->getFunctionType();
02070   Type *OldRetTy = Caller->getType();
02071   Type *NewRetTy = FT->getReturnType();
02072 
02073   // Check to see if we are changing the return type...
02074   if (OldRetTy != NewRetTy) {
02075 
02076     if (NewRetTy->isStructTy())
02077       return false; // TODO: Handle multiple return values.
02078 
02079     if (!CastInst::isBitOrNoopPointerCastable(NewRetTy, OldRetTy, DL)) {
02080       if (Callee->isDeclaration())
02081         return false;   // Cannot transform this return value.
02082 
02083       if (!Caller->use_empty() &&
02084           // void -> non-void is handled specially
02085           !NewRetTy->isVoidTy())
02086         return false;   // Cannot transform this return value.
02087     }
02088 
02089     if (!CallerPAL.isEmpty() && !Caller->use_empty()) {
02090       AttrBuilder RAttrs(CallerPAL, AttributeSet::ReturnIndex);
02091       if (RAttrs.overlaps(AttributeFuncs::typeIncompatible(NewRetTy)))
02092         return false;   // Attribute not compatible with transformed value.
02093     }
02094 
02095     // If the callsite is an invoke instruction, and the return value is used by
02096     // a PHI node in a successor, we cannot change the return type of the call
02097     // because there is no place to put the cast instruction (without breaking
02098     // the critical edge).  Bail out in this case.
02099     if (!Caller->use_empty())
02100       if (InvokeInst *II = dyn_cast<InvokeInst>(Caller))
02101         for (User *U : II->users())
02102           if (PHINode *PN = dyn_cast<PHINode>(U))
02103             if (PN->getParent() == II->getNormalDest() ||
02104                 PN->getParent() == II->getUnwindDest())
02105               return false;
02106   }
02107 
02108   unsigned NumActualArgs = CS.arg_size();
02109   unsigned NumCommonArgs = std::min(FT->getNumParams(), NumActualArgs);
02110 
02111   // Prevent us turning:
02112   // declare void @takes_i32_inalloca(i32* inalloca)
02113   //  call void bitcast (void (i32*)* @takes_i32_inalloca to void (i32)*)(i32 0)
02114   //
02115   // into:
02116   //  call void @takes_i32_inalloca(i32* null)
02117   //
02118   //  Similarly, avoid folding away bitcasts of byval calls.
02119   if (Callee->getAttributes().hasAttrSomewhere(Attribute::InAlloca) ||
02120       Callee->getAttributes().hasAttrSomewhere(Attribute::ByVal))
02121     return false;
02122 
02123   CallSite::arg_iterator AI = CS.arg_begin();
02124   for (unsigned i = 0, e = NumCommonArgs; i != e; ++i, ++AI) {
02125     Type *ParamTy = FT->getParamType(i);
02126     Type *ActTy = (*AI)->getType();
02127 
02128     if (!CastInst::isBitOrNoopPointerCastable(ActTy, ParamTy, DL))
02129       return false;   // Cannot transform this parameter value.
02130 
02131     if (AttrBuilder(CallerPAL.getParamAttributes(i + 1), i + 1).
02132           overlaps(AttributeFuncs::typeIncompatible(ParamTy)))
02133       return false;   // Attribute not compatible with transformed value.
02134 
02135     if (CS.isInAllocaArgument(i))
02136       return false;   // Cannot transform to and from inalloca.
02137 
02138     // If the parameter is passed as a byval argument, then we have to have a
02139     // sized type and the sized type has to have the same size as the old type.
02140     if (ParamTy != ActTy &&
02141         CallerPAL.getParamAttributes(i + 1).hasAttribute(i + 1,
02142                                                          Attribute::ByVal)) {
02143       PointerType *ParamPTy = dyn_cast<PointerType>(ParamTy);
02144       if (!ParamPTy || !ParamPTy->getElementType()->isSized())
02145         return false;
02146 
02147       Type *CurElTy = ActTy->getPointerElementType();
02148       if (DL.getTypeAllocSize(CurElTy) !=
02149           DL.getTypeAllocSize(ParamPTy->getElementType()))
02150         return false;
02151     }
02152   }
02153 
02154   if (Callee->isDeclaration()) {
02155     // Do not delete arguments unless we have a function body.
02156     if (FT->getNumParams() < NumActualArgs && !FT->isVarArg())
02157       return false;
02158 
02159     // If the callee is just a declaration, don't change the varargsness of the
02160     // call.  We don't want to introduce a varargs call where one doesn't
02161     // already exist.
02162     PointerType *APTy = cast<PointerType>(CS.getCalledValue()->getType());
02163     if (FT->isVarArg()!=cast<FunctionType>(APTy->getElementType())->isVarArg())
02164       return false;
02165 
02166     // If both the callee and the cast type are varargs, we still have to make
02167     // sure the number of fixed parameters are the same or we have the same
02168     // ABI issues as if we introduce a varargs call.
02169     if (FT->isVarArg() &&
02170         cast<FunctionType>(APTy->getElementType())->isVarArg() &&
02171         FT->getNumParams() !=
02172         cast<FunctionType>(APTy->getElementType())->getNumParams())
02173       return false;
02174   }
02175 
02176   if (FT->getNumParams() < NumActualArgs && FT->isVarArg() &&
02177       !CallerPAL.isEmpty())
02178     // In this case we have more arguments than the new function type, but we
02179     // won't be dropping them.  Check that these extra arguments have attributes
02180     // that are compatible with being a vararg call argument.
02181     for (unsigned i = CallerPAL.getNumSlots(); i; --i) {
02182       unsigned Index = CallerPAL.getSlotIndex(i - 1);
02183       if (Index <= FT->getNumParams())
02184         break;
02185 
02186       // Check if it has an attribute that's incompatible with varargs.
02187       AttributeSet PAttrs = CallerPAL.getSlotAttributes(i - 1);
02188       if (PAttrs.hasAttribute(Index, Attribute::StructRet))
02189         return false;
02190     }
02191 
02192 
02193   // Okay, we decided that this is a safe thing to do: go ahead and start
02194   // inserting cast instructions as necessary.
02195   std::vector<Value*> Args;
02196   Args.reserve(NumActualArgs);
02197   SmallVector<AttributeSet, 8> attrVec;
02198   attrVec.reserve(NumCommonArgs);
02199 
02200   // Get any return attributes.
02201   AttrBuilder RAttrs(CallerPAL, AttributeSet::ReturnIndex);
02202 
02203   // If the return value is not being used, the type may not be compatible
02204   // with the existing attributes.  Wipe out any problematic attributes.
02205   RAttrs.remove(AttributeFuncs::typeIncompatible(NewRetTy));
02206 
02207   // Add the new return attributes.
02208   if (RAttrs.hasAttributes())
02209     attrVec.push_back(AttributeSet::get(Caller->getContext(),
02210                                         AttributeSet::ReturnIndex, RAttrs));
02211 
02212   AI = CS.arg_begin();
02213   for (unsigned i = 0; i != NumCommonArgs; ++i, ++AI) {
02214     Type *ParamTy = FT->getParamType(i);
02215 
02216     if ((*AI)->getType() == ParamTy) {
02217       Args.push_back(*AI);
02218     } else {
02219       Args.push_back(Builder->CreateBitOrPointerCast(*AI, ParamTy));
02220     }
02221 
02222     // Add any parameter attributes.
02223     AttrBuilder PAttrs(CallerPAL.getParamAttributes(i + 1), i + 1);
02224     if (PAttrs.hasAttributes())
02225       attrVec.push_back(AttributeSet::get(Caller->getContext(), i + 1,
02226                                           PAttrs));
02227   }
02228 
02229   // If the function takes more arguments than the call was taking, add them
02230   // now.
02231   for (unsigned i = NumCommonArgs; i != FT->getNumParams(); ++i)
02232     Args.push_back(Constant::getNullValue(FT->getParamType(i)));
02233 
02234   // If we are removing arguments to the function, emit an obnoxious warning.
02235   if (FT->getNumParams() < NumActualArgs) {
02236     // TODO: if (!FT->isVarArg()) this call may be unreachable. PR14722
02237     if (FT->isVarArg()) {
02238       // Add all of the arguments in their promoted form to the arg list.
02239       for (unsigned i = FT->getNumParams(); i != NumActualArgs; ++i, ++AI) {
02240         Type *PTy = getPromotedType((*AI)->getType());
02241         if (PTy != (*AI)->getType()) {
02242           // Must promote to pass through va_arg area!
02243           Instruction::CastOps opcode =
02244             CastInst::getCastOpcode(*AI, false, PTy, false);
02245           Args.push_back(Builder->CreateCast(opcode, *AI, PTy));
02246         } else {
02247           Args.push_back(*AI);
02248         }
02249 
02250         // Add any parameter attributes.
02251         AttrBuilder PAttrs(CallerPAL.getParamAttributes(i + 1), i + 1);
02252         if (PAttrs.hasAttributes())
02253           attrVec.push_back(AttributeSet::get(FT->getContext(), i + 1,
02254                                               PAttrs));
02255       }
02256     }
02257   }
02258 
02259   AttributeSet FnAttrs = CallerPAL.getFnAttributes();
02260   if (CallerPAL.hasAttributes(AttributeSet::FunctionIndex))
02261     attrVec.push_back(AttributeSet::get(Callee->getContext(), FnAttrs));
02262 
02263   if (NewRetTy->isVoidTy())
02264     Caller->setName("");   // Void type should not have a name.
02265 
02266   const AttributeSet &NewCallerPAL = AttributeSet::get(Callee->getContext(),
02267                                                        attrVec);
02268 
02269   SmallVector<OperandBundleDef, 1> OpBundles;
02270   CS.getOperandBundlesAsDefs(OpBundles);
02271 
02272   Instruction *NC;
02273   if (InvokeInst *II = dyn_cast<InvokeInst>(Caller)) {
02274     NC = Builder->CreateInvoke(Callee, II->getNormalDest(), II->getUnwindDest(),
02275                                Args, OpBundles);
02276     NC->takeName(II);
02277     cast<InvokeInst>(NC)->setCallingConv(II->getCallingConv());
02278     cast<InvokeInst>(NC)->setAttributes(NewCallerPAL);
02279   } else {
02280     CallInst *CI = cast<CallInst>(Caller);
02281     NC = Builder->CreateCall(Callee, Args, OpBundles);
02282     NC->takeName(CI);
02283     if (CI->isTailCall())
02284       cast<CallInst>(NC)->setTailCall();
02285     cast<CallInst>(NC)->setCallingConv(CI->getCallingConv());
02286     cast<CallInst>(NC)->setAttributes(NewCallerPAL);
02287   }
02288 
02289   // Insert a cast of the return type as necessary.
02290   Value *NV = NC;
02291   if (OldRetTy != NV->getType() && !Caller->use_empty()) {
02292     if (!NV->getType()->isVoidTy()) {
02293       NV = NC = CastInst::CreateBitOrPointerCast(NC, OldRetTy);
02294       NC->setDebugLoc(Caller->getDebugLoc());
02295 
02296       // If this is an invoke instruction, we should insert it after the first
02297       // non-phi, instruction in the normal successor block.
02298       if (InvokeInst *II = dyn_cast<InvokeInst>(Caller)) {
02299         BasicBlock::iterator I = II->getNormalDest()->getFirstInsertionPt();
02300         InsertNewInstBefore(NC, *I);
02301       } else {
02302         // Otherwise, it's a call, just insert cast right after the call.
02303         InsertNewInstBefore(NC, *Caller);
02304       }
02305       Worklist.AddUsersToWorkList(*Caller);
02306     } else {
02307       NV = UndefValue::get(Caller->getType());
02308     }
02309   }
02310 
02311   if (!Caller->use_empty())
02312     ReplaceInstUsesWith(*Caller, NV);
02313   else if (Caller->hasValueHandle()) {
02314     if (OldRetTy == NV->getType())
02315       ValueHandleBase::ValueIsRAUWd(Caller, NV);
02316     else
02317       // We cannot call ValueIsRAUWd with a different type, and the
02318       // actual tracked value will disappear.
02319       ValueHandleBase::ValueIsDeleted(Caller);
02320   }
02321 
02322   EraseInstFromFunction(*Caller);
02323   return true;
02324 }
02325 
02326 /// Turn a call to a function created by init_trampoline / adjust_trampoline
02327 /// intrinsic pair into a direct call to the underlying function.
02328 Instruction *
02329 InstCombiner::transformCallThroughTrampoline(CallSite CS,
02330                                              IntrinsicInst *Tramp) {
02331   Value *Callee = CS.getCalledValue();
02332   PointerType *PTy = cast<PointerType>(Callee->getType());
02333   FunctionType *FTy = cast<FunctionType>(PTy->getElementType());
02334   const AttributeSet &Attrs = CS.getAttributes();
02335 
02336   // If the call already has the 'nest' attribute somewhere then give up -
02337   // otherwise 'nest' would occur twice after splicing in the chain.
02338   if (Attrs.hasAttrSomewhere(Attribute::Nest))
02339     return nullptr;
02340 
02341   assert(Tramp &&
02342          "transformCallThroughTrampoline called with incorrect CallSite.");
02343 
02344   Function *NestF =cast<Function>(Tramp->getArgOperand(1)->stripPointerCasts());
02345   FunctionType *NestFTy = cast<FunctionType>(NestF->getValueType());
02346 
02347   const AttributeSet &NestAttrs = NestF->getAttributes();
02348   if (!NestAttrs.isEmpty()) {
02349     unsigned NestIdx = 1;
02350     Type *NestTy = nullptr;
02351     AttributeSet NestAttr;
02352 
02353     // Look for a parameter marked with the 'nest' attribute.
02354     for (FunctionType::param_iterator I = NestFTy->param_begin(),
02355          E = NestFTy->param_end(); I != E; ++NestIdx, ++I)
02356       if (NestAttrs.hasAttribute(NestIdx, Attribute::Nest)) {
02357         // Record the parameter type and any other attributes.
02358         NestTy = *I;
02359         NestAttr = NestAttrs.getParamAttributes(NestIdx);
02360         break;
02361       }
02362 
02363     if (NestTy) {
02364       Instruction *Caller = CS.getInstruction();
02365       std::vector<Value*> NewArgs;
02366       NewArgs.reserve(CS.arg_size() + 1);
02367 
02368       SmallVector<AttributeSet, 8> NewAttrs;
02369       NewAttrs.reserve(Attrs.getNumSlots() + 1);
02370 
02371       // Insert the nest argument into the call argument list, which may
02372       // mean appending it.  Likewise for attributes.
02373 
02374       // Add any result attributes.
02375       if (Attrs.hasAttributes(AttributeSet::ReturnIndex))
02376         NewAttrs.push_back(AttributeSet::get(Caller->getContext(),
02377                                              Attrs.getRetAttributes()));
02378 
02379       {
02380         unsigned Idx = 1;
02381         CallSite::arg_iterator I = CS.arg_begin(), E = CS.arg_end();
02382         do {
02383           if (Idx == NestIdx) {
02384             // Add the chain argument and attributes.
02385             Value *NestVal = Tramp->getArgOperand(2);
02386             if (NestVal->getType() != NestTy)
02387               NestVal = Builder->CreateBitCast(NestVal, NestTy, "nest");
02388             NewArgs.push_back(NestVal);
02389             NewAttrs.push_back(AttributeSet::get(Caller->getContext(),
02390                                                  NestAttr));
02391           }
02392 
02393           if (I == E)
02394             break;
02395 
02396           // Add the original argument and attributes.
02397           NewArgs.push_back(*I);
02398           AttributeSet Attr = Attrs.getParamAttributes(Idx);
02399           if (Attr.hasAttributes(Idx)) {
02400             AttrBuilder B(Attr, Idx);
02401             NewAttrs.push_back(AttributeSet::get(Caller->getContext(),
02402                                                  Idx + (Idx >= NestIdx), B));
02403           }
02404 
02405           ++Idx, ++I;
02406         } while (1);
02407       }
02408 
02409       // Add any function attributes.
02410       if (Attrs.hasAttributes(AttributeSet::FunctionIndex))
02411         NewAttrs.push_back(AttributeSet::get(FTy->getContext(),
02412                                              Attrs.getFnAttributes()));
02413 
02414       // The trampoline may have been bitcast to a bogus type (FTy).
02415       // Handle this by synthesizing a new function type, equal to FTy
02416       // with the chain parameter inserted.
02417 
02418       std::vector<Type*> NewTypes;
02419       NewTypes.reserve(FTy->getNumParams()+1);
02420 
02421       // Insert the chain's type into the list of parameter types, which may
02422       // mean appending it.
02423       {
02424         unsigned Idx = 1;
02425         FunctionType::param_iterator I = FTy->param_begin(),
02426           E = FTy->param_end();
02427 
02428         do {
02429           if (Idx == NestIdx)
02430             // Add the chain's type.
02431             NewTypes.push_back(NestTy);
02432 
02433           if (I == E)
02434             break;
02435 
02436           // Add the original type.
02437           NewTypes.push_back(*I);
02438 
02439           ++Idx, ++I;
02440         } while (1);
02441       }
02442 
02443       // Replace the trampoline call with a direct call.  Let the generic
02444       // code sort out any function type mismatches.
02445       FunctionType *NewFTy = FunctionType::get(FTy->getReturnType(), NewTypes,
02446                                                 FTy->isVarArg());
02447       Constant *NewCallee =
02448         NestF->getType() == PointerType::getUnqual(NewFTy) ?
02449         NestF : ConstantExpr::getBitCast(NestF,
02450                                          PointerType::getUnqual(NewFTy));
02451       const AttributeSet &NewPAL =
02452           AttributeSet::get(FTy->getContext(), NewAttrs);
02453 
02454       Instruction *NewCaller;
02455       if (InvokeInst *II = dyn_cast<InvokeInst>(Caller)) {
02456         NewCaller = InvokeInst::Create(NewCallee,
02457                                        II->getNormalDest(), II->getUnwindDest(),
02458                                        NewArgs);
02459         cast<InvokeInst>(NewCaller)->setCallingConv(II->getCallingConv());
02460         cast<InvokeInst>(NewCaller)->setAttributes(NewPAL);
02461       } else {
02462         NewCaller = CallInst::Create(NewCallee, NewArgs);
02463         if (cast<CallInst>(Caller)->isTailCall())
02464           cast<CallInst>(NewCaller)->setTailCall();
02465         cast<CallInst>(NewCaller)->
02466           setCallingConv(cast<CallInst>(Caller)->getCallingConv());
02467         cast<CallInst>(NewCaller)->setAttributes(NewPAL);
02468       }
02469 
02470       return NewCaller;
02471     }
02472   }
02473 
02474   // Replace the trampoline call with a direct call.  Since there is no 'nest'
02475   // parameter, there is no need to adjust the argument list.  Let the generic
02476   // code sort out any function type mismatches.
02477   Constant *NewCallee =
02478     NestF->getType() == PTy ? NestF :
02479                               ConstantExpr::getBitCast(NestF, PTy);
02480   CS.setCalledFunction(NewCallee);
02481   return CS.getInstruction();
02482 }