28 #include "llvm/ADT/StringExtras.h" 29 #include "llvm/IR/CallSite.h" 30 #include "llvm/IR/DataLayout.h" 31 #include "llvm/IR/InlineAsm.h" 32 #include "llvm/IR/Intrinsics.h" 33 #include "llvm/IR/MDBuilder.h" 34 #include "llvm/Support/ConvertUTF.h" 35 #include "llvm/Support/ScopedPrinter.h" 36 #include "llvm/Support/TargetParser.h" 39 using namespace clang;
40 using namespace CodeGen;
44 int64_t
clamp(int64_t
Value, int64_t Low, int64_t High) {
61 if (FD->
hasAttr<AsmLabelAttr>())
62 Name = getMangledName(D);
66 llvm::FunctionType *Ty =
67 cast<llvm::FunctionType>(getTypes().ConvertType(FD->
getType()));
69 return GetOrCreateLLVMFunction(Name, Ty, D,
false);
75 QualType T, llvm::IntegerType *IntType) {
78 if (V->getType()->isPointerTy())
79 return CGF.
Builder.CreatePtrToInt(V, IntType);
81 assert(V->getType() == IntType);
89 if (ResultType->isPointerTy())
90 return CGF.
Builder.CreateIntToPtr(V, ResultType);
92 assert(V->getType() == ResultType);
99 llvm::AtomicRMWInst::BinOp
Kind,
108 unsigned AddrSpace = DestPtr->getType()->getPointerAddressSpace();
110 llvm::IntegerType *IntType =
113 llvm::Type *IntPtrType = IntType->getPointerTo(AddrSpace);
119 Args[1] =
EmitToInt(CGF, Args[1], T, IntType);
122 Kind, Args[0], Args[1], llvm::AtomicOrdering::SequentiallyConsistent);
133 Address, llvm::PointerType::getUnqual(Val->getType()),
"cast");
149 llvm::AtomicRMWInst::BinOp
Kind,
158 llvm::AtomicRMWInst::BinOp
Kind,
160 Instruction::BinaryOps Op,
161 bool Invert =
false) {
169 unsigned AddrSpace = DestPtr->getType()->getPointerAddressSpace();
171 llvm::IntegerType *IntType =
174 llvm::Type *IntPtrType = IntType->getPointerTo(AddrSpace);
179 Args[1] =
EmitToInt(CGF, Args[1], T, IntType);
183 Kind, Args[0], Args[1], llvm::AtomicOrdering::SequentiallyConsistent);
184 Result = CGF.
Builder.CreateBinOp(Op, Result, Args[1]);
186 Result = CGF.
Builder.CreateBinOp(llvm::Instruction::Xor, Result,
187 llvm::ConstantInt::get(IntType, -1));
207 unsigned AddrSpace = DestPtr->getType()->getPointerAddressSpace();
209 llvm::IntegerType *IntType = llvm::IntegerType::get(
211 llvm::Type *IntPtrType = IntType->getPointerTo(AddrSpace);
217 Args[1] =
EmitToInt(CGF, Args[1], T, IntType);
221 Args[0], Args[1], Args[2], llvm::AtomicOrdering::SequentiallyConsistent,
222 llvm::AtomicOrdering::SequentiallyConsistent);
225 return CGF.
Builder.CreateZExt(CGF.
Builder.CreateExtractValue(Pair, 1),
237 unsigned IntrinsicID) {
241 return CGF.
Builder.CreateCall(F, Src0);
247 unsigned IntrinsicID) {
252 return CGF.
Builder.CreateCall(F, { Src0, Src1 });
258 unsigned IntrinsicID) {
264 return CGF.
Builder.CreateCall(F, { Src0, Src1, Src2 });
270 unsigned IntrinsicID) {
275 return CGF.
Builder.CreateCall(F, {Src0, Src1});
281 llvm::CallInst *Call = CGF.
Builder.CreateCall(F, V);
282 Call->setDoesNotAccessMemory();
292 int Width = Ty->getPrimitiveSizeInBits();
293 llvm::Type *IntTy = llvm::IntegerType::get(C, Width);
295 if (Ty->isPPC_FP128Ty()) {
305 Value *ShiftCst = llvm::ConstantInt::get(IntTy, Width);
306 V = CGF.
Builder.CreateLShr(V, ShiftCst);
310 IntTy = llvm::IntegerType::get(C, Width);
311 V = CGF.
Builder.CreateTrunc(V, IntTy);
313 Value *Zero = llvm::Constant::getNullValue(IntTy);
314 return CGF.
Builder.CreateICmpSLT(V, Zero);
318 const CallExpr *E, llvm::Constant *calleeValue) {
337 assert(X->getType() == Y->getType() &&
338 "Arguments must be the same type. (Did you forget to make sure both " 339 "arguments have the same integer width?)");
343 Carry = CGF.
Builder.CreateExtractValue(Tmp, 1);
344 return CGF.
Builder.CreateExtractValue(Tmp, 0);
348 unsigned IntrinsicID,
351 llvm::MDNode *RNode = MDHelper.createRange(APInt(32, low), APInt(32, high));
353 llvm::Instruction *Call = CGF.
Builder.CreateCall(F);
354 Call->setMetadata(llvm::LLVMContext::MD_range, RNode);
359 struct WidthAndSignedness {
365 static WidthAndSignedness
368 assert(Type->
isIntegerType() &&
"Given type is not an integer.");
371 return {Width, Signed};
377 static struct WidthAndSignedness
379 assert(Types.size() > 0 &&
"Empty list of types.");
383 for (
const auto &
Type : Types) {
384 Signed |=
Type.Signed;
392 for (
const auto &
Type : Types) {
393 unsigned MinWidth =
Type.Width + (Signed && !
Type.Signed);
394 if (Width < MinWidth) {
399 return {Width, Signed};
404 if (ArgValue->getType() != DestType)
406 Builder.CreateBitCast(ArgValue, DestType, ArgValue->getName().data());
408 Intrinsic::ID inst = IsStart ? Intrinsic::vastart : Intrinsic::vaend;
409 return Builder.CreateCall(CGM.getIntrinsic(inst), ArgValue);
418 return From == To || (From == 0 && To == 1) || (From == 3 && To == 2);
423 return ConstantInt::get(ResType, (Type & 2) ? 0 : -1,
true);
427 CodeGenFunction::evaluateOrEmitBuiltinObjectSize(
const Expr *E,
unsigned Type,
428 llvm::IntegerType *ResType,
432 return emitBuiltinObjectSize(E,
Type, ResType, EmittedE);
433 return ConstantInt::get(ResType, ObjectSize,
true);
446 CodeGenFunction::emitBuiltinObjectSize(
const Expr *E,
unsigned Type,
447 llvm::IntegerType *ResType,
453 auto *PS = D->getDecl()->
getAttr<PassObjectSizeAttr>();
454 if (Param !=
nullptr && PS !=
nullptr &&
456 auto Iter = SizeArguments.find(Param);
457 assert(Iter != SizeArguments.end());
460 auto DIter = LocalDeclMap.find(D);
461 assert(DIter != LocalDeclMap.end());
463 return EmitLoadOfScalar(DIter->second,
false,
474 Value *Ptr = EmittedE ? EmittedE : EmitScalarExpr(E);
475 assert(Ptr->getType()->isPointerTy() &&
476 "Non-pointer passed to __builtin_object_size?");
478 Value *F = CGM.getIntrinsic(Intrinsic::objectsize, {ResType, Ptr->getType()});
481 Value *Min = Builder.getInt1((
Type & 2) != 0);
483 Value *NullIsUnknown = Builder.getTrue();
484 return Builder.CreateCall(F, {Ptr, Min, NullIsUnknown});
490 enum ActionKind : uint8_t { TestOnly, Complement, Reset, Set };
491 enum InterlockingKind : uint8_t {
500 InterlockingKind Interlocking;
503 static BitTest decodeBitTestBuiltin(
unsigned BuiltinID);
507 BitTest BitTest::decodeBitTestBuiltin(
unsigned BuiltinID) {
510 case Builtin::BI_bittest:
511 return {TestOnly, Unlocked,
false};
512 case Builtin::BI_bittestandcomplement:
513 return {Complement, Unlocked,
false};
514 case Builtin::BI_bittestandreset:
515 return {Reset, Unlocked,
false};
516 case Builtin::BI_bittestandset:
517 return {Set, Unlocked,
false};
518 case Builtin::BI_interlockedbittestandreset:
519 return {Reset, Sequential,
false};
520 case Builtin::BI_interlockedbittestandset:
521 return {Set, Sequential,
false};
524 case Builtin::BI_bittest64:
525 return {TestOnly, Unlocked,
true};
526 case Builtin::BI_bittestandcomplement64:
527 return {Complement, Unlocked,
true};
528 case Builtin::BI_bittestandreset64:
529 return {Reset, Unlocked,
true};
530 case Builtin::BI_bittestandset64:
531 return {Set, Unlocked,
true};
532 case Builtin::BI_interlockedbittestandreset64:
533 return {Reset, Sequential,
true};
534 case Builtin::BI_interlockedbittestandset64:
535 return {Set, Sequential,
true};
538 case Builtin::BI_interlockedbittestandset_acq:
539 return {Set, Acquire,
false};
540 case Builtin::BI_interlockedbittestandset_rel:
541 return {Set, Release,
false};
542 case Builtin::BI_interlockedbittestandset_nf:
543 return {Set, NoFence,
false};
544 case Builtin::BI_interlockedbittestandreset_acq:
545 return {Reset, Acquire,
false};
546 case Builtin::BI_interlockedbittestandreset_rel:
547 return {Reset, Release,
false};
548 case Builtin::BI_interlockedbittestandreset_nf:
549 return {Reset, NoFence,
false};
551 llvm_unreachable(
"expected only bittest intrinsics");
556 case BitTest::TestOnly:
return '\0';
557 case BitTest::Complement:
return 'c';
558 case BitTest::Reset:
return 'r';
559 case BitTest::Set:
return 's';
561 llvm_unreachable(
"invalid action");
569 char SizeSuffix = BT.Is64Bit ?
'q' :
'l';
573 raw_svector_ostream AsmOS(Asm);
574 if (BT.Interlocking != BitTest::Unlocked)
579 AsmOS << SizeSuffix <<
" $2, ($1)\n\tsetc ${0:b}";
582 std::string Constraints =
"=r,r,r,~{cc},~{flags},~{fpsr}";
583 llvm::IntegerType *IntType = llvm::IntegerType::get(
586 llvm::Type *IntPtrType = IntType->getPointerTo();
587 llvm::FunctionType *FTy =
588 llvm::FunctionType::get(CGF.
Int8Ty, {IntPtrType, IntType},
false);
590 llvm::InlineAsm *IA =
591 llvm::InlineAsm::get(FTy, Asm, Constraints,
true);
592 return CGF.
Builder.CreateCall(IA, {BitBase, BitPos});
595 static llvm::AtomicOrdering
598 case BitTest::Unlocked:
return llvm::AtomicOrdering::NotAtomic;
599 case BitTest::Sequential:
return llvm::AtomicOrdering::SequentiallyConsistent;
600 case BitTest::Acquire:
return llvm::AtomicOrdering::Acquire;
601 case BitTest::Release:
return llvm::AtomicOrdering::Release;
602 case BitTest::NoFence:
return llvm::AtomicOrdering::Monotonic;
604 llvm_unreachable(
"invalid interlocking");
617 BitTest BT = BitTest::decodeBitTestBuiltin(BuiltinID);
622 if (Arch == llvm::Triple::x86 || Arch == llvm::Triple::x86_64)
630 BitPos, llvm::ConstantInt::get(BitPos->getType(), 3),
"bittest.byteidx");
633 ByteIndex,
"bittest.byteaddr"),
637 llvm::ConstantInt::get(CGF.
Int8Ty, 0x7));
640 Value *Mask =
nullptr;
641 if (BT.Action != BitTest::TestOnly) {
642 Mask = CGF.
Builder.CreateShl(llvm::ConstantInt::get(CGF.
Int8Ty, 1), PosLow,
649 Value *OldByte =
nullptr;
650 if (Ordering != llvm::AtomicOrdering::NotAtomic) {
653 llvm::AtomicRMWInst::BinOp RMWOp = llvm::AtomicRMWInst::Or;
654 if (BT.Action == BitTest::Reset) {
655 Mask = CGF.
Builder.CreateNot(Mask);
658 OldByte = CGF.
Builder.CreateAtomicRMW(RMWOp, ByteAddr.getPointer(), Mask,
663 Value *NewByte =
nullptr;
665 case BitTest::TestOnly:
668 case BitTest::Complement:
669 NewByte = CGF.
Builder.CreateXor(OldByte, Mask);
672 NewByte = CGF.
Builder.CreateAnd(OldByte, CGF.
Builder.CreateNot(Mask));
675 NewByte = CGF.
Builder.CreateOr(OldByte, Mask);
684 Value *ShiftedByte = CGF.
Builder.CreateLShr(OldByte, PosLow,
"bittest.shr");
686 ShiftedByte, llvm::ConstantInt::get(CGF.
Int8Ty, 1),
"bittest.res");
705 bool IsVarArg =
false;
706 if (SJKind == MSVCSetJmpKind::_setjmp3) {
709 Arg1 = llvm::ConstantInt::get(CGF.
IntTy, 0);
712 Name = SJKind == MSVCSetJmpKind::_setjmp ?
"_setjmp" :
"_setjmpex";
715 llvm::ConstantInt::get(CGF.
Int32Ty, 0));
720 llvm::AttributeList ReturnsTwiceAttr = llvm::AttributeList::get(
722 llvm::Attribute::ReturnsTwice);
724 llvm::FunctionType::get(CGF.
IntTy, ArgTypes, IsVarArg), Name,
725 ReturnsTwiceAttr,
true);
731 CS.setAttributes(ReturnsTwiceAttr);
741 _InterlockedDecrement,
742 _InterlockedExchange,
743 _InterlockedExchangeAdd,
744 _InterlockedExchangeSub,
745 _InterlockedIncrement,
754 case MSVCIntrin::_BitScanForward:
755 case MSVCIntrin::_BitScanReverse: {
760 EmitScalarExpr(E->
getArg(0))->getType()->getPointerElementType();
763 Value *ArgZero = llvm::Constant::getNullValue(ArgType);
764 Value *ResZero = llvm::Constant::getNullValue(ResultType);
765 Value *ResOne = llvm::ConstantInt::get(ResultType, 1);
767 BasicBlock *
Begin = Builder.GetInsertBlock();
768 BasicBlock *
End = createBasicBlock(
"bitscan_end", this->CurFn);
769 Builder.SetInsertPoint(End);
770 PHINode *Result = Builder.CreatePHI(ResultType, 2,
"bitscan_result");
772 Builder.SetInsertPoint(Begin);
773 Value *IsZero = Builder.CreateICmpEQ(ArgValue, ArgZero);
774 BasicBlock *NotZero = createBasicBlock(
"bitscan_not_zero", this->CurFn);
775 Builder.CreateCondBr(IsZero, End, NotZero);
776 Result->addIncoming(ResZero, Begin);
778 Builder.SetInsertPoint(NotZero);
779 Address IndexAddress = EmitPointerWithAlignment(E->
getArg(0));
781 if (BuiltinID == MSVCIntrin::_BitScanForward) {
782 Value *F = CGM.getIntrinsic(Intrinsic::cttz, ArgType);
783 Value *ZeroCount = Builder.CreateCall(F, {ArgValue, Builder.getTrue()});
784 ZeroCount = Builder.CreateIntCast(ZeroCount, IndexType,
false);
785 Builder.CreateStore(ZeroCount, IndexAddress,
false);
787 unsigned ArgWidth = cast<llvm::IntegerType>(ArgType)->getBitWidth();
788 Value *ArgTypeLastIndex = llvm::ConstantInt::get(IndexType, ArgWidth - 1);
790 Value *F = CGM.getIntrinsic(Intrinsic::ctlz, ArgType);
791 Value *ZeroCount = Builder.CreateCall(F, {ArgValue, Builder.getTrue()});
792 ZeroCount = Builder.CreateIntCast(ZeroCount, IndexType,
false);
793 Value *Index = Builder.CreateNSWSub(ArgTypeLastIndex, ZeroCount);
794 Builder.CreateStore(Index, IndexAddress,
false);
796 Builder.CreateBr(End);
797 Result->addIncoming(ResOne, NotZero);
799 Builder.SetInsertPoint(End);
802 case MSVCIntrin::_InterlockedAnd:
804 case MSVCIntrin::_InterlockedExchange:
806 case MSVCIntrin::_InterlockedExchangeAdd:
808 case MSVCIntrin::_InterlockedExchangeSub:
810 case MSVCIntrin::_InterlockedOr:
812 case MSVCIntrin::_InterlockedXor:
815 case MSVCIntrin::_InterlockedDecrement: {
817 AtomicRMWInst *RMWI = Builder.CreateAtomicRMW(
819 EmitScalarExpr(E->
getArg(0)),
820 ConstantInt::get(IntTy, 1),
821 llvm::AtomicOrdering::SequentiallyConsistent);
822 return Builder.CreateSub(RMWI, ConstantInt::get(IntTy, 1));
824 case MSVCIntrin::_InterlockedIncrement: {
826 AtomicRMWInst *RMWI = Builder.CreateAtomicRMW(
828 EmitScalarExpr(E->
getArg(0)),
829 ConstantInt::get(IntTy, 1),
830 llvm::AtomicOrdering::SequentiallyConsistent);
831 return Builder.CreateAdd(RMWI, ConstantInt::get(IntTy, 1));
834 case MSVCIntrin::__fastfail: {
838 llvm::Triple::ArchType ISA = getTarget().getTriple().getArch();
839 StringRef Asm, Constraints;
842 ErrorUnsupported(E,
"__fastfail call for this architecture");
844 case llvm::Triple::x86:
845 case llvm::Triple::x86_64:
847 Constraints =
"{cx}";
849 case llvm::Triple::thumb:
851 Constraints =
"{r0}";
854 llvm::FunctionType *FTy = llvm::FunctionType::get(VoidTy, {Int32Ty},
false);
855 llvm::InlineAsm *IA =
856 llvm::InlineAsm::get(FTy, Asm, Constraints,
true);
857 llvm::AttributeList NoReturnAttr = llvm::AttributeList::get(
858 getLLVMContext(), llvm::AttributeList::FunctionIndex,
859 llvm::Attribute::NoReturn);
860 CallSite CS = Builder.CreateCall(IA, EmitScalarExpr(E->
getArg(0)));
861 CS.setAttributes(NoReturnAttr);
862 return CS.getInstruction();
865 llvm_unreachable(
"Incorrect MSVC intrinsic!");
871 CallObjCArcUse(
llvm::Value *
object) : object(
object) {}
882 assert((Kind == BCK_CLZPassedZero || Kind == BCK_CTZPassedZero)
883 &&
"Unsupported builtin check kind");
885 Value *ArgValue = EmitScalarExpr(E);
886 if (!SanOpts.has(SanitizerKind::Builtin) || !getTarget().isCLZForZeroUndef())
890 Value *Cond = Builder.CreateICmpNE(
891 ArgValue, llvm::Constant::getNullValue(ArgValue->getType()));
892 EmitCheck(std::make_pair(Cond, SanitizerKind::Builtin),
893 SanitizerHandler::InvalidBuiltin,
895 llvm::ConstantInt::get(Builder.getInt8Ty(),
Kind)},
913 raw_svector_ostream OS(Name);
914 OS <<
"__os_log_helper";
918 for (
const auto &Item : Layout.
Items)
919 OS <<
"_" << int(Item.getSizeByte()) <<
"_" 920 <<
int(Item.getDescriptorByte());
923 if (llvm::Function *F = CGM.getModule().getFunction(Name))
930 for (
unsigned int I = 0, E = Layout.
Items.size(); I < E; ++I) {
931 char Size = Layout.
Items[I].getSizeByte();
937 &Ctx.
Idents.
get(std::string(
"arg") + llvm::to_string(I)),
942 for (
auto &
P : Params)
949 CGM.getTypes().arrangeBuiltinFunctionDeclaration(Ctx.
VoidTy, Args);
950 llvm::FunctionType *FuncTy = CGM.getTypes().GetFunctionType(FI);
952 FuncTy, llvm::GlobalValue::LinkOnceODRLinkage, Name, &CGM.getModule());
954 CGM.SetLLVMFunctionAttributes(
nullptr, FI, Fn);
955 CGM.SetLLVMFunctionAttributesForDefinition(
nullptr, Fn);
958 if (CGM.getCodeGenOpts().OptimizeSize == 2)
959 Fn->addFnAttr(llvm::Attribute::NoInline);
967 StartFunction(FD, Ctx.
VoidTy, Fn, FI, Args);
973 Address BufAddr(Builder.CreateLoad(GetAddrOfLocalVar(&Params[0]),
"buf"),
976 Builder.CreateConstByteGEP(BufAddr, Offset++,
"summary"));
978 Builder.CreateConstByteGEP(BufAddr, Offset++,
"numArgs"));
981 for (
const auto &Item : Layout.
Items) {
983 Builder.getInt8(Item.getDescriptorByte()),
984 Builder.CreateConstByteGEP(BufAddr, Offset++,
"argDescriptor"));
986 Builder.getInt8(Item.getSizeByte()),
987 Builder.CreateConstByteGEP(BufAddr, Offset++,
"argSize"));
993 Address Arg = GetAddrOfLocalVar(&Params[I]);
994 Address Addr = Builder.CreateConstByteGEP(BufAddr, Offset,
"argData");
995 Addr = Builder.CreateBitCast(Addr, Arg.
getPointer()->getType(),
997 Builder.CreateStore(Builder.CreateLoad(Arg), Addr);
1009 "__builtin_os_log_format takes at least 2 arguments");
1020 for (
const auto &Item : Layout.
Items) {
1021 int Size = Item.getSizeByte();
1027 if (
const Expr *TheExpr = Item.getExpr()) {
1028 ArgVal = EmitScalarExpr(TheExpr,
false);
1031 if (TheExpr->getType()->isObjCRetainableType()) {
1032 assert(getEvaluationKind(TheExpr->getType()) ==
TEK_Scalar &&
1033 "Only scalar can be a ObjC retainable type");
1036 if (!isa<Constant>(ArgVal))
1037 RetainableOperands.push_back(ArgVal);
1040 ArgVal = Builder.getInt32(Item.getConstValue().getQuantity());
1043 unsigned ArgValSize =
1044 CGM.getDataLayout().getTypeSizeInBits(ArgVal->getType());
1045 llvm::IntegerType *IntTy = llvm::Type::getIntNTy(getLLVMContext(),
1047 ArgVal = Builder.CreateBitOrPointerCast(ArgVal, IntTy);
1050 ArgVal = Builder.CreateZExtOrBitCast(ArgVal, ConvertType(ArgTy));
1055 CGM.getTypes().arrangeBuiltinFunctionCall(Ctx.VoidTy, Args);
1066 if (!RetainableOperands.empty() && getLangOpts().ObjCAutoRefCount &&
1067 CGM.getCodeGenOpts().OptimizationLevel != 0)
1069 pushFullExprCleanup<CallObjCArcUse>(getARCCleanupKind(), Object);
1076 WidthAndSignedness Op1Info,
1077 WidthAndSignedness Op2Info,
1078 WidthAndSignedness ResultInfo) {
1079 return BuiltinID == Builtin::BI__builtin_mul_overflow &&
1080 Op1Info.Width == Op2Info.Width && Op1Info.Width >= ResultInfo.Width &&
1081 Op1Info.Signed != Op2Info.Signed;
1088 WidthAndSignedness Op1Info,
const clang::Expr *Op2,
1089 WidthAndSignedness Op2Info,
1091 WidthAndSignedness ResultInfo) {
1093 Op2Info, ResultInfo) &&
1094 "Not a mixed-sign multipliction we can specialize");
1097 const clang::Expr *SignedOp = Op1Info.Signed ? Op1 : Op2;
1098 const clang::Expr *UnsignedOp = Op1Info.Signed ? Op2 : Op1;
1103 llvm::Value *Zero = llvm::Constant::getNullValue(OpTy);
1111 CGF.
Builder.CreateSelect(IsNegative, AbsOfNegative, Signed);
1117 Unsigned, UnsignedOverflow);
1120 if (ResultInfo.Signed) {
1123 auto IntMax = llvm::APInt::getSignedMaxValue(ResultInfo.Width)
1124 .zextOrSelf(Op1Info.Width);
1126 CGF.
Builder.CreateAdd(llvm::ConstantInt::get(OpTy, IntMax),
1127 CGF.
Builder.CreateZExt(IsNegative, OpTy));
1129 CGF.
Builder.CreateICmpUGT(UnsignedResult, MaxResult);
1130 Overflow = CGF.
Builder.CreateOr(UnsignedOverflow, SignedOverflow);
1135 CGF.
Builder.CreateSelect(IsNegative, NegativeResult, UnsignedResult);
1136 Result = CGF.
Builder.CreateTrunc(SignedResult, ResTy);
1140 IsNegative, CGF.
Builder.CreateIsNotNull(UnsignedResult));
1141 Overflow = CGF.
Builder.CreateOr(UnsignedOverflow, Underflow);
1142 if (ResultInfo.Width < Op1Info.Width) {
1144 llvm::APInt::getMaxValue(ResultInfo.Width).zext(Op1Info.Width);
1146 UnsignedResult, llvm::ConstantInt::get(OpTy, IntMax));
1147 Overflow = CGF.
Builder.CreateOr(Overflow, TruncOverflow);
1151 Result = CGF.
Builder.CreateSelect(
1152 IsNegative, CGF.
Builder.CreateNeg(UnsignedResult), UnsignedResult);
1154 Result = CGF.
Builder.CreateTrunc(Result, ResTy);
1156 assert(Overflow && Result &&
"Missing overflow or result");
1173 std::string Pad = std::string(Lvl * 4,
' ');
1179 static llvm::DenseMap<QualType, const char *> Types;
1180 if (Types.empty()) {
1181 Types[Context.CharTy] =
"%c";
1182 Types[Context.BoolTy] =
"%d";
1183 Types[Context.SignedCharTy] =
"%hhd";
1184 Types[Context.UnsignedCharTy] =
"%hhu";
1185 Types[Context.IntTy] =
"%d";
1186 Types[Context.UnsignedIntTy] =
"%u";
1187 Types[Context.LongTy] =
"%ld";
1188 Types[Context.UnsignedLongTy] =
"%lu";
1189 Types[Context.LongLongTy] =
"%lld";
1190 Types[Context.UnsignedLongLongTy] =
"%llu";
1191 Types[Context.ShortTy] =
"%hd";
1192 Types[Context.UnsignedShortTy] =
"%hu";
1193 Types[Context.VoidPtrTy] =
"%p";
1194 Types[Context.FloatTy] =
"%f";
1195 Types[Context.DoubleTy] =
"%f";
1196 Types[Context.LongDoubleTy] =
"%Lf";
1197 Types[Context.getPointerType(Context.CharTy)] =
"%s";
1198 Types[Context.getPointerType(Context.getConstType(Context.CharTy))] =
"%s";
1201 for (
const auto *FD : RD->
fields()) {
1205 Value *FieldPtr = RecordPtr;
1207 FieldPtr = CGF.
Builder.CreatePointerCast(
1208 FieldPtr, CGF.
ConvertType(Context.getPointerType(FD->getType())));
1211 FD->getFieldIndex());
1213 GString = CGF.
Builder.CreateGlobalStringPtr(
1215 .concat(FD->getType().getAsString())
1216 .concat(llvm::Twine(
' '))
1217 .concat(FD->getNameAsString())
1220 Value *TmpRes = CGF.
Builder.CreateCall(Func, {GString});
1221 Res = CGF.
Builder.CreateAdd(Res, TmpRes);
1224 FD->getType().getUnqualifiedType().getCanonicalType();
1229 dumpRecord(CGF, CanonicalType, FieldPtr, Align, Func, Lvl + 1);
1230 Res = CGF.
Builder.CreateAdd(TmpRes, Res);
1235 llvm::Twine Format = Types.find(CanonicalType) == Types.end()
1236 ? Types[Context.VoidPtrTy]
1237 : Types[CanonicalType];
1243 GString = CGF.
Builder.CreateGlobalStringPtr(
1244 Format.concat(llvm::Twine(
'\n')).str());
1245 TmpRes = CGF.
Builder.CreateCall(Func, {GString, FieldPtr});
1246 Res = CGF.
Builder.CreateAdd(Res, TmpRes);
1249 GString = CGF.
Builder.CreateGlobalStringPtr(Pad +
"}\n");
1250 Value *TmpRes = CGF.
Builder.CreateCall(Func, {GString});
1251 Res = CGF.
Builder.CreateAdd(Res, TmpRes);
1256 unsigned BuiltinID,
const CallExpr *E,
1263 return RValue::get(llvm::ConstantInt::get(getLLVMContext(),
1266 return RValue::get(llvm::ConstantFP::get(getLLVMContext(),
1275 if (FD->
hasAttr<ConstAttr>()) {
1276 switch (BuiltinID) {
1277 case Builtin::BIceil:
1278 case Builtin::BIceilf:
1279 case Builtin::BIceill:
1280 case Builtin::BI__builtin_ceil:
1281 case Builtin::BI__builtin_ceilf:
1282 case Builtin::BI__builtin_ceill:
1285 case Builtin::BIcopysign:
1286 case Builtin::BIcopysignf:
1287 case Builtin::BIcopysignl:
1288 case Builtin::BI__builtin_copysign:
1289 case Builtin::BI__builtin_copysignf:
1290 case Builtin::BI__builtin_copysignl:
1291 case Builtin::BI__builtin_copysignf128:
1294 case Builtin::BIcos:
1295 case Builtin::BIcosf:
1296 case Builtin::BIcosl:
1297 case Builtin::BI__builtin_cos:
1298 case Builtin::BI__builtin_cosf:
1299 case Builtin::BI__builtin_cosl:
1302 case Builtin::BIexp:
1303 case Builtin::BIexpf:
1304 case Builtin::BIexpl:
1305 case Builtin::BI__builtin_exp:
1306 case Builtin::BI__builtin_expf:
1307 case Builtin::BI__builtin_expl:
1310 case Builtin::BIexp2:
1311 case Builtin::BIexp2f:
1312 case Builtin::BIexp2l:
1313 case Builtin::BI__builtin_exp2:
1314 case Builtin::BI__builtin_exp2f:
1315 case Builtin::BI__builtin_exp2l:
1318 case Builtin::BIfabs:
1319 case Builtin::BIfabsf:
1320 case Builtin::BIfabsl:
1321 case Builtin::BI__builtin_fabs:
1322 case Builtin::BI__builtin_fabsf:
1323 case Builtin::BI__builtin_fabsl:
1324 case Builtin::BI__builtin_fabsf128:
1327 case Builtin::BIfloor:
1328 case Builtin::BIfloorf:
1329 case Builtin::BIfloorl:
1330 case Builtin::BI__builtin_floor:
1331 case Builtin::BI__builtin_floorf:
1332 case Builtin::BI__builtin_floorl:
1335 case Builtin::BIfma:
1336 case Builtin::BIfmaf:
1337 case Builtin::BIfmal:
1338 case Builtin::BI__builtin_fma:
1339 case Builtin::BI__builtin_fmaf:
1340 case Builtin::BI__builtin_fmal:
1343 case Builtin::BIfmax:
1344 case Builtin::BIfmaxf:
1345 case Builtin::BIfmaxl:
1346 case Builtin::BI__builtin_fmax:
1347 case Builtin::BI__builtin_fmaxf:
1348 case Builtin::BI__builtin_fmaxl:
1351 case Builtin::BIfmin:
1352 case Builtin::BIfminf:
1353 case Builtin::BIfminl:
1354 case Builtin::BI__builtin_fmin:
1355 case Builtin::BI__builtin_fminf:
1356 case Builtin::BI__builtin_fminl:
1361 case Builtin::BIfmod:
1362 case Builtin::BIfmodf:
1363 case Builtin::BIfmodl:
1364 case Builtin::BI__builtin_fmod:
1365 case Builtin::BI__builtin_fmodf:
1366 case Builtin::BI__builtin_fmodl: {
1369 return RValue::get(Builder.CreateFRem(Arg1, Arg2,
"fmod"));
1372 case Builtin::BIlog:
1373 case Builtin::BIlogf:
1374 case Builtin::BIlogl:
1375 case Builtin::BI__builtin_log:
1376 case Builtin::BI__builtin_logf:
1377 case Builtin::BI__builtin_logl:
1380 case Builtin::BIlog10:
1381 case Builtin::BIlog10f:
1382 case Builtin::BIlog10l:
1383 case Builtin::BI__builtin_log10:
1384 case Builtin::BI__builtin_log10f:
1385 case Builtin::BI__builtin_log10l:
1388 case Builtin::BIlog2:
1389 case Builtin::BIlog2f:
1390 case Builtin::BIlog2l:
1391 case Builtin::BI__builtin_log2:
1392 case Builtin::BI__builtin_log2f:
1393 case Builtin::BI__builtin_log2l:
1396 case Builtin::BInearbyint:
1397 case Builtin::BInearbyintf:
1398 case Builtin::BInearbyintl:
1399 case Builtin::BI__builtin_nearbyint:
1400 case Builtin::BI__builtin_nearbyintf:
1401 case Builtin::BI__builtin_nearbyintl:
1404 case Builtin::BIpow:
1405 case Builtin::BIpowf:
1406 case Builtin::BIpowl:
1407 case Builtin::BI__builtin_pow:
1408 case Builtin::BI__builtin_powf:
1409 case Builtin::BI__builtin_powl:
1412 case Builtin::BIrint:
1413 case Builtin::BIrintf:
1414 case Builtin::BIrintl:
1415 case Builtin::BI__builtin_rint:
1416 case Builtin::BI__builtin_rintf:
1417 case Builtin::BI__builtin_rintl:
1420 case Builtin::BIround:
1421 case Builtin::BIroundf:
1422 case Builtin::BIroundl:
1423 case Builtin::BI__builtin_round:
1424 case Builtin::BI__builtin_roundf:
1425 case Builtin::BI__builtin_roundl:
1428 case Builtin::BIsin:
1429 case Builtin::BIsinf:
1430 case Builtin::BIsinl:
1431 case Builtin::BI__builtin_sin:
1432 case Builtin::BI__builtin_sinf:
1433 case Builtin::BI__builtin_sinl:
1436 case Builtin::BIsqrt:
1437 case Builtin::BIsqrtf:
1438 case Builtin::BIsqrtl:
1439 case Builtin::BI__builtin_sqrt:
1440 case Builtin::BI__builtin_sqrtf:
1441 case Builtin::BI__builtin_sqrtl:
1444 case Builtin::BItrunc:
1445 case Builtin::BItruncf:
1446 case Builtin::BItruncl:
1447 case Builtin::BI__builtin_trunc:
1448 case Builtin::BI__builtin_truncf:
1449 case Builtin::BI__builtin_truncl:
1457 switch (BuiltinID) {
1459 case Builtin::BI__builtin___CFStringMakeConstantString:
1460 case Builtin::BI__builtin___NSStringMakeConstantString:
1462 case Builtin::BI__builtin_stdarg_start:
1463 case Builtin::BI__builtin_va_start:
1464 case Builtin::BI__va_start:
1465 case Builtin::BI__builtin_va_end:
1467 EmitVAStartEnd(BuiltinID == Builtin::BI__va_start
1468 ? EmitScalarExpr(E->
getArg(0))
1469 : EmitVAListRef(E->
getArg(0)).getPointer(),
1470 BuiltinID != Builtin::BI__builtin_va_end));
1471 case Builtin::BI__builtin_va_copy: {
1472 Value *DstPtr = EmitVAListRef(E->
getArg(0)).getPointer();
1473 Value *SrcPtr = EmitVAListRef(E->
getArg(1)).getPointer();
1477 DstPtr = Builder.CreateBitCast(DstPtr, Type);
1478 SrcPtr = Builder.CreateBitCast(SrcPtr, Type);
1479 return RValue::get(Builder.CreateCall(CGM.getIntrinsic(Intrinsic::vacopy),
1482 case Builtin::BI__builtin_abs:
1483 case Builtin::BI__builtin_labs:
1484 case Builtin::BI__builtin_llabs: {
1488 Value *NegOp = Builder.CreateNSWNeg(ArgValue,
"neg");
1489 Constant *Zero = llvm::Constant::getNullValue(ArgValue->getType());
1490 Value *CmpResult = Builder.CreateICmpSLT(ArgValue, Zero,
"abscond");
1491 Value *Result = Builder.CreateSelect(CmpResult, NegOp, ArgValue,
"abs");
1494 case Builtin::BI__builtin_conj:
1495 case Builtin::BI__builtin_conjf:
1496 case Builtin::BI__builtin_conjl: {
1498 Value *Real = ComplexVal.first;
1499 Value *Imag = ComplexVal.second;
1501 Imag->getType()->isFPOrFPVectorTy()
1502 ? llvm::ConstantFP::getZeroValueForNegation(Imag->getType())
1503 : llvm::Constant::getNullValue(Imag->getType());
1505 Imag = Builder.CreateFSub(Zero, Imag,
"sub");
1508 case Builtin::BI__builtin_creal:
1509 case Builtin::BI__builtin_crealf:
1510 case Builtin::BI__builtin_creall:
1511 case Builtin::BIcreal:
1512 case Builtin::BIcrealf:
1513 case Builtin::BIcreall: {
1518 case Builtin::BI__builtin_dump_struct: {
1520 CharUnits Arg0Align = EmitPointerWithAlignment(E->
getArg(0)).getAlignment();
1525 Value *RecordPtr = EmitScalarExpr(Arg0);
1526 Value *Res =
dumpRecord(*
this, Arg0Type, RecordPtr, Arg0Align, Func, 0);
1530 case Builtin::BI__builtin_cimag:
1531 case Builtin::BI__builtin_cimagf:
1532 case Builtin::BI__builtin_cimagl:
1533 case Builtin::BIcimag:
1534 case Builtin::BIcimagf:
1535 case Builtin::BIcimagl: {
1540 case Builtin::BI__builtin_ctzs:
1541 case Builtin::BI__builtin_ctz:
1542 case Builtin::BI__builtin_ctzl:
1543 case Builtin::BI__builtin_ctzll: {
1544 Value *ArgValue = EmitCheckedArgForBuiltin(E->
getArg(0), BCK_CTZPassedZero);
1547 Value *F = CGM.getIntrinsic(Intrinsic::cttz, ArgType);
1550 Value *ZeroUndef = Builder.getInt1(getTarget().isCLZForZeroUndef());
1551 Value *Result = Builder.CreateCall(F, {ArgValue, ZeroUndef});
1552 if (Result->getType() != ResultType)
1553 Result = Builder.CreateIntCast(Result, ResultType,
true,
1557 case Builtin::BI__builtin_clzs:
1558 case Builtin::BI__builtin_clz:
1559 case Builtin::BI__builtin_clzl:
1560 case Builtin::BI__builtin_clzll: {
1561 Value *ArgValue = EmitCheckedArgForBuiltin(E->
getArg(0), BCK_CLZPassedZero);
1564 Value *F = CGM.getIntrinsic(Intrinsic::ctlz, ArgType);
1567 Value *ZeroUndef = Builder.getInt1(getTarget().isCLZForZeroUndef());
1568 Value *Result = Builder.CreateCall(F, {ArgValue, ZeroUndef});
1569 if (Result->getType() != ResultType)
1570 Result = Builder.CreateIntCast(Result, ResultType,
true,
1574 case Builtin::BI__builtin_ffs:
1575 case Builtin::BI__builtin_ffsl:
1576 case Builtin::BI__builtin_ffsll: {
1581 Value *F = CGM.getIntrinsic(Intrinsic::cttz, ArgType);
1585 Builder.CreateAdd(Builder.CreateCall(F, {ArgValue, Builder.getTrue()}),
1586 llvm::ConstantInt::get(ArgType, 1));
1587 Value *Zero = llvm::Constant::getNullValue(ArgType);
1588 Value *IsZero = Builder.CreateICmpEQ(ArgValue, Zero,
"iszero");
1589 Value *Result = Builder.CreateSelect(IsZero, Zero, Tmp,
"ffs");
1590 if (Result->getType() != ResultType)
1591 Result = Builder.CreateIntCast(Result, ResultType,
true,
1595 case Builtin::BI__builtin_parity:
1596 case Builtin::BI__builtin_parityl:
1597 case Builtin::BI__builtin_parityll: {
1602 Value *F = CGM.getIntrinsic(Intrinsic::ctpop, ArgType);
1605 Value *Tmp = Builder.CreateCall(F, ArgValue);
1606 Value *Result = Builder.CreateAnd(Tmp, llvm::ConstantInt::get(ArgType, 1));
1607 if (Result->getType() != ResultType)
1608 Result = Builder.CreateIntCast(Result, ResultType,
true,
1612 case Builtin::BI__popcnt16:
1613 case Builtin::BI__popcnt:
1614 case Builtin::BI__popcnt64:
1615 case Builtin::BI__builtin_popcount:
1616 case Builtin::BI__builtin_popcountl:
1617 case Builtin::BI__builtin_popcountll: {
1621 Value *F = CGM.getIntrinsic(Intrinsic::ctpop, ArgType);
1624 Value *Result = Builder.CreateCall(F, ArgValue);
1625 if (Result->getType() != ResultType)
1626 Result = Builder.CreateIntCast(Result, ResultType,
true,
1630 case Builtin::BI_rotr8:
1631 case Builtin::BI_rotr16:
1632 case Builtin::BI_rotr:
1633 case Builtin::BI_lrotr:
1634 case Builtin::BI_rotr64: {
1639 Shift = Builder.CreateIntCast(Shift, ArgType,
false);
1640 unsigned ArgWidth = ArgType->getIntegerBitWidth();
1641 Value *Mask = llvm::ConstantInt::get(ArgType, ArgWidth - 1);
1643 Value *RightShiftAmt = Builder.CreateAnd(Shift, Mask);
1644 Value *RightShifted = Builder.CreateLShr(Val, RightShiftAmt);
1645 Value *LeftShiftAmt = Builder.CreateAnd(Builder.CreateNeg(Shift), Mask);
1646 Value *LeftShifted = Builder.CreateShl(Val, LeftShiftAmt);
1647 Value *Result = Builder.CreateOr(LeftShifted, RightShifted);
1650 case Builtin::BI_rotl8:
1651 case Builtin::BI_rotl16:
1652 case Builtin::BI_rotl:
1653 case Builtin::BI_lrotl:
1654 case Builtin::BI_rotl64: {
1659 Shift = Builder.CreateIntCast(Shift, ArgType,
false);
1660 unsigned ArgWidth = ArgType->getIntegerBitWidth();
1661 Value *Mask = llvm::ConstantInt::get(ArgType, ArgWidth - 1);
1663 Value *LeftShiftAmt = Builder.CreateAnd(Shift, Mask);
1664 Value *LeftShifted = Builder.CreateShl(Val, LeftShiftAmt);
1665 Value *RightShiftAmt = Builder.CreateAnd(Builder.CreateNeg(Shift), Mask);
1666 Value *RightShifted = Builder.CreateLShr(Val, RightShiftAmt);
1667 Value *Result = Builder.CreateOr(LeftShifted, RightShifted);
1670 case Builtin::BI__builtin_unpredictable: {
1676 case Builtin::BI__builtin_expect: {
1680 Value *ExpectedValue = EmitScalarExpr(E->
getArg(1));
1684 if (CGM.getCodeGenOpts().OptimizationLevel == 0)
1687 Value *FnExpect = CGM.getIntrinsic(Intrinsic::expect, ArgType);
1689 Builder.CreateCall(FnExpect, {ArgValue, ExpectedValue},
"expval");
1692 case Builtin::BI__builtin_assume_aligned: {
1694 Value *OffsetValue =
1697 Value *AlignmentValue = EmitScalarExpr(E->
getArg(1));
1698 ConstantInt *AlignmentCI = cast<ConstantInt>(AlignmentValue);
1699 unsigned Alignment = (unsigned) AlignmentCI->getZExtValue();
1701 EmitAlignmentAssumption(PtrValue, Alignment, OffsetValue);
1704 case Builtin::BI__assume:
1705 case Builtin::BI__builtin_assume: {
1710 Value *FnAssume = CGM.getIntrinsic(Intrinsic::assume);
1711 return RValue::get(Builder.CreateCall(FnAssume, ArgValue));
1713 case Builtin::BI__builtin_bswap16:
1714 case Builtin::BI__builtin_bswap32:
1715 case Builtin::BI__builtin_bswap64: {
1718 case Builtin::BI__builtin_bitreverse8:
1719 case Builtin::BI__builtin_bitreverse16:
1720 case Builtin::BI__builtin_bitreverse32:
1721 case Builtin::BI__builtin_bitreverse64: {
1724 case Builtin::BI__builtin_object_size: {
1727 auto *ResType = cast<llvm::IntegerType>(ConvertType(E->
getType()));
1734 case Builtin::BI__builtin_prefetch: {
1738 llvm::ConstantInt::get(Int32Ty, 0);
1740 llvm::ConstantInt::get(Int32Ty, 3);
1741 Value *Data = llvm::ConstantInt::get(Int32Ty, 1);
1743 return RValue::get(Builder.CreateCall(F, {Address, RW, Locality, Data}));
1745 case Builtin::BI__builtin_readcyclecounter: {
1746 Value *F = CGM.getIntrinsic(Intrinsic::readcyclecounter);
1749 case Builtin::BI__builtin___clear_cache: {
1752 Value *F = CGM.getIntrinsic(Intrinsic::clear_cache);
1753 return RValue::get(Builder.CreateCall(F, {Begin, End}));
1755 case Builtin::BI__builtin_trap:
1756 return RValue::get(EmitTrapCall(Intrinsic::trap));
1757 case Builtin::BI__debugbreak:
1758 return RValue::get(EmitTrapCall(Intrinsic::debugtrap));
1759 case Builtin::BI__builtin_unreachable: {
1763 EmitBlock(createBasicBlock(
"unreachable.cont"));
1768 case Builtin::BI__builtin_powi:
1769 case Builtin::BI__builtin_powif:
1770 case Builtin::BI__builtin_powil: {
1775 return RValue::get(Builder.CreateCall(F, {Base, Exponent}));
1778 case Builtin::BI__builtin_isgreater:
1779 case Builtin::BI__builtin_isgreaterequal:
1780 case Builtin::BI__builtin_isless:
1781 case Builtin::BI__builtin_islessequal:
1782 case Builtin::BI__builtin_islessgreater:
1783 case Builtin::BI__builtin_isunordered: {
1789 switch (BuiltinID) {
1790 default: llvm_unreachable(
"Unknown ordered comparison");
1791 case Builtin::BI__builtin_isgreater:
1792 LHS = Builder.CreateFCmpOGT(LHS, RHS,
"cmp");
1794 case Builtin::BI__builtin_isgreaterequal:
1795 LHS = Builder.CreateFCmpOGE(LHS, RHS,
"cmp");
1797 case Builtin::BI__builtin_isless:
1798 LHS = Builder.CreateFCmpOLT(LHS, RHS,
"cmp");
1800 case Builtin::BI__builtin_islessequal:
1801 LHS = Builder.CreateFCmpOLE(LHS, RHS,
"cmp");
1803 case Builtin::BI__builtin_islessgreater:
1804 LHS = Builder.CreateFCmpONE(LHS, RHS,
"cmp");
1806 case Builtin::BI__builtin_isunordered:
1807 LHS = Builder.CreateFCmpUNO(LHS, RHS,
"cmp");
1813 case Builtin::BI__builtin_isnan: {
1815 V = Builder.CreateFCmpUNO(V, V,
"cmp");
1819 case Builtin::BIfinite:
1820 case Builtin::BI__finite:
1821 case Builtin::BIfinitef:
1822 case Builtin::BI__finitef:
1823 case Builtin::BIfinitel:
1824 case Builtin::BI__finitel:
1825 case Builtin::BI__builtin_isinf:
1826 case Builtin::BI__builtin_isfinite: {
1832 Constant *Infinity = ConstantFP::getInfinity(V->getType());
1833 CmpInst::Predicate Pred = (BuiltinID == Builtin::BI__builtin_isinf)
1835 : CmpInst::FCMP_ONE;
1836 Value *FCmp = Builder.CreateFCmp(Pred, Fabs, Infinity,
"cmpinf");
1840 case Builtin::BI__builtin_isinf_sign: {
1844 Value *IsInf = Builder.CreateFCmpOEQ(
1845 AbsArg, ConstantFP::getInfinity(Arg->getType()),
"isinf");
1849 Value *Zero = Constant::getNullValue(IntTy);
1850 Value *One = ConstantInt::get(IntTy, 1);
1851 Value *NegativeOne = ConstantInt::get(IntTy, -1);
1852 Value *SignResult = Builder.CreateSelect(IsNeg, NegativeOne, One);
1853 Value *Result = Builder.CreateSelect(IsInf, SignResult, Zero);
1857 case Builtin::BI__builtin_isnormal: {
1860 Value *Eq = Builder.CreateFCmpOEQ(V, V,
"iseq");
1863 Value *IsLessThanInf =
1864 Builder.CreateFCmpULT(Abs, ConstantFP::getInfinity(V->getType()),
"isinf");
1865 APFloat Smallest = APFloat::getSmallestNormalized(
1866 getContext().getFloatTypeSemantics(E->
getArg(0)->
getType()));
1868 Builder.CreateFCmpUGE(Abs, ConstantFP::get(V->getContext(), Smallest),
1870 V = Builder.CreateAnd(Eq, IsLessThanInf,
"and");
1871 V = Builder.CreateAnd(V, IsNormal,
"and");
1875 case Builtin::BI__builtin_fpclassify: {
1880 BasicBlock *
Begin = Builder.GetInsertBlock();
1881 BasicBlock *
End = createBasicBlock(
"fpclassify_end", this->CurFn);
1882 Builder.SetInsertPoint(End);
1885 "fpclassify_result");
1888 Builder.SetInsertPoint(Begin);
1889 Value *IsZero = Builder.CreateFCmpOEQ(V, Constant::getNullValue(Ty),
1891 Value *ZeroLiteral = EmitScalarExpr(E->
getArg(4));
1892 BasicBlock *NotZero = createBasicBlock(
"fpclassify_not_zero", this->CurFn);
1893 Builder.CreateCondBr(IsZero, End, NotZero);
1894 Result->addIncoming(ZeroLiteral, Begin);
1897 Builder.SetInsertPoint(NotZero);
1898 Value *IsNan = Builder.CreateFCmpUNO(V, V,
"cmp");
1900 BasicBlock *NotNan = createBasicBlock(
"fpclassify_not_nan", this->CurFn);
1901 Builder.CreateCondBr(IsNan, End, NotNan);
1902 Result->addIncoming(NanLiteral, NotZero);
1905 Builder.SetInsertPoint(NotNan);
1908 Builder.CreateFCmpOEQ(VAbs, ConstantFP::getInfinity(V->getType()),
1911 BasicBlock *NotInf = createBasicBlock(
"fpclassify_not_inf", this->CurFn);
1912 Builder.CreateCondBr(IsInf, End, NotInf);
1913 Result->addIncoming(InfLiteral, NotNan);
1916 Builder.SetInsertPoint(NotInf);
1917 APFloat Smallest = APFloat::getSmallestNormalized(
1918 getContext().getFloatTypeSemantics(E->
getArg(5)->
getType()));
1920 Builder.CreateFCmpUGE(VAbs, ConstantFP::get(V->getContext(), Smallest),
1922 Value *NormalResult =
1923 Builder.CreateSelect(IsNormal, EmitScalarExpr(E->
getArg(2)),
1924 EmitScalarExpr(E->
getArg(3)));
1925 Builder.CreateBr(End);
1926 Result->addIncoming(NormalResult, NotInf);
1929 Builder.SetInsertPoint(End);
1933 case Builtin::BIalloca:
1934 case Builtin::BI_alloca:
1935 case Builtin::BI__builtin_alloca: {
1937 const TargetInfo &TI = getContext().getTargetInfo();
1939 unsigned SuitableAlignmentInBytes =
1941 .toCharUnitsFromBits(TI.getSuitableAlign())
1943 AllocaInst *AI = Builder.CreateAlloca(Builder.getInt8Ty(), Size);
1944 AI->setAlignment(SuitableAlignmentInBytes);
1948 case Builtin::BI__builtin_alloca_with_align: {
1950 Value *AlignmentInBitsValue = EmitScalarExpr(E->
getArg(1));
1951 auto *AlignmentInBitsCI = cast<ConstantInt>(AlignmentInBitsValue);
1952 unsigned AlignmentInBits = AlignmentInBitsCI->getZExtValue();
1953 unsigned AlignmentInBytes =
1954 CGM.getContext().toCharUnitsFromBits(AlignmentInBits).getQuantity();
1955 AllocaInst *AI = Builder.CreateAlloca(Builder.getInt8Ty(), Size);
1956 AI->setAlignment(AlignmentInBytes);
1960 case Builtin::BIbzero:
1961 case Builtin::BI__builtin_bzero: {
1966 Builder.CreateMemSet(Dest, Builder.getInt8(0), SizeVal,
false);
1969 case Builtin::BImemcpy:
1970 case Builtin::BI__builtin_memcpy: {
1978 Builder.CreateMemCpy(Dest, Src, SizeVal,
false);
1982 case Builtin::BI__builtin_char_memchr:
1983 BuiltinID = Builtin::BI__builtin_memchr;
1986 case Builtin::BI__builtin___memcpy_chk: {
1988 llvm::APSInt Size, DstSize;
1992 if (Size.ugt(DstSize))
1996 Value *SizeVal = llvm::ConstantInt::get(Builder.getContext(), Size);
1997 Builder.CreateMemCpy(Dest, Src, SizeVal,
false);
2001 case Builtin::BI__builtin_objc_memmove_collectable: {
2005 CGM.getObjCRuntime().EmitGCMemmoveCollectable(*
this,
2006 DestAddr, SrcAddr, SizeVal);
2010 case Builtin::BI__builtin___memmove_chk: {
2012 llvm::APSInt Size, DstSize;
2016 if (Size.ugt(DstSize))
2020 Value *SizeVal = llvm::ConstantInt::get(Builder.getContext(), Size);
2021 Builder.CreateMemMove(Dest, Src, SizeVal,
false);
2025 case Builtin::BImemmove:
2026 case Builtin::BI__builtin_memmove: {
2034 Builder.CreateMemMove(Dest, Src, SizeVal,
false);
2037 case Builtin::BImemset:
2038 case Builtin::BI__builtin_memset: {
2040 Value *ByteVal = Builder.CreateTrunc(EmitScalarExpr(E->
getArg(1)),
2041 Builder.getInt8Ty());
2045 Builder.CreateMemSet(Dest, ByteVal, SizeVal,
false);
2048 case Builtin::BI__builtin___memset_chk: {
2050 llvm::APSInt Size, DstSize;
2054 if (Size.ugt(DstSize))
2057 Value *ByteVal = Builder.CreateTrunc(EmitScalarExpr(E->
getArg(1)),
2058 Builder.getInt8Ty());
2059 Value *SizeVal = llvm::ConstantInt::get(Builder.getContext(), Size);
2060 Builder.CreateMemSet(Dest, ByteVal, SizeVal,
false);
2063 case Builtin::BI__builtin_wmemcmp: {
2066 if (!getTarget().getTriple().isOSMSVCRT())
2069 llvm::Type *WCharTy = ConvertType(getContext().WCharTy);
2075 BasicBlock *Entry = Builder.GetInsertBlock();
2076 BasicBlock *CmpGT = createBasicBlock(
"wmemcmp.gt");
2077 BasicBlock *CmpLT = createBasicBlock(
"wmemcmp.lt");
2078 BasicBlock *Next = createBasicBlock(
"wmemcmp.next");
2079 BasicBlock *Exit = createBasicBlock(
"wmemcmp.exit");
2080 Value *SizeEq0 = Builder.CreateICmpEQ(Size, ConstantInt::get(SizeTy, 0));
2081 Builder.CreateCondBr(SizeEq0, Exit, CmpGT);
2084 PHINode *DstPhi = Builder.CreatePHI(Dst->getType(), 2);
2085 DstPhi->addIncoming(Dst, Entry);
2086 PHINode *SrcPhi = Builder.CreatePHI(Src->getType(), 2);
2087 SrcPhi->addIncoming(Src, Entry);
2088 PHINode *SizePhi = Builder.CreatePHI(SizeTy, 2);
2089 SizePhi->addIncoming(Size, Entry);
2091 getContext().getTypeAlignInChars(getContext().WCharTy);
2092 Value *DstCh = Builder.CreateAlignedLoad(WCharTy, DstPhi, WCharAlign);
2093 Value *SrcCh = Builder.CreateAlignedLoad(WCharTy, SrcPhi, WCharAlign);
2094 Value *DstGtSrc = Builder.CreateICmpUGT(DstCh, SrcCh);
2095 Builder.CreateCondBr(DstGtSrc, Exit, CmpLT);
2098 Value *DstLtSrc = Builder.CreateICmpULT(DstCh, SrcCh);
2099 Builder.CreateCondBr(DstLtSrc, Exit, Next);
2102 Value *NextDst = Builder.CreateConstInBoundsGEP1_32(WCharTy, DstPhi, 1);
2103 Value *NextSrc = Builder.CreateConstInBoundsGEP1_32(WCharTy, SrcPhi, 1);
2104 Value *NextSize = Builder.CreateSub(SizePhi, ConstantInt::get(SizeTy, 1));
2105 Value *NextSizeEq0 =
2106 Builder.CreateICmpEQ(NextSize, ConstantInt::get(SizeTy, 0));
2107 Builder.CreateCondBr(NextSizeEq0, Exit, CmpGT);
2108 DstPhi->addIncoming(NextDst, Next);
2109 SrcPhi->addIncoming(NextSrc, Next);
2110 SizePhi->addIncoming(NextSize, Next);
2113 PHINode *Ret = Builder.CreatePHI(IntTy, 4);
2114 Ret->addIncoming(ConstantInt::get(IntTy, 0), Entry);
2115 Ret->addIncoming(ConstantInt::get(IntTy, 1), CmpGT);
2116 Ret->addIncoming(ConstantInt::get(IntTy, -1), CmpLT);
2117 Ret->addIncoming(ConstantInt::get(IntTy, 0), Next);
2120 case Builtin::BI__builtin_dwarf_cfa: {
2131 Value *F = CGM.getIntrinsic(Intrinsic::eh_dwarf_cfa);
2133 llvm::ConstantInt::get(Int32Ty, Offset)));
2135 case Builtin::BI__builtin_return_address: {
2137 getContext().UnsignedIntTy);
2138 Value *F = CGM.getIntrinsic(Intrinsic::returnaddress);
2141 case Builtin::BI_ReturnAddress: {
2142 Value *F = CGM.getIntrinsic(Intrinsic::returnaddress);
2143 return RValue::get(Builder.CreateCall(F, Builder.getInt32(0)));
2145 case Builtin::BI__builtin_frame_address: {
2147 getContext().UnsignedIntTy);
2148 Value *F = CGM.getIntrinsic(Intrinsic::frameaddress);
2151 case Builtin::BI__builtin_extract_return_addr: {
2153 Value *Result = getTargetHooks().decodeReturnAddress(*
this, Address);
2156 case Builtin::BI__builtin_frob_return_addr: {
2158 Value *Result = getTargetHooks().encodeReturnAddress(*
this, Address);
2161 case Builtin::BI__builtin_dwarf_sp_column: {
2162 llvm::IntegerType *Ty
2163 = cast<llvm::IntegerType>(ConvertType(E->
getType()));
2164 int Column = getTargetHooks().getDwarfEHStackPointer(CGM);
2166 CGM.ErrorUnsupported(E,
"__builtin_dwarf_sp_column");
2169 return RValue::get(llvm::ConstantInt::get(Ty, Column,
true));
2171 case Builtin::BI__builtin_init_dwarf_reg_size_table: {
2173 if (getTargetHooks().initDwarfEHRegSizeTable(*
this, Address))
2174 CGM.ErrorUnsupported(E,
"__builtin_init_dwarf_reg_size_table");
2177 case Builtin::BI__builtin_eh_return: {
2181 llvm::IntegerType *IntTy = cast<llvm::IntegerType>(Int->getType());
2182 assert((IntTy->getBitWidth() == 32 || IntTy->getBitWidth() == 64) &&
2183 "LLVM's __builtin_eh_return only supports 32- and 64-bit variants");
2184 Value *F = CGM.getIntrinsic(IntTy->getBitWidth() == 32
2185 ? Intrinsic::eh_return_i32
2186 : Intrinsic::eh_return_i64);
2187 Builder.CreateCall(F, {Int, Ptr});
2188 Builder.CreateUnreachable();
2191 EmitBlock(createBasicBlock(
"builtin_eh_return.cont"));
2195 case Builtin::BI__builtin_unwind_init: {
2196 Value *F = CGM.getIntrinsic(Intrinsic::eh_unwind_init);
2199 case Builtin::BI__builtin_extend_pointer: {
2212 Value *Result = Builder.CreatePtrToInt(Ptr, IntPtrTy,
"extend.cast");
2215 if (IntPtrTy->getBitWidth() == 64)
2219 if (getTargetHooks().extendPointerWithSExt())
2220 return RValue::get(Builder.CreateSExt(Result, Int64Ty,
"extend.sext"));
2222 return RValue::get(Builder.CreateZExt(Result, Int64Ty,
"extend.zext"));
2224 case Builtin::BI__builtin_setjmp: {
2230 Builder.CreateCall(CGM.getIntrinsic(Intrinsic::frameaddress),
2231 ConstantInt::get(Int32Ty, 0));
2232 Builder.CreateStore(FrameAddr, Buf);
2236 Builder.CreateCall(CGM.getIntrinsic(Intrinsic::stacksave));
2238 Builder.CreateConstInBoundsGEP(Buf, 2, getPointerSize());
2239 Builder.CreateStore(StackAddr, StackSaveSlot);
2242 Value *F = CGM.getIntrinsic(Intrinsic::eh_sjlj_setjmp);
2243 Buf = Builder.CreateBitCast(Buf, Int8PtrTy);
2246 case Builtin::BI__builtin_longjmp: {
2248 Buf = Builder.CreateBitCast(Buf, Int8PtrTy);
2251 Builder.CreateCall(CGM.getIntrinsic(Intrinsic::eh_sjlj_longjmp), Buf);
2254 Builder.CreateUnreachable();
2257 EmitBlock(createBasicBlock(
"longjmp.cont"));
2261 case Builtin::BI__sync_fetch_and_add:
2262 case Builtin::BI__sync_fetch_and_sub:
2263 case Builtin::BI__sync_fetch_and_or:
2264 case Builtin::BI__sync_fetch_and_and:
2265 case Builtin::BI__sync_fetch_and_xor:
2266 case Builtin::BI__sync_fetch_and_nand:
2267 case Builtin::BI__sync_add_and_fetch:
2268 case Builtin::BI__sync_sub_and_fetch:
2269 case Builtin::BI__sync_and_and_fetch:
2270 case Builtin::BI__sync_or_and_fetch:
2271 case Builtin::BI__sync_xor_and_fetch:
2272 case Builtin::BI__sync_nand_and_fetch:
2273 case Builtin::BI__sync_val_compare_and_swap:
2274 case Builtin::BI__sync_bool_compare_and_swap:
2275 case Builtin::BI__sync_lock_test_and_set:
2276 case Builtin::BI__sync_lock_release:
2277 case Builtin::BI__sync_swap:
2278 llvm_unreachable(
"Shouldn't make it through sema");
2279 case Builtin::BI__sync_fetch_and_add_1:
2280 case Builtin::BI__sync_fetch_and_add_2:
2281 case Builtin::BI__sync_fetch_and_add_4:
2282 case Builtin::BI__sync_fetch_and_add_8:
2283 case Builtin::BI__sync_fetch_and_add_16:
2285 case Builtin::BI__sync_fetch_and_sub_1:
2286 case Builtin::BI__sync_fetch_and_sub_2:
2287 case Builtin::BI__sync_fetch_and_sub_4:
2288 case Builtin::BI__sync_fetch_and_sub_8:
2289 case Builtin::BI__sync_fetch_and_sub_16:
2291 case Builtin::BI__sync_fetch_and_or_1:
2292 case Builtin::BI__sync_fetch_and_or_2:
2293 case Builtin::BI__sync_fetch_and_or_4:
2294 case Builtin::BI__sync_fetch_and_or_8:
2295 case Builtin::BI__sync_fetch_and_or_16:
2297 case Builtin::BI__sync_fetch_and_and_1:
2298 case Builtin::BI__sync_fetch_and_and_2:
2299 case Builtin::BI__sync_fetch_and_and_4:
2300 case Builtin::BI__sync_fetch_and_and_8:
2301 case Builtin::BI__sync_fetch_and_and_16:
2303 case Builtin::BI__sync_fetch_and_xor_1:
2304 case Builtin::BI__sync_fetch_and_xor_2:
2305 case Builtin::BI__sync_fetch_and_xor_4:
2306 case Builtin::BI__sync_fetch_and_xor_8:
2307 case Builtin::BI__sync_fetch_and_xor_16:
2309 case Builtin::BI__sync_fetch_and_nand_1:
2310 case Builtin::BI__sync_fetch_and_nand_2:
2311 case Builtin::BI__sync_fetch_and_nand_4:
2312 case Builtin::BI__sync_fetch_and_nand_8:
2313 case Builtin::BI__sync_fetch_and_nand_16:
2317 case Builtin::BI__sync_fetch_and_min:
2319 case Builtin::BI__sync_fetch_and_max:
2321 case Builtin::BI__sync_fetch_and_umin:
2323 case Builtin::BI__sync_fetch_and_umax:
2326 case Builtin::BI__sync_add_and_fetch_1:
2327 case Builtin::BI__sync_add_and_fetch_2:
2328 case Builtin::BI__sync_add_and_fetch_4:
2329 case Builtin::BI__sync_add_and_fetch_8:
2330 case Builtin::BI__sync_add_and_fetch_16:
2332 llvm::Instruction::Add);
2333 case Builtin::BI__sync_sub_and_fetch_1:
2334 case Builtin::BI__sync_sub_and_fetch_2:
2335 case Builtin::BI__sync_sub_and_fetch_4:
2336 case Builtin::BI__sync_sub_and_fetch_8:
2337 case Builtin::BI__sync_sub_and_fetch_16:
2339 llvm::Instruction::Sub);
2340 case Builtin::BI__sync_and_and_fetch_1:
2341 case Builtin::BI__sync_and_and_fetch_2:
2342 case Builtin::BI__sync_and_and_fetch_4:
2343 case Builtin::BI__sync_and_and_fetch_8:
2344 case Builtin::BI__sync_and_and_fetch_16:
2347 case Builtin::BI__sync_or_and_fetch_1:
2348 case Builtin::BI__sync_or_and_fetch_2:
2349 case Builtin::BI__sync_or_and_fetch_4:
2350 case Builtin::BI__sync_or_and_fetch_8:
2351 case Builtin::BI__sync_or_and_fetch_16:
2353 llvm::Instruction::Or);
2354 case Builtin::BI__sync_xor_and_fetch_1:
2355 case Builtin::BI__sync_xor_and_fetch_2:
2356 case Builtin::BI__sync_xor_and_fetch_4:
2357 case Builtin::BI__sync_xor_and_fetch_8:
2358 case Builtin::BI__sync_xor_and_fetch_16:
2360 llvm::Instruction::Xor);
2361 case Builtin::BI__sync_nand_and_fetch_1:
2362 case Builtin::BI__sync_nand_and_fetch_2:
2363 case Builtin::BI__sync_nand_and_fetch_4:
2364 case Builtin::BI__sync_nand_and_fetch_8:
2365 case Builtin::BI__sync_nand_and_fetch_16:
2369 case Builtin::BI__sync_val_compare_and_swap_1:
2370 case Builtin::BI__sync_val_compare_and_swap_2:
2371 case Builtin::BI__sync_val_compare_and_swap_4:
2372 case Builtin::BI__sync_val_compare_and_swap_8:
2373 case Builtin::BI__sync_val_compare_and_swap_16:
2376 case Builtin::BI__sync_bool_compare_and_swap_1:
2377 case Builtin::BI__sync_bool_compare_and_swap_2:
2378 case Builtin::BI__sync_bool_compare_and_swap_4:
2379 case Builtin::BI__sync_bool_compare_and_swap_8:
2380 case Builtin::BI__sync_bool_compare_and_swap_16:
2383 case Builtin::BI__sync_swap_1:
2384 case Builtin::BI__sync_swap_2:
2385 case Builtin::BI__sync_swap_4:
2386 case Builtin::BI__sync_swap_8:
2387 case Builtin::BI__sync_swap_16:
2390 case Builtin::BI__sync_lock_test_and_set_1:
2391 case Builtin::BI__sync_lock_test_and_set_2:
2392 case Builtin::BI__sync_lock_test_and_set_4:
2393 case Builtin::BI__sync_lock_test_and_set_8:
2394 case Builtin::BI__sync_lock_test_and_set_16:
2397 case Builtin::BI__sync_lock_release_1:
2398 case Builtin::BI__sync_lock_release_2:
2399 case Builtin::BI__sync_lock_release_4:
2400 case Builtin::BI__sync_lock_release_8:
2401 case Builtin::BI__sync_lock_release_16: {
2404 CharUnits StoreSize = getContext().getTypeSizeInChars(ElTy);
2405 llvm::Type *ITy = llvm::IntegerType::get(getLLVMContext(),
2407 Ptr = Builder.CreateBitCast(Ptr, ITy->getPointerTo());
2408 llvm::StoreInst *
Store =
2409 Builder.CreateAlignedStore(llvm::Constant::getNullValue(ITy), Ptr,
2411 Store->setAtomic(llvm::AtomicOrdering::Release);
2415 case Builtin::BI__sync_synchronize: {
2423 Builder.CreateFence(llvm::AtomicOrdering::SequentiallyConsistent);
2427 case Builtin::BI__builtin_nontemporal_load:
2429 case Builtin::BI__builtin_nontemporal_store:
2431 case Builtin::BI__c11_atomic_is_lock_free:
2432 case Builtin::BI__atomic_is_lock_free: {
2436 const char *LibCallName =
"__atomic_is_lock_free";
2439 getContext().getSizeType());
2440 if (BuiltinID == Builtin::BI__atomic_is_lock_free)
2442 getContext().VoidPtrTy);
2445 getContext().VoidPtrTy);
2447 CGM.getTypes().arrangeBuiltinFunctionCall(E->
getType(), Args);
2448 llvm::FunctionType *FTy = CGM.getTypes().GetFunctionType(FuncInfo);
2449 llvm::Constant *Func = CGM.CreateRuntimeFunction(FTy, LibCallName);
2454 case Builtin::BI__atomic_test_and_set: {
2462 unsigned AddrSpace = Ptr->getType()->getPointerAddressSpace();
2463 Ptr = Builder.CreateBitCast(Ptr, Int8Ty->getPointerTo(AddrSpace));
2464 Value *NewVal = Builder.getInt8(1);
2466 if (isa<llvm::ConstantInt>(Order)) {
2467 int ord = cast<llvm::ConstantInt>(Order)->getZExtValue();
2468 AtomicRMWInst *Result =
nullptr;
2472 Result = Builder.CreateAtomicRMW(llvm::AtomicRMWInst::Xchg, Ptr, NewVal,
2473 llvm::AtomicOrdering::Monotonic);
2477 Result = Builder.CreateAtomicRMW(llvm::AtomicRMWInst::Xchg, Ptr, NewVal,
2478 llvm::AtomicOrdering::Acquire);
2481 Result = Builder.CreateAtomicRMW(llvm::AtomicRMWInst::Xchg, Ptr, NewVal,
2482 llvm::AtomicOrdering::Release);
2486 Result = Builder.CreateAtomicRMW(llvm::AtomicRMWInst::Xchg, Ptr, NewVal,
2487 llvm::AtomicOrdering::AcquireRelease);
2490 Result = Builder.CreateAtomicRMW(
2491 llvm::AtomicRMWInst::Xchg, Ptr, NewVal,
2492 llvm::AtomicOrdering::SequentiallyConsistent);
2495 Result->setVolatile(Volatile);
2496 return RValue::get(Builder.CreateIsNotNull(Result,
"tobool"));
2499 llvm::BasicBlock *ContBB = createBasicBlock(
"atomic.continue", CurFn);
2501 llvm::BasicBlock *BBs[5] = {
2502 createBasicBlock(
"monotonic", CurFn),
2503 createBasicBlock(
"acquire", CurFn),
2504 createBasicBlock(
"release", CurFn),
2505 createBasicBlock(
"acqrel", CurFn),
2506 createBasicBlock(
"seqcst", CurFn)
2508 llvm::AtomicOrdering Orders[5] = {
2509 llvm::AtomicOrdering::Monotonic, llvm::AtomicOrdering::Acquire,
2510 llvm::AtomicOrdering::Release, llvm::AtomicOrdering::AcquireRelease,
2511 llvm::AtomicOrdering::SequentiallyConsistent};
2513 Order = Builder.CreateIntCast(Order, Builder.getInt32Ty(),
false);
2514 llvm::SwitchInst *SI = Builder.CreateSwitch(Order, BBs[0]);
2516 Builder.SetInsertPoint(ContBB);
2517 PHINode *Result = Builder.CreatePHI(Int8Ty, 5,
"was_set");
2519 for (
unsigned i = 0; i < 5; ++i) {
2520 Builder.SetInsertPoint(BBs[i]);
2521 AtomicRMWInst *RMW = Builder.CreateAtomicRMW(llvm::AtomicRMWInst::Xchg,
2522 Ptr, NewVal, Orders[i]);
2523 RMW->setVolatile(Volatile);
2524 Result->addIncoming(RMW, BBs[i]);
2525 Builder.CreateBr(ContBB);
2528 SI->addCase(Builder.getInt32(0), BBs[0]);
2529 SI->addCase(Builder.getInt32(1), BBs[1]);
2530 SI->addCase(Builder.getInt32(2), BBs[1]);
2531 SI->addCase(Builder.getInt32(3), BBs[2]);
2532 SI->addCase(Builder.getInt32(4), BBs[3]);
2533 SI->addCase(Builder.getInt32(5), BBs[4]);
2535 Builder.SetInsertPoint(ContBB);
2536 return RValue::get(Builder.CreateIsNotNull(Result,
"tobool"));
2539 case Builtin::BI__atomic_clear: {
2545 unsigned AddrSpace = Ptr.
getPointer()->getType()->getPointerAddressSpace();
2546 Ptr = Builder.CreateBitCast(Ptr, Int8Ty->getPointerTo(AddrSpace));
2547 Value *NewVal = Builder.getInt8(0);
2549 if (isa<llvm::ConstantInt>(Order)) {
2550 int ord = cast<llvm::ConstantInt>(Order)->getZExtValue();
2551 StoreInst *
Store = Builder.CreateStore(NewVal, Ptr, Volatile);
2555 Store->setOrdering(llvm::AtomicOrdering::Monotonic);
2558 Store->setOrdering(llvm::AtomicOrdering::Release);
2561 Store->setOrdering(llvm::AtomicOrdering::SequentiallyConsistent);
2567 llvm::BasicBlock *ContBB = createBasicBlock(
"atomic.continue", CurFn);
2569 llvm::BasicBlock *BBs[3] = {
2570 createBasicBlock(
"monotonic", CurFn),
2571 createBasicBlock(
"release", CurFn),
2572 createBasicBlock(
"seqcst", CurFn)
2574 llvm::AtomicOrdering Orders[3] = {
2575 llvm::AtomicOrdering::Monotonic, llvm::AtomicOrdering::Release,
2576 llvm::AtomicOrdering::SequentiallyConsistent};
2578 Order = Builder.CreateIntCast(Order, Builder.getInt32Ty(),
false);
2579 llvm::SwitchInst *SI = Builder.CreateSwitch(Order, BBs[0]);
2581 for (
unsigned i = 0; i < 3; ++i) {
2582 Builder.SetInsertPoint(BBs[i]);
2583 StoreInst *
Store = Builder.CreateStore(NewVal, Ptr, Volatile);
2584 Store->setOrdering(Orders[i]);
2585 Builder.CreateBr(ContBB);
2588 SI->addCase(Builder.getInt32(0), BBs[0]);
2589 SI->addCase(Builder.getInt32(3), BBs[1]);
2590 SI->addCase(Builder.getInt32(5), BBs[2]);
2592 Builder.SetInsertPoint(ContBB);
2596 case Builtin::BI__atomic_thread_fence:
2597 case Builtin::BI__atomic_signal_fence:
2598 case Builtin::BI__c11_atomic_thread_fence:
2599 case Builtin::BI__c11_atomic_signal_fence: {
2601 if (BuiltinID == Builtin::BI__atomic_signal_fence ||
2602 BuiltinID == Builtin::BI__c11_atomic_signal_fence)
2603 SSID = llvm::SyncScope::SingleThread;
2607 if (isa<llvm::ConstantInt>(Order)) {
2608 int ord = cast<llvm::ConstantInt>(Order)->getZExtValue();
2615 Builder.CreateFence(llvm::AtomicOrdering::Acquire, SSID);
2618 Builder.CreateFence(llvm::AtomicOrdering::Release, SSID);
2621 Builder.CreateFence(llvm::AtomicOrdering::AcquireRelease, SSID);
2624 Builder.CreateFence(llvm::AtomicOrdering::SequentiallyConsistent, SSID);
2630 llvm::BasicBlock *AcquireBB, *ReleaseBB, *AcqRelBB, *SeqCstBB;
2631 AcquireBB = createBasicBlock(
"acquire", CurFn);
2632 ReleaseBB = createBasicBlock(
"release", CurFn);
2633 AcqRelBB = createBasicBlock(
"acqrel", CurFn);
2634 SeqCstBB = createBasicBlock(
"seqcst", CurFn);
2635 llvm::BasicBlock *ContBB = createBasicBlock(
"atomic.continue", CurFn);
2637 Order = Builder.CreateIntCast(Order, Builder.getInt32Ty(),
false);
2638 llvm::SwitchInst *SI = Builder.CreateSwitch(Order, ContBB);
2640 Builder.SetInsertPoint(AcquireBB);
2641 Builder.CreateFence(llvm::AtomicOrdering::Acquire, SSID);
2642 Builder.CreateBr(ContBB);
2643 SI->addCase(Builder.getInt32(1), AcquireBB);
2644 SI->addCase(Builder.getInt32(2), AcquireBB);
2646 Builder.SetInsertPoint(ReleaseBB);
2647 Builder.CreateFence(llvm::AtomicOrdering::Release, SSID);
2648 Builder.CreateBr(ContBB);
2649 SI->addCase(Builder.getInt32(3), ReleaseBB);
2651 Builder.SetInsertPoint(AcqRelBB);
2652 Builder.CreateFence(llvm::AtomicOrdering::AcquireRelease, SSID);
2653 Builder.CreateBr(ContBB);
2654 SI->addCase(Builder.getInt32(4), AcqRelBB);
2656 Builder.SetInsertPoint(SeqCstBB);
2657 Builder.CreateFence(llvm::AtomicOrdering::SequentiallyConsistent, SSID);
2658 Builder.CreateBr(ContBB);
2659 SI->addCase(Builder.getInt32(5), SeqCstBB);
2661 Builder.SetInsertPoint(ContBB);
2665 case Builtin::BI__builtin_signbit:
2666 case Builtin::BI__builtin_signbitf:
2667 case Builtin::BI__builtin_signbitl: {
2672 case Builtin::BI__annotation: {
2676 const auto *Str = cast<StringLiteral>(Arg->IgnoreParenCasts());
2677 assert(Str->getCharByteWidth() == 2);
2678 StringRef WideBytes = Str->getBytes();
2679 std::string StrUtf8;
2680 if (!convertUTF16ToUTF8String(
2681 makeArrayRef(WideBytes.data(), WideBytes.size()), StrUtf8)) {
2682 CGM.ErrorUnsupported(E,
"non-UTF16 __annotation argument");
2685 Strings.push_back(llvm::MDString::get(getLLVMContext(), StrUtf8));
2689 llvm::Value *F = CGM.getIntrinsic(llvm::Intrinsic::codeview_annotation, {});
2690 MDTuple *StrTuple = MDTuple::get(getLLVMContext(), Strings);
2691 Builder.CreateCall(F, MetadataAsValue::get(getLLVMContext(), StrTuple));
2694 case Builtin::BI__builtin_annotation: {
2696 llvm::Value *F = CGM.getIntrinsic(llvm::Intrinsic::annotation,
2702 StringRef Str = cast<StringLiteral>(AnnotationStrExpr)->getString();
2705 case Builtin::BI__builtin_addcb:
2706 case Builtin::BI__builtin_addcs:
2707 case Builtin::BI__builtin_addc:
2708 case Builtin::BI__builtin_addcl:
2709 case Builtin::BI__builtin_addcll:
2710 case Builtin::BI__builtin_subcb:
2711 case Builtin::BI__builtin_subcs:
2712 case Builtin::BI__builtin_subc:
2713 case Builtin::BI__builtin_subcl:
2714 case Builtin::BI__builtin_subcll: {
2737 Address CarryOutPtr = EmitPointerWithAlignment(E->
getArg(3));
2741 switch (BuiltinID) {
2742 default: llvm_unreachable(
"Unknown multiprecision builtin id.");
2743 case Builtin::BI__builtin_addcb:
2744 case Builtin::BI__builtin_addcs:
2745 case Builtin::BI__builtin_addc:
2746 case Builtin::BI__builtin_addcl:
2747 case Builtin::BI__builtin_addcll:
2748 IntrinsicId = llvm::Intrinsic::uadd_with_overflow;
2750 case Builtin::BI__builtin_subcb:
2751 case Builtin::BI__builtin_subcs:
2752 case Builtin::BI__builtin_subc:
2753 case Builtin::BI__builtin_subcl:
2754 case Builtin::BI__builtin_subcll:
2755 IntrinsicId = llvm::Intrinsic::usub_with_overflow;
2765 Sum1, Carryin, Carry2);
2766 llvm::Value *CarryOut = Builder.CreateZExt(Builder.CreateOr(Carry1, Carry2),
2768 Builder.CreateStore(CarryOut, CarryOutPtr);
2772 case Builtin::BI__builtin_add_overflow:
2773 case Builtin::BI__builtin_sub_overflow:
2774 case Builtin::BI__builtin_mul_overflow: {
2782 WidthAndSignedness LeftInfo =
2784 WidthAndSignedness RightInfo =
2786 WidthAndSignedness ResultInfo =
2793 RightInfo, ResultArg, ResultQTy,
2796 WidthAndSignedness EncompassingInfo =
2800 llvm::IntegerType::get(CGM.getLLVMContext(), EncompassingInfo.Width);
2802 llvm::Type *ResultLLVMTy = CGM.getTypes().ConvertType(ResultQTy);
2805 switch (BuiltinID) {
2807 llvm_unreachable(
"Unknown overflow builtin id.");
2808 case Builtin::BI__builtin_add_overflow:
2809 IntrinsicId = EncompassingInfo.Signed
2810 ? llvm::Intrinsic::sadd_with_overflow
2811 : llvm::Intrinsic::uadd_with_overflow;
2813 case Builtin::BI__builtin_sub_overflow:
2814 IntrinsicId = EncompassingInfo.Signed
2815 ? llvm::Intrinsic::ssub_with_overflow
2816 : llvm::Intrinsic::usub_with_overflow;
2818 case Builtin::BI__builtin_mul_overflow:
2819 IntrinsicId = EncompassingInfo.Signed
2820 ? llvm::Intrinsic::smul_with_overflow
2821 : llvm::Intrinsic::umul_with_overflow;
2827 Address ResultPtr = EmitPointerWithAlignment(ResultArg);
2830 Left = Builder.CreateIntCast(Left, EncompassingLLVMTy, LeftInfo.Signed);
2831 Right = Builder.CreateIntCast(Right, EncompassingLLVMTy, RightInfo.Signed);
2837 if (EncompassingInfo.Width > ResultInfo.Width) {
2840 llvm::Value *ResultTrunc = Builder.CreateTrunc(Result, ResultLLVMTy);
2844 llvm::Value *ResultTruncExt = Builder.CreateIntCast(
2845 ResultTrunc, EncompassingLLVMTy, ResultInfo.Signed);
2847 Builder.CreateICmpNE(Result, ResultTruncExt);
2849 Overflow = Builder.CreateOr(Overflow, TruncationOverflow);
2850 Result = ResultTrunc;
2856 Builder.CreateStore(EmitToMemory(Result, ResultQTy), ResultPtr, isVolatile);
2861 case Builtin::BI__builtin_uadd_overflow:
2862 case Builtin::BI__builtin_uaddl_overflow:
2863 case Builtin::BI__builtin_uaddll_overflow:
2864 case Builtin::BI__builtin_usub_overflow:
2865 case Builtin::BI__builtin_usubl_overflow:
2866 case Builtin::BI__builtin_usubll_overflow:
2867 case Builtin::BI__builtin_umul_overflow:
2868 case Builtin::BI__builtin_umull_overflow:
2869 case Builtin::BI__builtin_umulll_overflow:
2870 case Builtin::BI__builtin_sadd_overflow:
2871 case Builtin::BI__builtin_saddl_overflow:
2872 case Builtin::BI__builtin_saddll_overflow:
2873 case Builtin::BI__builtin_ssub_overflow:
2874 case Builtin::BI__builtin_ssubl_overflow:
2875 case Builtin::BI__builtin_ssubll_overflow:
2876 case Builtin::BI__builtin_smul_overflow:
2877 case Builtin::BI__builtin_smull_overflow:
2878 case Builtin::BI__builtin_smulll_overflow: {
2889 switch (BuiltinID) {
2890 default: llvm_unreachable(
"Unknown overflow builtin id.");
2891 case Builtin::BI__builtin_uadd_overflow:
2892 case Builtin::BI__builtin_uaddl_overflow:
2893 case Builtin::BI__builtin_uaddll_overflow:
2894 IntrinsicId = llvm::Intrinsic::uadd_with_overflow;
2896 case Builtin::BI__builtin_usub_overflow:
2897 case Builtin::BI__builtin_usubl_overflow:
2898 case Builtin::BI__builtin_usubll_overflow:
2899 IntrinsicId = llvm::Intrinsic::usub_with_overflow;
2901 case Builtin::BI__builtin_umul_overflow:
2902 case Builtin::BI__builtin_umull_overflow:
2903 case Builtin::BI__builtin_umulll_overflow:
2904 IntrinsicId = llvm::Intrinsic::umul_with_overflow;
2906 case Builtin::BI__builtin_sadd_overflow:
2907 case Builtin::BI__builtin_saddl_overflow:
2908 case Builtin::BI__builtin_saddll_overflow:
2909 IntrinsicId = llvm::Intrinsic::sadd_with_overflow;
2911 case Builtin::BI__builtin_ssub_overflow:
2912 case Builtin::BI__builtin_ssubl_overflow:
2913 case Builtin::BI__builtin_ssubll_overflow:
2914 IntrinsicId = llvm::Intrinsic::ssub_with_overflow;
2916 case Builtin::BI__builtin_smul_overflow:
2917 case Builtin::BI__builtin_smull_overflow:
2918 case Builtin::BI__builtin_smulll_overflow:
2919 IntrinsicId = llvm::Intrinsic::smul_with_overflow;
2926 Builder.CreateStore(Sum, SumOutPtr);
2930 case Builtin::BI__builtin_addressof:
2932 case Builtin::BI__builtin_operator_new:
2933 return EmitBuiltinNewDeleteCall(
2935 case Builtin::BI__builtin_operator_delete:
2936 return EmitBuiltinNewDeleteCall(
2939 case Builtin::BI__noop:
2942 case Builtin::BI__builtin_call_with_static_chain: {
2946 EmitCallee(Call->
getCallee()), Call, ReturnValue,
2947 EmitScalarExpr(Chain));
2949 case Builtin::BI_InterlockedExchange8:
2950 case Builtin::BI_InterlockedExchange16:
2951 case Builtin::BI_InterlockedExchange:
2952 case Builtin::BI_InterlockedExchangePointer:
2954 EmitMSVCBuiltinExpr(MSVCIntrin::_InterlockedExchange, E));
2955 case Builtin::BI_InterlockedCompareExchangePointer: {
2957 llvm::IntegerType *IntType =
2958 IntegerType::get(getLLVMContext(),
2959 getContext().getTypeSize(E->
getType()));
2960 llvm::Type *IntPtrType = IntType->getPointerTo();
2963 Builder.CreateBitCast(EmitScalarExpr(E->
getArg(0)), IntPtrType);
2966 RTy = Exchange->getType();
2967 Exchange = Builder.CreatePtrToInt(Exchange, IntType);
2970 Builder.CreatePtrToInt(EmitScalarExpr(E->
getArg(2)), IntType);
2973 Builder.CreateAtomicCmpXchg(Destination, Comparand, Exchange,
2974 AtomicOrdering::SequentiallyConsistent,
2975 AtomicOrdering::SequentiallyConsistent);
2976 Result->setVolatile(
true);
2978 return RValue::get(Builder.CreateIntToPtr(Builder.CreateExtractValue(Result,
2982 case Builtin::BI_InterlockedCompareExchange8:
2983 case Builtin::BI_InterlockedCompareExchange16:
2984 case Builtin::BI_InterlockedCompareExchange:
2985 case Builtin::BI_InterlockedCompareExchange64: {
2986 AtomicCmpXchgInst *CXI = Builder.CreateAtomicCmpXchg(
2987 EmitScalarExpr(E->
getArg(0)),
2988 EmitScalarExpr(E->
getArg(2)),
2989 EmitScalarExpr(E->
getArg(1)),
2990 AtomicOrdering::SequentiallyConsistent,
2991 AtomicOrdering::SequentiallyConsistent);
2992 CXI->setVolatile(
true);
2993 return RValue::get(Builder.CreateExtractValue(CXI, 0));
2995 case Builtin::BI_InterlockedIncrement16:
2996 case Builtin::BI_InterlockedIncrement:
2998 EmitMSVCBuiltinExpr(MSVCIntrin::_InterlockedIncrement, E));
2999 case Builtin::BI_InterlockedDecrement16:
3000 case Builtin::BI_InterlockedDecrement:
3002 EmitMSVCBuiltinExpr(MSVCIntrin::_InterlockedDecrement, E));
3003 case Builtin::BI_InterlockedAnd8:
3004 case Builtin::BI_InterlockedAnd16:
3005 case Builtin::BI_InterlockedAnd:
3006 return RValue::get(EmitMSVCBuiltinExpr(MSVCIntrin::_InterlockedAnd, E));
3007 case Builtin::BI_InterlockedExchangeAdd8:
3008 case Builtin::BI_InterlockedExchangeAdd16:
3009 case Builtin::BI_InterlockedExchangeAdd:
3011 EmitMSVCBuiltinExpr(MSVCIntrin::_InterlockedExchangeAdd, E));
3012 case Builtin::BI_InterlockedExchangeSub8:
3013 case Builtin::BI_InterlockedExchangeSub16:
3014 case Builtin::BI_InterlockedExchangeSub:
3016 EmitMSVCBuiltinExpr(MSVCIntrin::_InterlockedExchangeSub, E));
3017 case Builtin::BI_InterlockedOr8:
3018 case Builtin::BI_InterlockedOr16:
3019 case Builtin::BI_InterlockedOr:
3020 return RValue::get(EmitMSVCBuiltinExpr(MSVCIntrin::_InterlockedOr, E));
3021 case Builtin::BI_InterlockedXor8:
3022 case Builtin::BI_InterlockedXor16:
3023 case Builtin::BI_InterlockedXor:
3024 return RValue::get(EmitMSVCBuiltinExpr(MSVCIntrin::_InterlockedXor, E));
3026 case Builtin::BI_bittest64:
3027 case Builtin::BI_bittest:
3028 case Builtin::BI_bittestandcomplement64:
3029 case Builtin::BI_bittestandcomplement:
3030 case Builtin::BI_bittestandreset64:
3031 case Builtin::BI_bittestandreset:
3032 case Builtin::BI_bittestandset64:
3033 case Builtin::BI_bittestandset:
3034 case Builtin::BI_interlockedbittestandreset:
3035 case Builtin::BI_interlockedbittestandreset64:
3036 case Builtin::BI_interlockedbittestandset64:
3037 case Builtin::BI_interlockedbittestandset:
3038 case Builtin::BI_interlockedbittestandset_acq:
3039 case Builtin::BI_interlockedbittestandset_rel:
3040 case Builtin::BI_interlockedbittestandset_nf:
3041 case Builtin::BI_interlockedbittestandreset_acq:
3042 case Builtin::BI_interlockedbittestandreset_rel:
3043 case Builtin::BI_interlockedbittestandreset_nf:
3046 case Builtin::BI__exception_code:
3047 case Builtin::BI_exception_code:
3049 case Builtin::BI__exception_info:
3050 case Builtin::BI_exception_info:
3052 case Builtin::BI__abnormal_termination:
3053 case Builtin::BI_abnormal_termination:
3055 case Builtin::BI_setjmpex:
3056 if (getTarget().getTriple().isOSMSVCRT())
3059 case Builtin::BI_setjmp:
3060 if (getTarget().getTriple().isOSMSVCRT()) {
3061 if (getTarget().getTriple().getArch() == llvm::Triple::x86)
3063 else if (getTarget().getTriple().getArch() == llvm::Triple::aarch64)
3069 case Builtin::BI__GetExceptionInfo: {
3070 if (llvm::GlobalVariable *GV =
3072 return RValue::get(llvm::ConstantExpr::getBitCast(GV, CGM.Int8PtrTy));
3076 case Builtin::BI__fastfail:
3077 return RValue::get(EmitMSVCBuiltinExpr(MSVCIntrin::__fastfail, E));
3079 case Builtin::BI__builtin_coro_size: {
3080 auto & Context = getContext();
3082 auto T = Builder.getIntNTy(Context.
getTypeSize(SizeTy));
3083 Value *F = CGM.getIntrinsic(Intrinsic::coro_size, T);
3087 case Builtin::BI__builtin_coro_id:
3088 return EmitCoroutineIntrinsic(E, Intrinsic::coro_id);
3089 case Builtin::BI__builtin_coro_promise:
3090 return EmitCoroutineIntrinsic(E, Intrinsic::coro_promise);
3091 case Builtin::BI__builtin_coro_resume:
3092 return EmitCoroutineIntrinsic(E, Intrinsic::coro_resume);
3093 case Builtin::BI__builtin_coro_frame:
3094 return EmitCoroutineIntrinsic(E, Intrinsic::coro_frame);
3095 case Builtin::BI__builtin_coro_noop:
3096 return EmitCoroutineIntrinsic(E, Intrinsic::coro_noop);
3097 case Builtin::BI__builtin_coro_free:
3098 return EmitCoroutineIntrinsic(E, Intrinsic::coro_free);
3099 case Builtin::BI__builtin_coro_destroy:
3100 return EmitCoroutineIntrinsic(E, Intrinsic::coro_destroy);
3101 case Builtin::BI__builtin_coro_done:
3102 return EmitCoroutineIntrinsic(E, Intrinsic::coro_done);
3103 case Builtin::BI__builtin_coro_alloc:
3104 return EmitCoroutineIntrinsic(E, Intrinsic::coro_alloc);
3105 case Builtin::BI__builtin_coro_begin:
3106 return EmitCoroutineIntrinsic(E, Intrinsic::coro_begin);
3107 case Builtin::BI__builtin_coro_end:
3108 return EmitCoroutineIntrinsic(E, Intrinsic::coro_end);
3109 case Builtin::BI__builtin_coro_suspend:
3110 return EmitCoroutineIntrinsic(E, Intrinsic::coro_suspend);
3111 case Builtin::BI__builtin_coro_param:
3112 return EmitCoroutineIntrinsic(E, Intrinsic::coro_param);
3115 case Builtin::BIread_pipe:
3116 case Builtin::BIwrite_pipe: {
3118 *Arg1 = EmitScalarExpr(E->
getArg(1));
3120 Value *PacketSize = OpenCLRT.getPipeElemSize(E->
getArg(0));
3121 Value *PacketAlign = OpenCLRT.getPipeElemAlign(E->
getArg(0));
3124 unsigned GenericAS =
3127 llvm::Type::getInt8Ty(getLLVMContext()), GenericAS);
3131 const char *Name = (BuiltinID == Builtin::BIread_pipe) ?
"__read_pipe_2" 3135 llvm::Type *ArgTys[] = {Arg0->getType(), I8PTy, Int32Ty, Int32Ty};
3136 llvm::FunctionType *FTy = llvm::FunctionType::get(
3138 Value *BCast = Builder.CreatePointerCast(Arg1, I8PTy);
3140 Builder.CreateCall(CGM.CreateRuntimeFunction(FTy, Name),
3141 {Arg0, BCast, PacketSize, PacketAlign}));
3144 "Illegal number of parameters to pipe function");
3145 const char *Name = (BuiltinID == Builtin::BIread_pipe) ?
"__read_pipe_4" 3148 llvm::Type *ArgTys[] = {Arg0->getType(), Arg1->getType(), Int32Ty, I8PTy,
3151 *Arg3 = EmitScalarExpr(E->
getArg(3));
3152 llvm::FunctionType *FTy = llvm::FunctionType::get(
3154 Value *BCast = Builder.CreatePointerCast(Arg3, I8PTy);
3157 if (Arg2->getType() != Int32Ty)
3158 Arg2 = Builder.CreateZExtOrTrunc(Arg2, Int32Ty);
3160 CGM.CreateRuntimeFunction(FTy, Name),
3161 {Arg0, Arg1, Arg2, BCast, PacketSize, PacketAlign}));
3166 case Builtin::BIreserve_read_pipe:
3167 case Builtin::BIreserve_write_pipe:
3168 case Builtin::BIwork_group_reserve_read_pipe:
3169 case Builtin::BIwork_group_reserve_write_pipe:
3170 case Builtin::BIsub_group_reserve_read_pipe:
3171 case Builtin::BIsub_group_reserve_write_pipe: {
3174 if (BuiltinID == Builtin::BIreserve_read_pipe)
3175 Name =
"__reserve_read_pipe";
3176 else if (BuiltinID == Builtin::BIreserve_write_pipe)
3177 Name =
"__reserve_write_pipe";
3178 else if (BuiltinID == Builtin::BIwork_group_reserve_read_pipe)
3179 Name =
"__work_group_reserve_read_pipe";
3180 else if (BuiltinID == Builtin::BIwork_group_reserve_write_pipe)
3181 Name =
"__work_group_reserve_write_pipe";
3182 else if (BuiltinID == Builtin::BIsub_group_reserve_read_pipe)
3183 Name =
"__sub_group_reserve_read_pipe";
3185 Name =
"__sub_group_reserve_write_pipe";
3188 *Arg1 = EmitScalarExpr(E->
getArg(1));
3189 llvm::Type *ReservedIDTy = ConvertType(getContext().OCLReserveIDTy);
3195 llvm::Type *ArgTys[] = {Arg0->getType(), Int32Ty, Int32Ty, Int32Ty};
3196 llvm::FunctionType *FTy = llvm::FunctionType::get(
3200 if (Arg1->getType() != Int32Ty)
3201 Arg1 = Builder.CreateZExtOrTrunc(Arg1, Int32Ty);
3203 Builder.CreateCall(CGM.CreateRuntimeFunction(FTy, Name),
3204 {Arg0, Arg1, PacketSize, PacketAlign}));
3208 case Builtin::BIcommit_read_pipe:
3209 case Builtin::BIcommit_write_pipe:
3210 case Builtin::BIwork_group_commit_read_pipe:
3211 case Builtin::BIwork_group_commit_write_pipe:
3212 case Builtin::BIsub_group_commit_read_pipe:
3213 case Builtin::BIsub_group_commit_write_pipe: {
3215 if (BuiltinID == Builtin::BIcommit_read_pipe)
3216 Name =
"__commit_read_pipe";
3217 else if (BuiltinID == Builtin::BIcommit_write_pipe)
3218 Name =
"__commit_write_pipe";
3219 else if (BuiltinID == Builtin::BIwork_group_commit_read_pipe)
3220 Name =
"__work_group_commit_read_pipe";
3221 else if (BuiltinID == Builtin::BIwork_group_commit_write_pipe)
3222 Name =
"__work_group_commit_write_pipe";
3223 else if (BuiltinID == Builtin::BIsub_group_commit_read_pipe)
3224 Name =
"__sub_group_commit_read_pipe";
3226 Name =
"__sub_group_commit_write_pipe";
3229 *Arg1 = EmitScalarExpr(E->
getArg(1));
3231 Value *PacketSize = OpenCLRT.getPipeElemSize(E->
getArg(0));
3232 Value *PacketAlign = OpenCLRT.getPipeElemAlign(E->
getArg(0));
3235 llvm::Type *ArgTys[] = {Arg0->getType(), Arg1->getType(), Int32Ty, Int32Ty};
3236 llvm::FunctionType *FTy =
3237 llvm::FunctionType::get(llvm::Type::getVoidTy(getLLVMContext()),
3241 Builder.CreateCall(CGM.CreateRuntimeFunction(FTy, Name),
3242 {Arg0, Arg1, PacketSize, PacketAlign}));
3245 case Builtin::BIget_pipe_num_packets:
3246 case Builtin::BIget_pipe_max_packets: {
3247 const char *BaseName;
3249 if (BuiltinID == Builtin::BIget_pipe_num_packets)
3250 BaseName =
"__get_pipe_num_packets";
3252 BaseName =
"__get_pipe_max_packets";
3253 auto Name = std::string(BaseName) +
3254 std::string(PipeTy->
isReadOnly() ?
"_ro" :
"_wo");
3259 Value *PacketSize = OpenCLRT.getPipeElemSize(E->
getArg(0));
3260 Value *PacketAlign = OpenCLRT.getPipeElemAlign(E->
getArg(0));
3261 llvm::Type *ArgTys[] = {Arg0->getType(), Int32Ty, Int32Ty};
3262 llvm::FunctionType *FTy = llvm::FunctionType::get(
3265 return RValue::get(Builder.CreateCall(CGM.CreateRuntimeFunction(FTy, Name),
3266 {Arg0, PacketSize, PacketAlign}));
3270 case Builtin::BIto_global:
3271 case Builtin::BIto_local:
3272 case Builtin::BIto_private: {
3273 auto Arg0 = EmitScalarExpr(E->
getArg(0));
3274 auto NewArgT = llvm::PointerType::get(Int8Ty,
3276 auto NewRetT = llvm::PointerType::get(Int8Ty,
3277 CGM.getContext().getTargetAddressSpace(
3279 auto FTy = llvm::FunctionType::get(NewRetT, {NewArgT},
false);
3281 if (Arg0->getType()->getPointerAddressSpace() !=
3282 NewArgT->getPointerAddressSpace())
3283 NewArg = Builder.CreateAddrSpaceCast(Arg0, NewArgT);
3285 NewArg = Builder.CreateBitOrPointerCast(Arg0, NewArgT);
3288 Builder.CreateCall(CGM.CreateRuntimeFunction(FTy, NewName), {NewArg});
3289 return RValue::get(Builder.CreateBitOrPointerCast(NewCall,
3295 case Builtin::BIenqueue_kernel: {
3299 llvm::Type *QueueTy = ConvertType(getContext().OCLQueueTy);
3300 llvm::Type *GenericVoidPtrTy = Builder.getInt8PtrTy(
3306 llvm::Value *Range = NDRangeL.getAddress().getPointer();
3307 llvm::Type *RangeTy = NDRangeL.getAddress().getType();
3312 Name =
"__enqueue_kernel_basic";
3313 llvm::Type *ArgTys[] = {QueueTy, Int32Ty, RangeTy, GenericVoidPtrTy,
3315 llvm::FunctionType *FTy = llvm::FunctionType::get(
3319 CGM.getOpenCLRuntime().emitOpenCLEnqueuedBlock(*
this, E->
getArg(3));
3321 Builder.CreatePointerCast(Info.Kernel, GenericVoidPtrTy);
3323 Builder.CreatePointerCast(Info.BlockArg, GenericVoidPtrTy);
3326 B.addAttribute(Attribute::ByVal);
3327 llvm::AttributeList ByValAttrSet =
3328 llvm::AttributeList::get(CGM.getModule().getContext(), 3U, B);
3331 Builder.CreateCall(CGM.CreateRuntimeFunction(FTy, Name, ByValAttrSet),
3332 {Queue, Flags, Range, Kernel, Block});
3333 RTCall->setAttributes(ByValAttrSet);
3336 assert(NumArgs >= 5 &&
"Invalid enqueue_kernel signature");
3340 auto CreateArrayForSizeVar = [=](
unsigned First) {
3341 auto *AT = llvm::ArrayType::get(SizeTy, NumArgs -
First);
3342 auto *Arr = Builder.CreateAlloca(AT);
3346 auto *Zero = llvm::ConstantInt::get(IntTy, 0);
3347 for (
unsigned I =
First; I < NumArgs; ++I) {
3348 auto *Index = llvm::ConstantInt::get(IntTy, I -
First);
3349 auto *GEP = Builder.CreateGEP(Arr, {Zero, Index});
3353 Builder.CreateZExtOrTrunc(EmitScalarExpr(E->
getArg(I)), SizeTy);
3354 Builder.CreateAlignedStore(
3355 V, GEP, CGM.getDataLayout().getPrefTypeAlignment(SizeTy));
3363 Name =
"__enqueue_kernel_varargs";
3365 CGM.getOpenCLRuntime().emitOpenCLEnqueuedBlock(*
this, E->
getArg(3));
3367 Builder.CreatePointerCast(Info.Kernel, GenericVoidPtrTy);
3368 auto *Block = Builder.CreatePointerCast(Info.BlockArg, GenericVoidPtrTy);
3369 auto *PtrToSizeArray = CreateArrayForSizeVar(4);
3373 std::vector<llvm::Value *> Args = {
3374 Queue, Flags, Range,
3375 Kernel, Block, ConstantInt::get(IntTy, NumArgs - 4),
3377 std::vector<llvm::Type *> ArgTys = {
3378 QueueTy, IntTy, RangeTy,
3379 GenericVoidPtrTy, GenericVoidPtrTy, IntTy,
3380 PtrToSizeArray->getType()};
3382 llvm::FunctionType *FTy = llvm::FunctionType::get(
3385 Builder.CreateCall(CGM.CreateRuntimeFunction(FTy, Name),
3390 llvm::Type *EventTy = ConvertType(getContext().OCLClkEventTy);
3391 llvm::Type *EventPtrTy = EventTy->getPointerTo(
3395 Builder.CreateZExtOrTrunc(EmitScalarExpr(E->
getArg(3)), Int32Ty);
3398 ? EmitArrayToPointerDecay(E->
getArg(4)).getPointer()
3399 : EmitScalarExpr(E->
getArg(4));
3402 EventList = Builder.CreatePointerCast(EventList, EventPtrTy);
3403 ClkEvent = Builder.CreatePointerCast(ClkEvent, EventPtrTy);
3405 CGM.getOpenCLRuntime().emitOpenCLEnqueuedBlock(*
this, E->
getArg(6));
3407 Builder.CreatePointerCast(Info.Kernel, GenericVoidPtrTy);
3409 Builder.CreatePointerCast(Info.BlockArg, GenericVoidPtrTy);
3411 std::vector<llvm::Type *> ArgTys = {
3412 QueueTy, Int32Ty, RangeTy, Int32Ty,
3413 EventPtrTy, EventPtrTy, GenericVoidPtrTy, GenericVoidPtrTy};
3415 std::vector<llvm::Value *> Args = {Queue, Flags, Range, NumEvents,
3416 EventList, ClkEvent, Kernel, Block};
3420 Name =
"__enqueue_kernel_basic_events";
3421 llvm::FunctionType *FTy = llvm::FunctionType::get(
3424 Builder.CreateCall(CGM.CreateRuntimeFunction(FTy, Name),
3429 Args.push_back(ConstantInt::get(Int32Ty, NumArgs - 7));
3430 ArgTys.push_back(Int32Ty);
3431 Name =
"__enqueue_kernel_events_varargs";
3433 auto *PtrToSizeArray = CreateArrayForSizeVar(7);
3434 Args.push_back(PtrToSizeArray);
3435 ArgTys.push_back(PtrToSizeArray->getType());
3437 llvm::FunctionType *FTy = llvm::FunctionType::get(
3440 Builder.CreateCall(CGM.CreateRuntimeFunction(FTy, Name),
3447 case Builtin::BIget_kernel_work_group_size: {
3448 llvm::Type *GenericVoidPtrTy = Builder.getInt8PtrTy(
3451 CGM.getOpenCLRuntime().emitOpenCLEnqueuedBlock(*
this, E->
getArg(0));
3452 Value *Kernel = Builder.CreatePointerCast(Info.Kernel, GenericVoidPtrTy);
3453 Value *Arg = Builder.CreatePointerCast(Info.BlockArg, GenericVoidPtrTy);
3455 CGM.CreateRuntimeFunction(
3456 llvm::FunctionType::get(IntTy, {GenericVoidPtrTy, GenericVoidPtrTy},
3458 "__get_kernel_work_group_size_impl"),
3461 case Builtin::BIget_kernel_preferred_work_group_size_multiple: {
3462 llvm::Type *GenericVoidPtrTy = Builder.getInt8PtrTy(
3465 CGM.getOpenCLRuntime().emitOpenCLEnqueuedBlock(*
this, E->
getArg(0));
3466 Value *Kernel = Builder.CreatePointerCast(Info.Kernel, GenericVoidPtrTy);
3467 Value *Arg = Builder.CreatePointerCast(Info.BlockArg, GenericVoidPtrTy);
3469 CGM.CreateRuntimeFunction(
3470 llvm::FunctionType::get(IntTy, {GenericVoidPtrTy, GenericVoidPtrTy},
3472 "__get_kernel_preferred_work_group_size_multiple_impl"),
3475 case Builtin::BIget_kernel_max_sub_group_size_for_ndrange:
3476 case Builtin::BIget_kernel_sub_group_count_for_ndrange: {
3477 llvm::Type *GenericVoidPtrTy = Builder.getInt8PtrTy(
3482 CGM.getOpenCLRuntime().emitOpenCLEnqueuedBlock(*
this, E->
getArg(1));
3483 Value *Kernel = Builder.CreatePointerCast(Info.Kernel, GenericVoidPtrTy);
3484 Value *Block = Builder.CreatePointerCast(Info.BlockArg, GenericVoidPtrTy);
3486 BuiltinID == Builtin::BIget_kernel_max_sub_group_size_for_ndrange
3487 ?
"__get_kernel_max_sub_group_size_for_ndrange_impl" 3488 :
"__get_kernel_sub_group_count_for_ndrange_impl";
3490 CGM.CreateRuntimeFunction(
3491 llvm::FunctionType::get(
3492 IntTy, {NDRange->getType(), GenericVoidPtrTy, GenericVoidPtrTy},
3495 {NDRange, Kernel, Block}));
3498 case Builtin::BI__builtin_store_half:
3499 case Builtin::BI__builtin_store_halff: {
3502 Value *HalfVal = Builder.CreateFPTrunc(Val, Builder.getHalfTy());
3503 return RValue::get(Builder.CreateStore(HalfVal, Address));
3505 case Builtin::BI__builtin_load_half: {
3507 Value *HalfVal = Builder.CreateLoad(Address);
3508 return RValue::get(Builder.CreateFPExt(HalfVal, Builder.getDoubleTy()));
3510 case Builtin::BI__builtin_load_halff: {
3512 Value *HalfVal = Builder.CreateLoad(Address);
3513 return RValue::get(Builder.CreateFPExt(HalfVal, Builder.getFloatTy()));
3515 case Builtin::BIprintf:
3516 if (getTarget().getTriple().isNVPTX())
3517 return EmitNVPTXDevicePrintfCallExpr(E, ReturnValue);
3519 case Builtin::BI__builtin_canonicalize:
3520 case Builtin::BI__builtin_canonicalizef:
3521 case Builtin::BI__builtin_canonicalizel:
3524 case Builtin::BI__builtin_thread_pointer: {
3525 if (!getContext().getTargetInfo().isTLSSupported())
3526 CGM.ErrorUnsupported(E,
"__builtin_thread_pointer");
3530 case Builtin::BI__builtin_os_log_format:
3531 return emitBuiltinOSLogFormat(*E);
3533 case Builtin::BI__builtin_os_log_format_buffer_size: {
3540 case Builtin::BI__xray_customevent: {
3541 if (!ShouldXRayInstrumentFunction())
3544 if (!CGM.getCodeGenOpts().XRayInstrumentationBundle.has(
3548 if (
const auto *XRayAttr = CurFuncDecl->getAttr<XRayInstrumentAttr>())
3549 if (XRayAttr->neverXRayInstrument() && !AlwaysEmitXRayCustomEvents())
3552 Function *F = CGM.getIntrinsic(Intrinsic::xray_customevent);
3553 auto FTy = F->getFunctionType();
3554 auto Arg0 = E->
getArg(0);
3555 auto Arg0Val = EmitScalarExpr(Arg0);
3556 auto Arg0Ty = Arg0->getType();
3557 auto PTy0 = FTy->getParamType(0);
3558 if (PTy0 != Arg0Val->getType()) {
3559 if (Arg0Ty->isArrayType())
3560 Arg0Val = EmitArrayToPointerDecay(Arg0).getPointer();
3562 Arg0Val = Builder.CreatePointerCast(Arg0Val, PTy0);
3564 auto Arg1 = EmitScalarExpr(E->
getArg(1));
3565 auto PTy1 = FTy->getParamType(1);
3566 if (PTy1 != Arg1->getType())
3567 Arg1 = Builder.CreateTruncOrBitCast(Arg1, PTy1);
3568 return RValue::get(Builder.CreateCall(F, {Arg0Val, Arg1}));
3571 case Builtin::BI__xray_typedevent: {
3575 if (!ShouldXRayInstrumentFunction())
3578 if (!CGM.getCodeGenOpts().XRayInstrumentationBundle.has(
3582 if (
const auto *XRayAttr = CurFuncDecl->getAttr<XRayInstrumentAttr>())
3583 if (XRayAttr->neverXRayInstrument() && !AlwaysEmitXRayTypedEvents())
3586 Function *F = CGM.getIntrinsic(Intrinsic::xray_typedevent);
3587 auto FTy = F->getFunctionType();
3588 auto Arg0 = EmitScalarExpr(E->
getArg(0));
3589 auto PTy0 = FTy->getParamType(0);
3590 if (PTy0 != Arg0->getType())
3591 Arg0 = Builder.CreateTruncOrBitCast(Arg0, PTy0);
3592 auto Arg1 = E->
getArg(1);
3593 auto Arg1Val = EmitScalarExpr(Arg1);
3594 auto Arg1Ty = Arg1->getType();
3595 auto PTy1 = FTy->getParamType(1);
3596 if (PTy1 != Arg1Val->getType()) {
3597 if (Arg1Ty->isArrayType())
3598 Arg1Val = EmitArrayToPointerDecay(Arg1).getPointer();
3600 Arg1Val = Builder.CreatePointerCast(Arg1Val, PTy1);
3602 auto Arg2 = EmitScalarExpr(E->
getArg(2));
3603 auto PTy2 = FTy->getParamType(2);
3604 if (PTy2 != Arg2->getType())
3605 Arg2 = Builder.CreateTruncOrBitCast(Arg2, PTy2);
3606 return RValue::get(Builder.CreateCall(F, {Arg0, Arg1Val, Arg2}));
3609 case Builtin::BI__builtin_ms_va_start:
3610 case Builtin::BI__builtin_ms_va_end:
3612 EmitVAStartEnd(EmitMSVAListRef(E->
getArg(0)).getPointer(),
3613 BuiltinID == Builtin::BI__builtin_ms_va_start));
3615 case Builtin::BI__builtin_ms_va_copy: {
3632 Value *ArgPtr = Builder.CreateLoad(SrcAddr,
"ap.val");
3633 return RValue::get(Builder.CreateStore(ArgPtr, DestAddr));
3640 if (getContext().BuiltinInfo.isLibFunction(BuiltinID))
3642 CGM.getBuiltinLibFunction(FD, BuiltinID));
3646 if (getContext().BuiltinInfo.isPredefinedLibFunction(BuiltinID))
3648 cast<llvm::Constant>(EmitScalarExpr(E->
getCallee())));
3655 checkTargetFeatures(E, FD);
3657 if (
unsigned VectorWidth = getContext().
BuiltinInfo.getRequiredVectorWidth(BuiltinID))
3658 LargestVectorWidth =
std::max(LargestVectorWidth, VectorWidth);
3661 const char *Name = getContext().BuiltinInfo.getName(BuiltinID);
3664 llvm::Triple::getArchTypePrefix(getTarget().getTriple().getArch());
3665 if (!Prefix.empty()) {
3666 IntrinsicID = Intrinsic::getIntrinsicForGCCBuiltin(Prefix.data(), Name);
3670 if (IntrinsicID == Intrinsic::not_intrinsic)
3671 IntrinsicID = Intrinsic::getIntrinsicForMSBuiltin(Prefix.data(), Name);
3674 if (IntrinsicID != Intrinsic::not_intrinsic) {
3679 unsigned ICEArguments = 0;
3681 getContext().GetBuiltinType(BuiltinID, Error, &ICEArguments);
3684 Function *F = CGM.getIntrinsic(IntrinsicID);
3685 llvm::FunctionType *FTy = F->getFunctionType();
3687 for (
unsigned i = 0, e = E->
getNumArgs(); i != e; ++i) {
3690 if ((ICEArguments & (1 << i)) == 0) {
3691 ArgValue = EmitScalarExpr(E->
getArg(i));
3695 llvm::APSInt Result;
3697 assert(IsConst &&
"Constant arg isn't actually constant?");
3699 ArgValue = llvm::ConstantInt::get(getLLVMContext(), Result);
3705 if (PTy != ArgValue->getType()) {
3706 assert(PTy->canLosslesslyBitCastTo(FTy->getParamType(i)) &&
3707 "Must be able to losslessly bit cast to param");
3708 ArgValue = Builder.CreateBitCast(ArgValue, PTy);
3711 Args.push_back(ArgValue);
3714 Value *V = Builder.CreateCall(F, Args);
3719 RetTy = ConvertType(BuiltinRetType);
3721 if (RetTy != V->getType()) {
3722 assert(V->getType()->canLosslesslyBitCastTo(RetTy) &&
3723 "Must be able to losslessly bit cast result type");
3724 V = Builder.CreateBitCast(V, RetTy);
3731 if (
Value *V = EmitTargetBuiltinExpr(BuiltinID, E))
3734 ErrorUnsupported(E,
"builtin function");
3737 return GetUndefRValue(E->
getType());
3741 unsigned BuiltinID,
const CallExpr *E,
3742 llvm::Triple::ArchType Arch) {
3744 case llvm::Triple::arm:
3745 case llvm::Triple::armeb:
3746 case llvm::Triple::thumb:
3747 case llvm::Triple::thumbeb:
3749 case llvm::Triple::aarch64:
3750 case llvm::Triple::aarch64_be:
3752 case llvm::Triple::x86:
3753 case llvm::Triple::x86_64:
3755 case llvm::Triple::ppc:
3756 case llvm::Triple::ppc64:
3757 case llvm::Triple::ppc64le:
3759 case llvm::Triple::r600:
3760 case llvm::Triple::amdgcn:
3762 case llvm::Triple::systemz:
3764 case llvm::Triple::nvptx:
3765 case llvm::Triple::nvptx64:
3767 case llvm::Triple::wasm32:
3768 case llvm::Triple::wasm64:
3770 case llvm::Triple::hexagon:
3779 if (getContext().
BuiltinInfo.isAuxBuiltinID(BuiltinID)) {
3780 assert(getContext().getAuxTargetInfo() &&
"Missing aux target info");
3782 this, getContext().
BuiltinInfo.getAuxBuiltinID(BuiltinID), E,
3783 getContext().getAuxTargetInfo()->getTriple().getArch());
3787 getTarget().getTriple().getArch());
3792 bool HasLegalHalfType=
true,
3794 int IsQuad = TypeFlags.
isQuad();
3798 return llvm::VectorType::get(CGF->
Int8Ty, V1Ty ? 1 : (8 << IsQuad));
3801 return llvm::VectorType::get(CGF->
Int16Ty, V1Ty ? 1 : (4 << IsQuad));
3803 if (HasLegalHalfType)
3804 return llvm::VectorType::get(CGF->
HalfTy, V1Ty ? 1 : (4 << IsQuad));
3806 return llvm::VectorType::get(CGF->
Int16Ty, V1Ty ? 1 : (4 << IsQuad));
3808 return llvm::VectorType::get(CGF->
Int32Ty, V1Ty ? 1 : (2 << IsQuad));
3811 return llvm::VectorType::get(CGF->
Int64Ty, V1Ty ? 1 : (1 << IsQuad));
3816 return llvm::VectorType::get(CGF->
Int8Ty, 16);
3818 return llvm::VectorType::get(CGF->
FloatTy, V1Ty ? 1 : (2 << IsQuad));
3820 return llvm::VectorType::get(CGF->
DoubleTy, V1Ty ? 1 : (1 << IsQuad));
3822 llvm_unreachable(
"Unknown vector element type!");
3827 int IsQuad = IntTypeFlags.
isQuad();
3830 return llvm::VectorType::get(CGF->
HalfTy, (4 << IsQuad));
3832 return llvm::VectorType::get(CGF->
FloatTy, (2 << IsQuad));
3834 return llvm::VectorType::get(CGF->
DoubleTy, (1 << IsQuad));
3836 llvm_unreachable(
"Type can't be converted to floating-point!");
3841 unsigned nElts = V->getType()->getVectorNumElements();
3842 Value* SV = llvm::ConstantVector::getSplat(nElts, C);
3843 return Builder.CreateShuffleVector(V, V, SV,
"lane");
3848 unsigned shift,
bool rightshift) {
3850 for (Function::const_arg_iterator ai = F->arg_begin(), ae = F->arg_end();
3851 ai != ae; ++ai, ++j)
3852 if (shift > 0 && shift == j)
3853 Ops[j] = EmitNeonShiftVector(Ops[j], ai->getType(), rightshift);
3855 Ops[j] = Builder.CreateBitCast(Ops[j], ai->getType(), name);
3857 return Builder.CreateCall(F, Ops, name);
3862 int SV = cast<ConstantInt>(V)->getSExtValue();
3863 return ConstantInt::get(Ty, neg ? -SV : SV);
3870 llvm::VectorType *VTy = cast<llvm::VectorType>(Ty);
3872 int ShiftAmt = cast<ConstantInt>(
Shift)->getSExtValue();
3873 int EltSize = VTy->getScalarSizeInBits();
3875 Vec = Builder.CreateBitCast(Vec, Ty);
3879 if (ShiftAmt == EltSize) {
3882 return llvm::ConstantAggregateZero::get(VTy);
3887 Shift = ConstantInt::get(VTy->getElementType(), ShiftAmt);
3891 Shift = EmitNeonShiftVector(Shift, Ty,
false);
3893 return Builder.CreateLShr(Vec, Shift, name);
3895 return Builder.CreateAShr(Vec, Shift, name);
3921 struct NeonIntrinsicInfo {
3922 const char *NameHint;
3924 unsigned LLVMIntrinsic;
3925 unsigned AltLLVMIntrinsic;
3926 unsigned TypeModifier;
3928 bool operator<(
unsigned RHSBuiltinID)
const {
3929 return BuiltinID < RHSBuiltinID;
3931 bool operator<(
const NeonIntrinsicInfo &TE)
const {
3932 return BuiltinID < TE.BuiltinID;
3937 #define NEONMAP0(NameBase) \ 3938 { #NameBase, NEON::BI__builtin_neon_ ## NameBase, 0, 0, 0 } 3940 #define NEONMAP1(NameBase, LLVMIntrinsic, TypeModifier) \ 3941 { #NameBase, NEON:: BI__builtin_neon_ ## NameBase, \ 3942 Intrinsic::LLVMIntrinsic, 0, TypeModifier } 3944 #define NEONMAP2(NameBase, LLVMIntrinsic, AltLLVMIntrinsic, TypeModifier) \ 3945 { #NameBase, NEON:: BI__builtin_neon_ ## NameBase, \ 3946 Intrinsic::LLVMIntrinsic, Intrinsic::AltLLVMIntrinsic, \ 3952 NEONMAP1(vabs_v, arm_neon_vabs, 0),
3953 NEONMAP1(vabsq_v, arm_neon_vabs, 0),
3955 NEONMAP1(vaesdq_v, arm_neon_aesd, 0),
3956 NEONMAP1(vaeseq_v, arm_neon_aese, 0),
3957 NEONMAP1(vaesimcq_v, arm_neon_aesimc, 0),
3958 NEONMAP1(vaesmcq_v, arm_neon_aesmc, 0),
3961 NEONMAP1(vcage_v, arm_neon_vacge, 0),
3962 NEONMAP1(vcageq_v, arm_neon_vacge, 0),
3963 NEONMAP1(vcagt_v, arm_neon_vacgt, 0),
3964 NEONMAP1(vcagtq_v, arm_neon_vacgt, 0),
3965 NEONMAP1(vcale_v, arm_neon_vacge, 0),
3966 NEONMAP1(vcaleq_v, arm_neon_vacge, 0),
3967 NEONMAP1(vcalt_v, arm_neon_vacgt, 0),
3968 NEONMAP1(vcaltq_v, arm_neon_vacgt, 0),
3985 NEONMAP1(vcvt_f16_f32, arm_neon_vcvtfp2hf, 0),
3987 NEONMAP1(vcvt_f32_f16, arm_neon_vcvthf2fp, 0),
3989 NEONMAP2(vcvt_n_f16_v, arm_neon_vcvtfxu2fp, arm_neon_vcvtfxs2fp, 0),
3990 NEONMAP2(vcvt_n_f32_v, arm_neon_vcvtfxu2fp, arm_neon_vcvtfxs2fp, 0),
3991 NEONMAP1(vcvt_n_s16_v, arm_neon_vcvtfp2fxs, 0),
3992 NEONMAP1(vcvt_n_s32_v, arm_neon_vcvtfp2fxs, 0),
3993 NEONMAP1(vcvt_n_s64_v, arm_neon_vcvtfp2fxs, 0),
3994 NEONMAP1(vcvt_n_u16_v, arm_neon_vcvtfp2fxu, 0),
3995 NEONMAP1(vcvt_n_u32_v, arm_neon_vcvtfp2fxu, 0),
3996 NEONMAP1(vcvt_n_u64_v, arm_neon_vcvtfp2fxu, 0),
4003 NEONMAP1(vcvta_s16_v, arm_neon_vcvtas, 0),
4004 NEONMAP1(vcvta_s32_v, arm_neon_vcvtas, 0),
4005 NEONMAP1(vcvta_s64_v, arm_neon_vcvtas, 0),
4006 NEONMAP1(vcvta_u16_v, arm_neon_vcvtau, 0),
4007 NEONMAP1(vcvta_u32_v, arm_neon_vcvtau, 0),
4008 NEONMAP1(vcvta_u64_v, arm_neon_vcvtau, 0),
4009 NEONMAP1(vcvtaq_s16_v, arm_neon_vcvtas, 0),
4010 NEONMAP1(vcvtaq_s32_v, arm_neon_vcvtas, 0),
4011 NEONMAP1(vcvtaq_s64_v, arm_neon_vcvtas, 0),
4012 NEONMAP1(vcvtaq_u16_v, arm_neon_vcvtau, 0),
4013 NEONMAP1(vcvtaq_u32_v, arm_neon_vcvtau, 0),
4014 NEONMAP1(vcvtaq_u64_v, arm_neon_vcvtau, 0),
4015 NEONMAP1(vcvtm_s16_v, arm_neon_vcvtms, 0),
4016 NEONMAP1(vcvtm_s32_v, arm_neon_vcvtms, 0),
4017 NEONMAP1(vcvtm_s64_v, arm_neon_vcvtms, 0),
4018 NEONMAP1(vcvtm_u16_v, arm_neon_vcvtmu, 0),
4019 NEONMAP1(vcvtm_u32_v, arm_neon_vcvtmu, 0),
4020 NEONMAP1(vcvtm_u64_v, arm_neon_vcvtmu, 0),
4021 NEONMAP1(vcvtmq_s16_v, arm_neon_vcvtms, 0),
4022 NEONMAP1(vcvtmq_s32_v, arm_neon_vcvtms, 0),
4023 NEONMAP1(vcvtmq_s64_v, arm_neon_vcvtms, 0),
4024 NEONMAP1(vcvtmq_u16_v, arm_neon_vcvtmu, 0),
4025 NEONMAP1(vcvtmq_u32_v, arm_neon_vcvtmu, 0),
4026 NEONMAP1(vcvtmq_u64_v, arm_neon_vcvtmu, 0),
4027 NEONMAP1(vcvtn_s16_v, arm_neon_vcvtns, 0),
4028 NEONMAP1(vcvtn_s32_v, arm_neon_vcvtns, 0),
4029 NEONMAP1(vcvtn_s64_v, arm_neon_vcvtns, 0),
4030 NEONMAP1(vcvtn_u16_v, arm_neon_vcvtnu, 0),
4031 NEONMAP1(vcvtn_u32_v, arm_neon_vcvtnu, 0),
4032 NEONMAP1(vcvtn_u64_v, arm_neon_vcvtnu, 0),
4033 NEONMAP1(vcvtnq_s16_v, arm_neon_vcvtns, 0),
4034 NEONMAP1(vcvtnq_s32_v, arm_neon_vcvtns, 0),
4035 NEONMAP1(vcvtnq_s64_v, arm_neon_vcvtns, 0),
4036 NEONMAP1(vcvtnq_u16_v, arm_neon_vcvtnu, 0),
4037 NEONMAP1(vcvtnq_u32_v, arm_neon_vcvtnu, 0),
4038 NEONMAP1(vcvtnq_u64_v, arm_neon_vcvtnu, 0),
4039 NEONMAP1(vcvtp_s16_v, arm_neon_vcvtps, 0),
4040 NEONMAP1(vcvtp_s32_v, arm_neon_vcvtps, 0),
4041 NEONMAP1(vcvtp_s64_v, arm_neon_vcvtps, 0),
4042 NEONMAP1(vcvtp_u16_v, arm_neon_vcvtpu, 0),
4043 NEONMAP1(vcvtp_u32_v, arm_neon_vcvtpu, 0),
4044 NEONMAP1(vcvtp_u64_v, arm_neon_vcvtpu, 0),
4045 NEONMAP1(vcvtpq_s16_v, arm_neon_vcvtps, 0),
4046 NEONMAP1(vcvtpq_s32_v, arm_neon_vcvtps, 0),
4047 NEONMAP1(vcvtpq_s64_v, arm_neon_vcvtps, 0),
4048 NEONMAP1(vcvtpq_u16_v, arm_neon_vcvtpu, 0),
4049 NEONMAP1(vcvtpq_u32_v, arm_neon_vcvtpu, 0),
4050 NEONMAP1(vcvtpq_u64_v, arm_neon_vcvtpu, 0),
4053 NEONMAP2(vcvtq_n_f16_v, arm_neon_vcvtfxu2fp, arm_neon_vcvtfxs2fp, 0),
4054 NEONMAP2(vcvtq_n_f32_v, arm_neon_vcvtfxu2fp, arm_neon_vcvtfxs2fp, 0),
4055 NEONMAP1(vcvtq_n_s16_v, arm_neon_vcvtfp2fxs, 0),
4056 NEONMAP1(vcvtq_n_s32_v, arm_neon_vcvtfp2fxs, 0),
4057 NEONMAP1(vcvtq_n_s64_v, arm_neon_vcvtfp2fxs, 0),
4058 NEONMAP1(vcvtq_n_u16_v, arm_neon_vcvtfp2fxu, 0),
4059 NEONMAP1(vcvtq_n_u32_v, arm_neon_vcvtfp2fxu, 0),
4060 NEONMAP1(vcvtq_n_u64_v, arm_neon_vcvtfp2fxu, 0),
4067 NEONMAP2(vdot_v, arm_neon_udot, arm_neon_sdot, 0),
4068 NEONMAP2(vdotq_v, arm_neon_udot, arm_neon_sdot, 0),
4078 NEONMAP1(vld1_v, arm_neon_vld1, 0),
4079 NEONMAP1(vld1_x2_v, arm_neon_vld1x2, 0),
4080 NEONMAP1(vld1_x3_v, arm_neon_vld1x3, 0),
4081 NEONMAP1(vld1_x4_v, arm_neon_vld1x4, 0),
4083 NEONMAP1(vld1q_v, arm_neon_vld1, 0),
4084 NEONMAP1(vld1q_x2_v, arm_neon_vld1x2, 0),
4085 NEONMAP1(vld1q_x3_v, arm_neon_vld1x3, 0),
4086 NEONMAP1(vld1q_x4_v, arm_neon_vld1x4, 0),
4087 NEONMAP1(vld2_dup_v, arm_neon_vld2dup, 0),
4088 NEONMAP1(vld2_lane_v, arm_neon_vld2lane, 0),
4089 NEONMAP1(vld2_v, arm_neon_vld2, 0),
4090 NEONMAP1(vld2q_dup_v, arm_neon_vld2dup, 0),
4091 NEONMAP1(vld2q_lane_v, arm_neon_vld2lane, 0),
4092 NEONMAP1(vld2q_v, arm_neon_vld2, 0),
4093 NEONMAP1(vld3_dup_v, arm_neon_vld3dup, 0),
4094 NEONMAP1(vld3_lane_v, arm_neon_vld3lane, 0),
4095 NEONMAP1(vld3_v, arm_neon_vld3, 0),
4096 NEONMAP1(vld3q_dup_v, arm_neon_vld3dup, 0),
4097 NEONMAP1(vld3q_lane_v, arm_neon_vld3lane, 0),
4098 NEONMAP1(vld3q_v, arm_neon_vld3, 0),
4099 NEONMAP1(vld4_dup_v, arm_neon_vld4dup, 0),
4100 NEONMAP1(vld4_lane_v, arm_neon_vld4lane, 0),
4101 NEONMAP1(vld4_v, arm_neon_vld4, 0),
4102 NEONMAP1(vld4q_dup_v, arm_neon_vld4dup, 0),
4103 NEONMAP1(vld4q_lane_v, arm_neon_vld4lane, 0),
4104 NEONMAP1(vld4q_v, arm_neon_vld4, 0),
4130 NEONMAP2(vqdmlal_v, arm_neon_vqdmull, arm_neon_vqadds, 0),
4131 NEONMAP2(vqdmlsl_v, arm_neon_vqdmull, arm_neon_vqsubs, 0),
4147 NEONMAP1(vqshlu_n_v, arm_neon_vqshiftsu, 0),
4148 NEONMAP1(vqshluq_n_v, arm_neon_vqshiftsu, 0),
4152 NEONMAP2(vrecpe_v, arm_neon_vrecpe, arm_neon_vrecpe, 0),
4153 NEONMAP2(vrecpeq_v, arm_neon_vrecpe, arm_neon_vrecpe, 0),
4176 NEONMAP2(vrsqrte_v, arm_neon_vrsqrte, arm_neon_vrsqrte, 0),
4177 NEONMAP2(vrsqrteq_v, arm_neon_vrsqrte, arm_neon_vrsqrte, 0),
4181 NEONMAP1(vsha1su0q_v, arm_neon_sha1su0, 0),
4182 NEONMAP1(vsha1su1q_v, arm_neon_sha1su1, 0),
4183 NEONMAP1(vsha256h2q_v, arm_neon_sha256h2, 0),
4184 NEONMAP1(vsha256hq_v, arm_neon_sha256h, 0),
4185 NEONMAP1(vsha256su0q_v, arm_neon_sha256su0, 0),
4186 NEONMAP1(vsha256su1q_v, arm_neon_sha256su1, 0),
4195 NEONMAP1(vst1_v, arm_neon_vst1, 0),
4196 NEONMAP1(vst1_x2_v, arm_neon_vst1x2, 0),
4197 NEONMAP1(vst1_x3_v, arm_neon_vst1x3, 0),
4198 NEONMAP1(vst1_x4_v, arm_neon_vst1x4, 0),
4199 NEONMAP1(vst1q_v, arm_neon_vst1, 0),
4200 NEONMAP1(vst1q_x2_v, arm_neon_vst1x2, 0),
4201 NEONMAP1(vst1q_x3_v, arm_neon_vst1x3, 0),
4202 NEONMAP1(vst1q_x4_v, arm_neon_vst1x4, 0),
4203 NEONMAP1(vst2_lane_v, arm_neon_vst2lane, 0),
4204 NEONMAP1(vst2_v, arm_neon_vst2, 0),
4205 NEONMAP1(vst2q_lane_v, arm_neon_vst2lane, 0),
4206 NEONMAP1(vst2q_v, arm_neon_vst2, 0),
4207 NEONMAP1(vst3_lane_v, arm_neon_vst3lane, 0),
4208 NEONMAP1(vst3_v, arm_neon_vst3, 0),
4209 NEONMAP1(vst3q_lane_v, arm_neon_vst3lane, 0),
4210 NEONMAP1(vst3q_v, arm_neon_vst3, 0),
4211 NEONMAP1(vst4_lane_v, arm_neon_vst4lane, 0),
4212 NEONMAP1(vst4_v, arm_neon_vst4, 0),
4213 NEONMAP1(vst4q_lane_v, arm_neon_vst4lane, 0),
4214 NEONMAP1(vst4q_v, arm_neon_vst4, 0),
4227 NEONMAP1(vabs_v, aarch64_neon_abs, 0),
4228 NEONMAP1(vabsq_v, aarch64_neon_abs, 0),
4230 NEONMAP1(vaesdq_v, aarch64_crypto_aesd, 0),
4231 NEONMAP1(vaeseq_v, aarch64_crypto_aese, 0),
4232 NEONMAP1(vaesimcq_v, aarch64_crypto_aesimc, 0),
4233 NEONMAP1(vaesmcq_v, aarch64_crypto_aesmc, 0),
4234 NEONMAP1(vcage_v, aarch64_neon_facge, 0),
4235 NEONMAP1(vcageq_v, aarch64_neon_facge, 0),
4236 NEONMAP1(vcagt_v, aarch64_neon_facgt, 0),
4237 NEONMAP1(vcagtq_v, aarch64_neon_facgt, 0),
4238 NEONMAP1(vcale_v, aarch64_neon_facge, 0),
4239 NEONMAP1(vcaleq_v, aarch64_neon_facge, 0),
4240 NEONMAP1(vcalt_v, aarch64_neon_facgt, 0),
4241 NEONMAP1(vcaltq_v, aarch64_neon_facgt, 0),
4258 NEONMAP1(vcvt_f16_f32, aarch64_neon_vcvtfp2hf, 0),
4260 NEONMAP1(vcvt_f32_f16, aarch64_neon_vcvthf2fp, 0),
4262 NEONMAP2(vcvt_n_f16_v, aarch64_neon_vcvtfxu2fp, aarch64_neon_vcvtfxs2fp, 0),
4263 NEONMAP2(vcvt_n_f32_v, aarch64_neon_vcvtfxu2fp, aarch64_neon_vcvtfxs2fp, 0),
4264 NEONMAP2(vcvt_n_f64_v, aarch64_neon_vcvtfxu2fp, aarch64_neon_vcvtfxs2fp, 0),
4265 NEONMAP1(vcvt_n_s16_v, aarch64_neon_vcvtfp2fxs, 0),
4266 NEONMAP1(vcvt_n_s32_v, aarch64_neon_vcvtfp2fxs, 0),
4267 NEONMAP1(vcvt_n_s64_v, aarch64_neon_vcvtfp2fxs, 0),
4268 NEONMAP1(vcvt_n_u16_v, aarch64_neon_vcvtfp2fxu, 0),
4269 NEONMAP1(vcvt_n_u32_v, aarch64_neon_vcvtfp2fxu, 0),
4270 NEONMAP1(vcvt_n_u64_v, aarch64_neon_vcvtfp2fxu, 0),
4273 NEONMAP2(vcvtq_n_f16_v, aarch64_neon_vcvtfxu2fp, aarch64_neon_vcvtfxs2fp, 0),
4274 NEONMAP2(vcvtq_n_f32_v, aarch64_neon_vcvtfxu2fp, aarch64_neon_vcvtfxs2fp, 0),
4275 NEONMAP2(vcvtq_n_f64_v, aarch64_neon_vcvtfxu2fp, aarch64_neon_vcvtfxs2fp, 0),
4276 NEONMAP1(vcvtq_n_s16_v, aarch64_neon_vcvtfp2fxs, 0),
4277 NEONMAP1(vcvtq_n_s32_v, aarch64_neon_vcvtfp2fxs, 0),
4278 NEONMAP1(vcvtq_n_s64_v, aarch64_neon_vcvtfp2fxs, 0),
4279 NEONMAP1(vcvtq_n_u16_v, aarch64_neon_vcvtfp2fxu, 0),
4280 NEONMAP1(vcvtq_n_u32_v, aarch64_neon_vcvtfp2fxu, 0),
4281 NEONMAP1(vcvtq_n_u64_v, aarch64_neon_vcvtfp2fxu, 0),
4283 NEONMAP2(vdot_v, aarch64_neon_udot, aarch64_neon_sdot, 0),
4284 NEONMAP2(vdotq_v, aarch64_neon_udot, aarch64_neon_sdot, 0),
4293 NEONMAP1(vld1_x2_v, aarch64_neon_ld1x2, 0),
4294 NEONMAP1(vld1_x3_v, aarch64_neon_ld1x3, 0),
4295 NEONMAP1(vld1_x4_v, aarch64_neon_ld1x4, 0),
4296 NEONMAP1(vld1q_x2_v, aarch64_neon_ld1x2, 0),
4297 NEONMAP1(vld1q_x3_v, aarch64_neon_ld1x3, 0),
4298 NEONMAP1(vld1q_x4_v, aarch64_neon_ld1x4, 0),
4311 NEONMAP2(vqdmlal_v, aarch64_neon_sqdmull, aarch64_neon_sqadd, 0),
4312 NEONMAP2(vqdmlsl_v, aarch64_neon_sqdmull, aarch64_neon_sqsub, 0),
4328 NEONMAP1(vqshlu_n_v, aarch64_neon_sqshlu, 0),
4329 NEONMAP1(vqshluq_n_v, aarch64_neon_sqshlu, 0),
4333 NEONMAP2(vrecpe_v, aarch64_neon_frecpe, aarch64_neon_urecpe, 0),
4334 NEONMAP2(vrecpeq_v, aarch64_neon_frecpe, aarch64_neon_urecpe, 0),
4345 NEONMAP2(vrsqrte_v, aarch64_neon_frsqrte, aarch64_neon_ursqrte, 0),
4346 NEONMAP2(vrsqrteq_v, aarch64_neon_frsqrte, aarch64_neon_ursqrte, 0),
4350 NEONMAP1(vsha1su0q_v, aarch64_crypto_sha1su0, 0),
4351 NEONMAP1(vsha1su1q_v, aarch64_crypto_sha1su1, 0),
4352 NEONMAP1(vsha256h2q_v, aarch64_crypto_sha256h2, 0),
4353 NEONMAP1(vsha256hq_v, aarch64_crypto_sha256h, 0),
4354 NEONMAP1(vsha256su0q_v, aarch64_crypto_sha256su0, 0),
4355 NEONMAP1(vsha256su1q_v, aarch64_crypto_sha256su1, 0),
4364 NEONMAP1(vst1_x2_v, aarch64_neon_st1x2, 0),
4365 NEONMAP1(vst1_x3_v, aarch64_neon_st1x3, 0),
4366 NEONMAP1(vst1_x4_v, aarch64_neon_st1x4, 0),
4367 NEONMAP1(vst1q_x2_v, aarch64_neon_st1x2, 0),
4368 NEONMAP1(vst1q_x3_v, aarch64_neon_st1x3, 0),
4369 NEONMAP1(vst1q_x4_v, aarch64_neon_st1x4, 0),
4424 NEONMAP1(vcvtxd_f32_f64, aarch64_sisd_fcvtxn, 0),
4445 NEONMAP1(vmull_p64, aarch64_neon_pmull64, 0),
4473 NEONMAP1(vqdmulls_s32, aarch64_neon_sqdmulls_scalar, 0),
4550 NEONMAP1(vsha1cq_u32, aarch64_crypto_sha1c, 0),
4551 NEONMAP1(vsha1h_u32, aarch64_crypto_sha1h, 0),
4552 NEONMAP1(vsha1mq_u32, aarch64_crypto_sha1m, 0),
4553 NEONMAP1(vsha1pq_u32, aarch64_crypto_sha1p, 0),
4611 static const NeonIntrinsicInfo *
4613 unsigned BuiltinID,
bool &MapProvenSorted) {
4616 if (!MapProvenSorted) {
4617 assert(std::is_sorted(std::begin(IntrinsicMap), std::end(IntrinsicMap)));
4618 MapProvenSorted =
true;
4622 const NeonIntrinsicInfo *Builtin =
4623 std::lower_bound(IntrinsicMap.begin(), IntrinsicMap.end(), BuiltinID);
4625 if (Builtin != IntrinsicMap.end() && Builtin->BuiltinID == BuiltinID)
4646 Ty = llvm::VectorType::get(
4647 Ty, VectorSize ? VectorSize / Ty->getPrimitiveSizeInBits() : 1);
4654 int Elts = VectorSize ? VectorSize / ArgType->getPrimitiveSizeInBits() : 1;
4655 ArgType = llvm::VectorType::get(ArgType, Elts);
4659 Tys.push_back(ArgType);
4662 Tys.push_back(ArgType);
4665 Tys.push_back(FloatTy);
4667 return CGM.getIntrinsic(IntrinsicID, Tys);
4671 const NeonIntrinsicInfo &SISDInfo,
4674 unsigned BuiltinID = SISDInfo.BuiltinID;
4675 unsigned int Int = SISDInfo.LLVMIntrinsic;
4676 unsigned Modifier = SISDInfo.TypeModifier;
4677 const char *s = SISDInfo.NameHint;
4679 switch (BuiltinID) {
4680 case NEON::BI__builtin_neon_vcled_s64:
4681 case NEON::BI__builtin_neon_vcled_u64:
4682 case NEON::BI__builtin_neon_vcles_f32:
4683 case NEON::BI__builtin_neon_vcled_f64:
4684 case NEON::BI__builtin_neon_vcltd_s64:
4685 case NEON::BI__builtin_neon_vcltd_u64:
4686 case NEON::BI__builtin_neon_vclts_f32:
4687 case NEON::BI__builtin_neon_vcltd_f64:
4688 case NEON::BI__builtin_neon_vcales_f32:
4689 case NEON::BI__builtin_neon_vcaled_f64:
4690 case NEON::BI__builtin_neon_vcalts_f32:
4691 case NEON::BI__builtin_neon_vcaltd_f64:
4695 std::swap(Ops[0], Ops[1]);
4699 assert(Int &&
"Generic code assumes a valid intrinsic");
4707 ConstantInt *C0 = ConstantInt::get(CGF.
SizeTy, 0);
4708 for (Function::const_arg_iterator ai = F->arg_begin(), ae = F->arg_end();
4709 ai != ae; ++ai, ++j) {
4711 if (Ops[j]->getType()->getPrimitiveSizeInBits() ==
4712 ArgTy->getPrimitiveSizeInBits())
4715 assert(ArgTy->isVectorTy() && !Ops[j]->getType()->isVectorTy());
4719 CGF.
Builder.CreateTruncOrBitCast(Ops[j], ArgTy->getVectorElementType());
4721 CGF.
Builder.CreateInsertElement(UndefValue::get(ArgTy), Ops[j], C0);
4726 if (ResultType->getPrimitiveSizeInBits() <
4727 Result->getType()->getPrimitiveSizeInBits())
4728 return CGF.
Builder.CreateExtractElement(Result, C0);
4734 unsigned BuiltinID,
unsigned LLVMIntrinsic,
unsigned AltLLVMIntrinsic,
4737 llvm::Triple::ArchType Arch) {
4739 llvm::APSInt NeonTypeConst;
4746 bool Usgn =
Type.isUnsigned();
4747 bool Quad =
Type.isQuad();
4748 const bool HasLegalHalfType = getTarget().hasLegalHalfType();
4755 auto getAlignmentValue32 = [&](
Address addr) ->
Value* {
4756 return Builder.getInt32(addr.getAlignment().getQuantity());
4759 unsigned Int = LLVMIntrinsic;
4761 Int = AltLLVMIntrinsic;
4763 switch (BuiltinID) {
4765 case NEON::BI__builtin_neon_vabs_v:
4766 case NEON::BI__builtin_neon_vabsq_v:
4767 if (VTy->getElementType()->isFloatingPointTy())
4768 return EmitNeonCall(CGM.getIntrinsic(
Intrinsic::fabs, Ty), Ops,
"vabs");
4769 return EmitNeonCall(CGM.getIntrinsic(LLVMIntrinsic, Ty), Ops,
"vabs");
4770 case NEON::BI__builtin_neon_vaddhn_v: {
4771 llvm::VectorType *SrcTy =
4772 llvm::VectorType::getExtendedElementVectorType(VTy);
4775 Ops[0] = Builder.CreateBitCast(Ops[0], SrcTy);
4776 Ops[1] = Builder.CreateBitCast(Ops[1], SrcTy);
4777 Ops[0] = Builder.CreateAdd(Ops[0], Ops[1],
"vaddhn");
4780 Constant *ShiftAmt =
4781 ConstantInt::get(SrcTy, SrcTy->getScalarSizeInBits() / 2);
4782 Ops[0] = Builder.CreateLShr(Ops[0], ShiftAmt,
"vaddhn");
4785 return Builder.CreateTrunc(Ops[0], VTy,
"vaddhn");
4787 case NEON::BI__builtin_neon_vcale_v:
4788 case NEON::BI__builtin_neon_vcaleq_v:
4789 case NEON::BI__builtin_neon_vcalt_v:
4790 case NEON::BI__builtin_neon_vcaltq_v:
4791 std::swap(Ops[0], Ops[1]);
4793 case NEON::BI__builtin_neon_vcage_v:
4794 case NEON::BI__builtin_neon_vcageq_v:
4795 case NEON::BI__builtin_neon_vcagt_v:
4796 case NEON::BI__builtin_neon_vcagtq_v: {
4798 switch (VTy->getScalarSizeInBits()) {
4799 default: llvm_unreachable(
"unexpected type");
4810 llvm::Type *VecFlt = llvm::VectorType::get(Ty, VTy->getNumElements());
4812 Function *F = CGM.getIntrinsic(LLVMIntrinsic, Tys);
4813 return EmitNeonCall(F, Ops, NameHint);
4815 case NEON::BI__builtin_neon_vceqz_v:
4816 case NEON::BI__builtin_neon_vceqzq_v:
4817 return EmitAArch64CompareBuiltinExpr(Ops[0], Ty, ICmpInst::FCMP_OEQ,
4818 ICmpInst::ICMP_EQ,
"vceqz");
4819 case NEON::BI__builtin_neon_vcgez_v:
4820 case NEON::BI__builtin_neon_vcgezq_v:
4821 return EmitAArch64CompareBuiltinExpr(Ops[0], Ty, ICmpInst::FCMP_OGE,
4822 ICmpInst::ICMP_SGE,
"vcgez");
4823 case NEON::BI__builtin_neon_vclez_v:
4824 case NEON::BI__builtin_neon_vclezq_v:
4825 return EmitAArch64CompareBuiltinExpr(Ops[0], Ty, ICmpInst::FCMP_OLE,
4826 ICmpInst::ICMP_SLE,
"vclez");
4827 case NEON::BI__builtin_neon_vcgtz_v:
4828 case NEON::BI__builtin_neon_vcgtzq_v:
4829 return EmitAArch64CompareBuiltinExpr(Ops[0], Ty, ICmpInst::FCMP_OGT,
4830 ICmpInst::ICMP_SGT,
"vcgtz");
4831 case NEON::BI__builtin_neon_vcltz_v:
4832 case NEON::BI__builtin_neon_vcltzq_v:
4833 return EmitAArch64CompareBuiltinExpr(Ops[0], Ty, ICmpInst::FCMP_OLT,
4834 ICmpInst::ICMP_SLT,
"vcltz");
4835 case NEON::BI__builtin_neon_vclz_v:
4836 case NEON::BI__builtin_neon_vclzq_v:
4839 Ops.push_back(Builder.getInt1(getTarget().isCLZForZeroUndef()));
4841 case NEON::BI__builtin_neon_vcvt_f32_v:
4842 case NEON::BI__builtin_neon_vcvtq_f32_v:
4843 Ops[0] = Builder.CreateBitCast(Ops[0], Ty);
4846 return Usgn ? Builder.CreateUIToFP(Ops[0], Ty,
"vcvt")
4847 : Builder.CreateSIToFP(Ops[0], Ty,
"vcvt");
4848 case NEON::BI__builtin_neon_vcvt_f16_v:
4849 case NEON::BI__builtin_neon_vcvtq_f16_v:
4850 Ops[0] = Builder.CreateBitCast(Ops[0], Ty);
4853 return Usgn ? Builder.CreateUIToFP(Ops[0], Ty,
"vcvt")
4854 : Builder.CreateSIToFP(Ops[0], Ty,
"vcvt");
4855 case NEON::BI__builtin_neon_vcvt_n_f16_v:
4856 case NEON::BI__builtin_neon_vcvt_n_f32_v:
4857 case NEON::BI__builtin_neon_vcvt_n_f64_v:
4858 case NEON::BI__builtin_neon_vcvtq_n_f16_v:
4859 case NEON::BI__builtin_neon_vcvtq_n_f32_v:
4860 case NEON::BI__builtin_neon_vcvtq_n_f64_v: {
4862 Int = Usgn ? LLVMIntrinsic : AltLLVMIntrinsic;
4863 Function *F = CGM.getIntrinsic(Int, Tys);
4864 return EmitNeonCall(F, Ops,
"vcvt_n");
4866 case NEON::BI__builtin_neon_vcvt_n_s16_v:
4867 case NEON::BI__builtin_neon_vcvt_n_s32_v:
4868 case NEON::BI__builtin_neon_vcvt_n_u16_v:
4869 case NEON::BI__builtin_neon_vcvt_n_u32_v:
4870 case NEON::BI__builtin_neon_vcvt_n_s64_v:
4871 case NEON::BI__builtin_neon_vcvt_n_u64_v:
4872 case NEON::BI__builtin_neon_vcvtq_n_s16_v:
4873 case NEON::BI__builtin_neon_vcvtq_n_s32_v:
4874 case NEON::BI__builtin_neon_vcvtq_n_u16_v:
4875 case NEON::BI__builtin_neon_vcvtq_n_u32_v:
4876 case NEON::BI__builtin_neon_vcvtq_n_s64_v:
4877 case NEON::BI__builtin_neon_vcvtq_n_u64_v: {
4879 Function *F = CGM.getIntrinsic(LLVMIntrinsic, Tys);
4880 return EmitNeonCall(F, Ops,
"vcvt_n");
4882 case NEON::BI__builtin_neon_vcvt_s32_v:
4883 case NEON::BI__builtin_neon_vcvt_u32_v:
4884 case NEON::BI__builtin_neon_vcvt_s64_v:
4885 case NEON::BI__builtin_neon_vcvt_u64_v:
4886 case NEON::BI__builtin_neon_vcvt_s16_v:
4887 case NEON::BI__builtin_neon_vcvt_u16_v:
4888 case NEON::BI__builtin_neon_vcvtq_s32_v:
4889 case NEON::BI__builtin_neon_vcvtq_u32_v:
4890 case NEON::BI__builtin_neon_vcvtq_s64_v:
4891 case NEON::BI__builtin_neon_vcvtq_u64_v:
4892 case NEON::BI__builtin_neon_vcvtq_s16_v:
4893 case NEON::BI__builtin_neon_vcvtq_u16_v: {
4895 return Usgn ? Builder.CreateFPToUI(Ops[0], Ty,
"vcvt")
4896 : Builder.CreateFPToSI(Ops[0], Ty,
"vcvt");
4898 case NEON::BI__builtin_neon_vcvta_s16_v:
4899 case NEON::BI__builtin_neon_vcvta_s32_v:
4900 case NEON::BI__builtin_neon_vcvta_s64_v:
4901 case NEON::BI__builtin_neon_vcvta_u16_v:
4902 case NEON::BI__builtin_neon_vcvta_u32_v:
4903 case NEON::BI__builtin_neon_vcvta_u64_v:
4904 case NEON::BI__builtin_neon_vcvtaq_s16_v:
4905 case NEON::BI__builtin_neon_vcvtaq_s32_v:
4906 case NEON::BI__builtin_neon_vcvtaq_s64_v:
4907 case NEON::BI__builtin_neon_vcvtaq_u16_v:
4908 case NEON::BI__builtin_neon_vcvtaq_u32_v:
4909 case NEON::BI__builtin_neon_vcvtaq_u64_v:
4910 case NEON::BI__builtin_neon_vcvtn_s16_v:
4911 case NEON::BI__builtin_neon_vcvtn_s32_v:
4912 case NEON::BI__builtin_neon_vcvtn_s64_v:
4913 case NEON::BI__builtin_neon_vcvtn_u16_v:
4914 case NEON::BI__builtin_neon_vcvtn_u32_v:
4915 case NEON::BI__builtin_neon_vcvtn_u64_v:
4916 case NEON::BI__builtin_neon_vcvtnq_s16_v:
4917 case NEON::BI__builtin_neon_vcvtnq_s32_v:
4918 case NEON::BI__builtin_neon_vcvtnq_s64_v:
4919 case NEON::BI__builtin_neon_vcvtnq_u16_v:
4920 case NEON::BI__builtin_neon_vcvtnq_u32_v:
4921 case NEON::BI__builtin_neon_vcvtnq_u64_v:
4922 case NEON::BI__builtin_neon_vcvtp_s16_v:
4923 case NEON::BI__builtin_neon_vcvtp_s32_v:
4924 case NEON::BI__builtin_neon_vcvtp_s64_v:
4925 case NEON::BI__builtin_neon_vcvtp_u16_v:
4926 case NEON::BI__builtin_neon_vcvtp_u32_v:
4927 case NEON::BI__builtin_neon_vcvtp_u64_v:
4928 case NEON::BI__builtin_neon_vcvtpq_s16_v:
4929 case NEON::BI__builtin_neon_vcvtpq_s32_v:
4930 case NEON::BI__builtin_neon_vcvtpq_s64_v:
4931 case NEON::BI__builtin_neon_vcvtpq_u16_v:
4932 case NEON::BI__builtin_neon_vcvtpq_u32_v:
4933 case NEON::BI__builtin_neon_vcvtpq_u64_v:
4934 case NEON::BI__builtin_neon_vcvtm_s16_v:
4935 case NEON::BI__builtin_neon_vcvtm_s32_v:
4936 case NEON::BI__builtin_neon_vcvtm_s64_v:
4937 case NEON::BI__builtin_neon_vcvtm_u16_v:
4938 case NEON::BI__builtin_neon_vcvtm_u32_v:
4939 case NEON::BI__builtin_neon_vcvtm_u64_v:
4940 case NEON::BI__builtin_neon_vcvtmq_s16_v:
4941 case NEON::BI__builtin_neon_vcvtmq_s32_v:
4942 case NEON::BI__builtin_neon_vcvtmq_s64_v:
4943 case NEON::BI__builtin_neon_vcvtmq_u16_v:
4944 case NEON::BI__builtin_neon_vcvtmq_u32_v:
4945 case NEON::BI__builtin_neon_vcvtmq_u64_v: {
4947 return EmitNeonCall(CGM.getIntrinsic(LLVMIntrinsic, Tys), Ops, NameHint);
4949 case NEON::BI__builtin_neon_vext_v:
4950 case NEON::BI__builtin_neon_vextq_v: {
4951 int CV = cast<ConstantInt>(Ops[2])->getSExtValue();
4953 for (
unsigned i = 0, e = VTy->getNumElements(); i != e; ++i)
4954 Indices.push_back(i+CV);
4956 Ops[0] = Builder.CreateBitCast(Ops[0], Ty);
4957 Ops[1] = Builder.CreateBitCast(Ops[1], Ty);
4958 return Builder.CreateShuffleVector(Ops[0], Ops[1], Indices,
"vext");
4960 case NEON::BI__builtin_neon_vfma_v:
4961 case NEON::BI__builtin_neon_vfmaq_v: {
4963 Ops[0] = Builder.CreateBitCast(Ops[0], Ty);
4964 Ops[1] = Builder.CreateBitCast(Ops[1], Ty);
4965 Ops[2] = Builder.CreateBitCast(Ops[2], Ty);
4968 return Builder.CreateCall(F, {Ops[1], Ops[2], Ops[0]});
4970 case NEON::BI__builtin_neon_vld1_v:
4971 case NEON::BI__builtin_neon_vld1q_v: {
4973 Ops.push_back(getAlignmentValue32(PtrOp0));
4974 return EmitNeonCall(CGM.getIntrinsic(LLVMIntrinsic, Tys), Ops,
"vld1");
4976 case NEON::BI__builtin_neon_vld1_x2_v:
4977 case NEON::BI__builtin_neon_vld1q_x2_v:
4978 case NEON::BI__builtin_neon_vld1_x3_v:
4979 case NEON::BI__builtin_neon_vld1q_x3_v:
4980 case NEON::BI__builtin_neon_vld1_x4_v:
4981 case NEON::BI__builtin_neon_vld1q_x4_v: {
4982 llvm::Type *PTy = llvm::PointerType::getUnqual(VTy->getVectorElementType());
4983 Ops[1] = Builder.CreateBitCast(Ops[1], PTy);
4985 Function *F = CGM.getIntrinsic(LLVMIntrinsic, Tys);
4986 Ops[1] = Builder.CreateCall(F, Ops[1],
"vld1xN");
4987 Ty = llvm::PointerType::getUnqual(Ops[1]->getType());
4988 Ops[0] = Builder.CreateBitCast(Ops[0], Ty);
4989 return Builder.CreateDefaultAlignedStore(Ops[1], Ops[0]);
4991 case NEON::BI__builtin_neon_vld2_v:
4992 case NEON::BI__builtin_neon_vld2q_v:
4993 case NEON::BI__builtin_neon_vld3_v:
4994 case NEON::BI__builtin_neon_vld3q_v:
4995 case NEON::BI__builtin_neon_vld4_v:
4996 case NEON::BI__builtin_neon_vld4q_v:
4997 case NEON::BI__builtin_neon_vld2_dup_v:
4998 case NEON::BI__builtin_neon_vld2q_dup_v:
4999 case NEON::BI__builtin_neon_vld3_dup_v:
5000 case NEON::BI__builtin_neon_vld3q_dup_v:
5001 case NEON::BI__builtin_neon_vld4_dup_v:
5002 case NEON::BI__builtin_neon_vld4q_dup_v: {
5004 Function *F = CGM.getIntrinsic(LLVMIntrinsic, Tys);
5005 Value *Align = getAlignmentValue32(PtrOp1);
5006 Ops[1] = Builder.CreateCall(F, {Ops[1], Align}, NameHint);
5007 Ty = llvm::PointerType::getUnqual(Ops[1]->getType());
5008 Ops[0] = Builder.CreateBitCast(Ops[0], Ty);
5009 return Builder.CreateDefaultAlignedStore(Ops[1], Ops[0]);
5011 case NEON::BI__builtin_neon_vld1_dup_v:
5012 case NEON::BI__builtin_neon_vld1q_dup_v: {
5013 Value *V = UndefValue::get(Ty);
5014 Ty = llvm::PointerType::getUnqual(VTy->getElementType());
5015 PtrOp0 = Builder.CreateBitCast(PtrOp0, Ty);
5016 LoadInst *Ld = Builder.CreateLoad(PtrOp0);
5017 llvm::Constant *CI = ConstantInt::get(SizeTy, 0);
5018 Ops[0] = Builder.CreateInsertElement(V, Ld, CI);
5019 return EmitNeonSplat(Ops[0], CI);
5021 case NEON::BI__builtin_neon_vld2_lane_v:
5022 case NEON::BI__builtin_neon_vld2q_lane_v:
5023 case NEON::BI__builtin_neon_vld3_lane_v:
5024 case NEON::BI__builtin_neon_vld3q_lane_v:
5025 case NEON::BI__builtin_neon_vld4_lane_v:
5026 case NEON::BI__builtin_neon_vld4q_lane_v: {
5028 Function *F = CGM.getIntrinsic(LLVMIntrinsic, Tys);
5029 for (
unsigned I = 2; I < Ops.size() - 1; ++I)
5030 Ops[I] = Builder.CreateBitCast(Ops[I], Ty);
5031 Ops.push_back(getAlignmentValue32(PtrOp1));
5032 Ops[1] = Builder.CreateCall(F, makeArrayRef(Ops).slice(1), NameHint);
5033 Ty = llvm::PointerType::getUnqual(Ops[1]->getType());
5034 Ops[0] = Builder.CreateBitCast(Ops[0], Ty);
5035 return Builder.CreateDefaultAlignedStore(Ops[1], Ops[0]);
5037 case NEON::BI__builtin_neon_vmovl_v: {
5038 llvm::Type *DTy =llvm::VectorType::getTruncatedElementVectorType(VTy);
5039 Ops[0] = Builder.CreateBitCast(Ops[0], DTy);
5041 return Builder.CreateZExt(Ops[0], Ty,
"vmovl");
5042 return Builder.CreateSExt(Ops[0], Ty,
"vmovl");
5044 case NEON::BI__builtin_neon_vmovn_v: {
5045 llvm::Type *QTy = llvm::VectorType::getExtendedElementVectorType(VTy);
5046 Ops[0] = Builder.CreateBitCast(Ops[0], QTy);
5047 return Builder.CreateTrunc(Ops[0], Ty,
"vmovn");
5049 case NEON::BI__builtin_neon_vmull_v:
5055 Int = Usgn ? Intrinsic::arm_neon_vmullu : Intrinsic::arm_neon_vmulls;
5056 Int =
Type.isPoly() ? (unsigned)Intrinsic::arm_neon_vmullp : Int;
5057 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops,
"vmull");
5058 case NEON::BI__builtin_neon_vpadal_v:
5059 case NEON::BI__builtin_neon_vpadalq_v: {
5061 unsigned EltBits = VTy->getElementType()->getPrimitiveSizeInBits();
5063 llvm::IntegerType::get(getLLVMContext(), EltBits / 2);
5065 llvm::VectorType::get(EltTy, VTy->getNumElements() * 2);
5067 return EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, NameHint);
5069 case NEON::BI__builtin_neon_vpaddl_v:
5070 case NEON::BI__builtin_neon_vpaddlq_v: {
5072 unsigned EltBits = VTy->getElementType()->getPrimitiveSizeInBits();
5073 llvm::Type *EltTy = llvm::IntegerType::get(getLLVMContext(), EltBits / 2);
5075 llvm::VectorType::get(EltTy, VTy->getNumElements() * 2);
5077 return EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops,
"vpaddl");
5079 case NEON::BI__builtin_neon_vqdmlal_v:
5080 case NEON::BI__builtin_neon_vqdmlsl_v: {
5083 EmitNeonCall(CGM.getIntrinsic(LLVMIntrinsic, Ty), MulOps,
"vqdmlal");
5085 return EmitNeonCall(CGM.getIntrinsic(AltLLVMIntrinsic, Ty), Ops, NameHint);
5087 case NEON::BI__builtin_neon_vqshl_n_v:
5088 case NEON::BI__builtin_neon_vqshlq_n_v:
5089 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops,
"vqshl_n",
5091 case NEON::BI__builtin_neon_vqshlu_n_v:
5092 case NEON::BI__builtin_neon_vqshluq_n_v:
5093 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops,
"vqshlu_n",
5095 case NEON::BI__builtin_neon_vrecpe_v:
5096 case NEON::BI__builtin_neon_vrecpeq_v:
5097 case NEON::BI__builtin_neon_vrsqrte_v:
5098 case NEON::BI__builtin_neon_vrsqrteq_v:
5099 Int = Ty->isFPOrFPVectorTy() ? LLVMIntrinsic : AltLLVMIntrinsic;
5100 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, NameHint);
5101 case NEON::BI__builtin_neon_vrndi_v:
5102 case NEON::BI__builtin_neon_vrndiq_v:
5104 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, NameHint);
5105 case NEON::BI__builtin_neon_vrshr_n_v:
5106 case NEON::BI__builtin_neon_vrshrq_n_v:
5107 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops,
"vrshr_n",
5109 case NEON::BI__builtin_neon_vshl_n_v:
5110 case NEON::BI__builtin_neon_vshlq_n_v:
5111 Ops[1] = EmitNeonShiftVector(Ops[1], Ty,
false);
5112 return Builder.CreateShl(Builder.CreateBitCast(Ops[0],Ty), Ops[1],
5114 case NEON::BI__builtin_neon_vshll_n_v: {
5115 llvm::Type *SrcTy = llvm::VectorType::getTruncatedElementVectorType(VTy);
5116 Ops[0] = Builder.CreateBitCast(Ops[0], SrcTy);
5118 Ops[0] = Builder.CreateZExt(Ops[0], VTy);
5120 Ops[0] = Builder.CreateSExt(Ops[0], VTy);
5121 Ops[1] = EmitNeonShiftVector(Ops[1], VTy,
false);
5122 return Builder.CreateShl(Ops[0], Ops[1],
"vshll_n");
5124 case NEON::BI__builtin_neon_vshrn_n_v: {
5125 llvm::Type *SrcTy = llvm::VectorType::getExtendedElementVectorType(VTy);
5126 Ops[0] = Builder.CreateBitCast(Ops[0], SrcTy);
5127 Ops[1] = EmitNeonShiftVector(Ops[1], SrcTy,
false);
5129 Ops[0] = Builder.CreateLShr(Ops[0], Ops[1]);
5131 Ops[0] = Builder.CreateAShr(Ops[0], Ops[1]);
5132 return Builder.CreateTrunc(Ops[0], Ty,
"vshrn_n");
5134 case NEON::BI__builtin_neon_vshr_n_v:
5135 case NEON::BI__builtin_neon_vshrq_n_v:
5136 return EmitNeonRShiftImm(Ops[0], Ops[1], Ty, Usgn,
"vshr_n");
5137 case NEON::BI__builtin_neon_vst1_v:
5138 case NEON::BI__builtin_neon_vst1q_v:
5139 case NEON::BI__builtin_neon_vst2_v:
5140 case NEON::BI__builtin_neon_vst2q_v:
5141 case NEON::BI__builtin_neon_vst3_v:
5142 case NEON::BI__builtin_neon_vst3q_v:
5143 case NEON::BI__builtin_neon_vst4_v:
5144 case NEON::BI__builtin_neon_vst4q_v:
5145 case NEON::BI__builtin_neon_vst2_lane_v:
5146 case NEON::BI__builtin_neon_vst2q_lane_v:
5147 case NEON::BI__builtin_neon_vst3_lane_v:
5148 case NEON::BI__builtin_neon_vst3q_lane_v:
5149 case NEON::BI__builtin_neon_vst4_lane_v:
5150 case NEON::BI__builtin_neon_vst4q_lane_v: {
5152 Ops.push_back(getAlignmentValue32(PtrOp0));
5153 return EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops,
"");
5155 case NEON::BI__builtin_neon_vst1_x2_v:
5156 case NEON::BI__builtin_neon_vst1q_x2_v:
5157 case NEON::BI__builtin_neon_vst1_x3_v:
5158 case NEON::BI__builtin_neon_vst1q_x3_v:
5159 case NEON::BI__builtin_neon_vst1_x4_v:
5160 case NEON::BI__builtin_neon_vst1q_x4_v: {
5161 llvm::Type *PTy = llvm::PointerType::getUnqual(VTy->getVectorElementType());
5164 if (Arch == llvm::Triple::aarch64 || Arch == llvm::Triple::aarch64_be) {
5166 std::rotate(Ops.begin(), Ops.begin() + 1, Ops.end());
5167 return EmitNeonCall(CGM.getIntrinsic(LLVMIntrinsic, Tys), Ops,
"");
5170 return EmitNeonCall(CGM.getIntrinsic(LLVMIntrinsic, Tys), Ops,
"");
5172 case NEON::BI__builtin_neon_vsubhn_v: {
5173 llvm::VectorType *SrcTy =
5174 llvm::VectorType::getExtendedElementVectorType(VTy);
5177 Ops[0] = Builder.CreateBitCast(Ops[0], SrcTy);
5178 Ops[1] = Builder.CreateBitCast(Ops[1], SrcTy);
5179 Ops[0] = Builder.CreateSub(Ops[0], Ops[1],
"vsubhn");
5182 Constant *ShiftAmt =
5183 ConstantInt::get(SrcTy, SrcTy->getScalarSizeInBits() / 2);
5184 Ops[0] = Builder.CreateLShr(Ops[0], ShiftAmt,
"vsubhn");
5187 return Builder.CreateTrunc(Ops[0], VTy,
"vsubhn");
5189 case NEON::BI__builtin_neon_vtrn_v:
5190 case NEON::BI__builtin_neon_vtrnq_v: {
5191 Ops[0] = Builder.CreateBitCast(Ops[0], llvm::PointerType::getUnqual(Ty));
5192 Ops[1] = Builder.CreateBitCast(Ops[1], Ty);
5193 Ops[2] = Builder.CreateBitCast(Ops[2], Ty);
5194 Value *SV =
nullptr;
5196 for (
unsigned vi = 0; vi != 2; ++vi) {
5198 for (
unsigned i = 0, e = VTy->getNumElements(); i != e; i += 2) {
5199 Indices.push_back(i+vi);
5200 Indices.push_back(i+e+vi);
5202 Value *Addr = Builder.CreateConstInBoundsGEP1_32(Ty, Ops[0], vi);
5203 SV = Builder.CreateShuffleVector(Ops[1], Ops[2], Indices,
"vtrn");
5204 SV = Builder.CreateDefaultAlignedStore(SV, Addr);
5208 case NEON::BI__builtin_neon_vtst_v:
5209 case NEON::BI__builtin_neon_vtstq_v: {
5210 Ops[0] = Builder.CreateBitCast(Ops[0], Ty);
5211 Ops[1] = Builder.CreateBitCast(Ops[1], Ty);
5212 Ops[0] = Builder.CreateAnd(Ops[0], Ops[1]);
5213 Ops[0] = Builder.CreateICmp(ICmpInst::ICMP_NE, Ops[0],
5214 ConstantAggregateZero::get(Ty));
5215 return Builder.CreateSExt(Ops[0], Ty,
"vtst");
5217 case NEON::BI__builtin_neon_vuzp_v:
5218 case NEON::BI__builtin_neon_vuzpq_v: {
5219 Ops[0] = Builder.CreateBitCast(Ops[0], llvm::PointerType::getUnqual(Ty));
5220 Ops[1] = Builder.CreateBitCast(Ops[1], Ty);
5221 Ops[2] = Builder.CreateBitCast(Ops[2], Ty);
5222 Value *SV =
nullptr;
5224 for (
unsigned vi = 0; vi != 2; ++vi) {
5226 for (
unsigned i = 0, e = VTy->getNumElements(); i != e; ++i)
5227 Indices.push_back(2*i+vi);
5229 Value *Addr = Builder.CreateConstInBoundsGEP1_32(Ty, Ops[0], vi);
5230 SV = Builder.CreateShuffleVector(Ops[1], Ops[2], Indices,
"vuzp");
5231 SV = Builder.CreateDefaultAlignedStore(SV, Addr);
5235 case NEON::BI__builtin_neon_vzip_v:
5236 case NEON::BI__builtin_neon_vzipq_v: {
5237 Ops[0] = Builder.CreateBitCast(Ops[0], llvm::PointerType::getUnqual(Ty));
5238 Ops[1] = Builder.CreateBitCast(Ops[1], Ty);
5239 Ops[2] = Builder.CreateBitCast(Ops[2], Ty);
5240 Value *SV =
nullptr;
5242 for (
unsigned vi = 0; vi != 2; ++vi) {
5244 for (
unsigned i = 0, e = VTy->getNumElements(); i != e; i += 2) {
5245 Indices.push_back((i + vi*e) >> 1);
5246 Indices.push_back(((i + vi*e) >> 1)+e);
5248 Value *Addr = Builder.CreateConstInBoundsGEP1_32(Ty, Ops[0], vi);
5249 SV = Builder.CreateShuffleVector(Ops[1], Ops[2], Indices,
"vzip");
5250 SV = Builder.CreateDefaultAlignedStore(SV, Addr);
5254 case NEON::BI__builtin_neon_vdot_v:
5255 case NEON::BI__builtin_neon_vdotq_v: {
5257 llvm::VectorType::get(Int8Ty, Ty->getPrimitiveSizeInBits() / 8);
5259 Int = Usgn ? LLVMIntrinsic : AltLLVMIntrinsic;
5260 return EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops,
"vdot");
5264 assert(Int &&
"Expected valid intrinsic number");
5267 Function *F = LookupNeonLLVMIntrinsic(Int, Modifier, Ty, E);
5269 Value *Result = EmitNeonCall(F, Ops, NameHint);
5273 return Builder.CreateBitCast(Result, ResultType, NameHint);
5278 const CmpInst::Predicate Ip,
const Twine &Name) {
5285 if (BitCastInst *BI = dyn_cast<BitCastInst>(Op))
5286 OTy = BI->getOperand(0)->getType();
5288 Op = Builder.CreateBitCast(Op, OTy);
5289 if (OTy->getScalarType()->isFloatingPointTy()) {
5290 Op = Builder.CreateFCmp(Fp, Op, Constant::getNullValue(OTy));
5292 Op = Builder.CreateICmp(Ip, Op, Constant::getNullValue(OTy));
5294 return Builder.CreateSExt(Op, Ty, Name);
5303 TblOps.push_back(ExtOp);
5307 llvm::VectorType *TblTy = cast<llvm::VectorType>(Ops[0]->getType());
5308 for (
unsigned i = 0, e = TblTy->getNumElements(); i != e; ++i) {
5309 Indices.push_back(2*i);
5310 Indices.push_back(2*i+1);
5313 int PairPos = 0,
End = Ops.size() - 1;
5314 while (PairPos <
End) {
5315 TblOps.push_back(CGF.
Builder.CreateShuffleVector(Ops[PairPos],
5316 Ops[PairPos+1], Indices,
5323 if (PairPos ==
End) {
5324 Value *ZeroTbl = ConstantAggregateZero::get(TblTy);
5325 TblOps.push_back(CGF.
Builder.CreateShuffleVector(Ops[PairPos],
5326 ZeroTbl, Indices, Name));
5330 TblOps.push_back(IndexOp);
5336 Value *CodeGenFunction::GetValueForARMHint(
unsigned BuiltinID) {
5338 switch (BuiltinID) {
5341 case ARM::BI__builtin_arm_nop:
5344 case ARM::BI__builtin_arm_yield:
5345 case ARM::BI__yield:
5348 case ARM::BI__builtin_arm_wfe:
5352 case ARM::BI__builtin_arm_wfi:
5356 case ARM::BI__builtin_arm_sev:
5360 case ARM::BI__builtin_arm_sevl:
5366 return Builder.CreateCall(CGM.getIntrinsic(Intrinsic::arm_hint),
5367 llvm::ConstantInt::get(Int32Ty, Value));
5378 StringRef SysReg =
"") {
5380 assert((RegisterType->isIntegerTy(32) || RegisterType->isIntegerTy(64))
5381 &&
"Unsupported size for register.");
5387 if (SysReg.empty()) {
5389 SysReg = cast<clang::StringLiteral>(SysRegStrExpr)->getString();
5392 llvm::Metadata *Ops[] = { llvm::MDString::get(Context, SysReg) };
5393 llvm::MDNode *RegName = llvm::MDNode::get(Context, Ops);
5394 llvm::Value *Metadata = llvm::MetadataAsValue::get(Context, RegName);
5398 bool MixedTypes = RegisterType->isIntegerTy(64) && ValueType->isIntegerTy(32);
5399 assert(!(RegisterType->isIntegerTy(32) && ValueType->isIntegerTy(64))
5400 &&
"Can't fit 64-bit value in 32-bit register");
5404 llvm::Value *Call = Builder.CreateCall(F, Metadata);
5408 return Builder.CreateTrunc(Call, ValueType);
5410 if (ValueType->isPointerTy())
5412 return Builder.CreateIntToPtr(Call, ValueType);
5421 ArgValue = Builder.CreateZExt(ArgValue, RegisterType);
5422 return Builder.CreateCall(F, { Metadata, ArgValue });
5425 if (ValueType->isPointerTy()) {
5427 ArgValue = Builder.CreatePtrToInt(ArgValue, RegisterType);
5428 return Builder.CreateCall(F, { Metadata, ArgValue });
5431 return Builder.CreateCall(F, { Metadata, ArgValue });
5437 switch (BuiltinID) {
5439 case NEON::BI__builtin_neon_vget_lane_i8:
5440 case NEON::BI__builtin_neon_vget_lane_i16:
5441 case NEON::BI__builtin_neon_vget_lane_i32:
5442 case NEON::BI__builtin_neon_vget_lane_i64:
5443 case NEON::BI__builtin_neon_vget_lane_f32:
5444 case NEON::BI__builtin_neon_vgetq_lane_i8:
5445 case NEON::BI__builtin_neon_vgetq_lane_i16:
5446 case NEON::BI__builtin_neon_vgetq_lane_i32:
5447 case NEON::BI__builtin_neon_vgetq_lane_i64:
5448 case NEON::BI__builtin_neon_vgetq_lane_f32:
5449 case NEON::BI__builtin_neon_vset_lane_i8:
5450 case NEON::BI__builtin_neon_vset_lane_i16:
5451 case NEON::BI__builtin_neon_vset_lane_i32:
5452 case NEON::BI__builtin_neon_vset_lane_i64:
5453 case NEON::BI__builtin_neon_vset_lane_f32:
5454 case NEON::BI__builtin_neon_vsetq_lane_i8:
5455 case NEON::BI__builtin_neon_vsetq_lane_i16:
5456 case NEON::BI__builtin_neon_vsetq_lane_i32:
5457 case NEON::BI__builtin_neon_vsetq_lane_i64:
5458 case NEON::BI__builtin_neon_vsetq_lane_f32:
5459 case NEON::BI__builtin_neon_vsha1h_u32:
5460 case NEON::BI__builtin_neon_vsha1cq_u32:
5461 case NEON::BI__builtin_neon_vsha1pq_u32:
5462 case NEON::BI__builtin_neon_vsha1mq_u32:
5463 case clang::ARM::BI_MoveToCoprocessor:
5464 case clang::ARM::BI_MoveToCoprocessor2:
5473 CharUnits LoadSize = getContext().getTypeSizeInChars(ElTy);
5474 llvm::Type *ITy = llvm::IntegerType::get(getLLVMContext(),
5476 Ptr = Builder.CreateBitCast(Ptr, ITy->getPointerTo());
5477 llvm::LoadInst *Load =
5478 Builder.CreateAlignedLoad(Ptr, LoadSize);
5479 Load->setVolatile(
true);
5487 CharUnits StoreSize = getContext().getTypeSizeInChars(ElTy);
5488 llvm::Type *ITy = llvm::IntegerType::get(getLLVMContext(),
5490 Ptr = Builder.CreateBitCast(Ptr, ITy->getPointerTo());
5491 llvm::StoreInst *
Store =
5492 Builder.CreateAlignedStore(Value, Ptr,
5494 Store->setVolatile(
true);
5500 llvm::Triple::ArchType Arch) {
5501 if (
auto Hint = GetValueForARMHint(BuiltinID))
5504 if (BuiltinID == ARM::BI__emit) {
5505 bool IsThumb = getTarget().getTriple().getArch() == llvm::Triple::thumb;
5506 llvm::FunctionType *FTy =
5507 llvm::FunctionType::get(VoidTy,
false);
5511 llvm_unreachable(
"Sema will ensure that the parameter is constant");
5513 uint64_t ZExtValue = Value.zextOrTrunc(IsThumb ? 16 : 32).getZExtValue();
5515 llvm::InlineAsm *Emit =
5516 IsThumb ? InlineAsm::get(FTy,
".inst.n 0x" + utohexstr(ZExtValue),
"",
5518 : InlineAsm::get(FTy,
".inst 0x" + utohexstr(ZExtValue),
"",
5521 return Builder.CreateCall(Emit);
5524 if (BuiltinID == ARM::BI__builtin_arm_dbg) {
5526 return Builder.CreateCall(CGM.getIntrinsic(Intrinsic::arm_dbg), Option);
5529 if (BuiltinID == ARM::BI__builtin_arm_prefetch) {
5535 Value *Locality = llvm::ConstantInt::get(Int32Ty, 3);
5538 return Builder.CreateCall(F, {Address, RW, Locality, IsData});
5541 if (BuiltinID == ARM::BI__builtin_arm_rbit) {
5543 return Builder.CreateCall(
5544 CGM.getIntrinsic(Intrinsic::bitreverse, Arg->getType()), Arg,
"rbit");
5547 if (BuiltinID == ARM::BI__clear_cache) {
5548 assert(E->
getNumArgs() == 2 &&
"__clear_cache takes 2 arguments");
5551 for (
unsigned i = 0; i < 2; i++)
5552 Ops[i] = EmitScalarExpr(E->
getArg(i));
5553 llvm::Type *Ty = CGM.getTypes().ConvertType(FD->getType());
5554 llvm::FunctionType *FTy = cast<llvm::FunctionType>(Ty);
5555 StringRef Name = FD->getName();
5556 return EmitNounwindRuntimeCall(CGM.CreateRuntimeFunction(FTy, Name), Ops);
5559 if (BuiltinID == ARM::BI__builtin_arm_mcrr ||
5560 BuiltinID == ARM::BI__builtin_arm_mcrr2) {
5563 switch (BuiltinID) {
5564 default: llvm_unreachable(
"unexpected builtin");
5565 case ARM::BI__builtin_arm_mcrr:
5566 F = CGM.getIntrinsic(Intrinsic::arm_mcrr);
5568 case ARM::BI__builtin_arm_mcrr2:
5569 F = CGM.getIntrinsic(Intrinsic::arm_mcrr2);
5585 Value *C1 = llvm::ConstantInt::get(Int64Ty, 32);
5586 Value *Rt = Builder.CreateTruncOrBitCast(RtAndRt2, Int32Ty);
5587 Value *Rt2 = Builder.CreateLShr(RtAndRt2, C1);
5588 Rt2 = Builder.CreateTruncOrBitCast(Rt2, Int32Ty);
5590 return Builder.CreateCall(F, {Coproc, Opc1, Rt, Rt2, CRm});
5593 if (BuiltinID == ARM::BI__builtin_arm_mrrc ||
5594 BuiltinID == ARM::BI__builtin_arm_mrrc2) {
5597 switch (BuiltinID) {
5598 default: llvm_unreachable(
"unexpected builtin");
5599 case ARM::BI__builtin_arm_mrrc:
5600 F = CGM.getIntrinsic(Intrinsic::arm_mrrc);
5602 case ARM::BI__builtin_arm_mrrc2:
5603 F = CGM.getIntrinsic(Intrinsic::arm_mrrc2);
5610 Value *RtAndRt2 = Builder.CreateCall(F, {Coproc, Opc1, CRm});
5615 Value *Rt = Builder.CreateExtractValue(RtAndRt2, 1);
5616 Value *Rt1 = Builder.CreateExtractValue(RtAndRt2, 0);
5617 Rt = Builder.CreateZExt(Rt, Int64Ty);
5618 Rt1 = Builder.CreateZExt(Rt1, Int64Ty);
5620 Value *ShiftCast = llvm::ConstantInt::get(Int64Ty, 32);
5621 RtAndRt2 = Builder.CreateShl(Rt, ShiftCast,
"shl",
true);
5622 RtAndRt2 = Builder.CreateOr(RtAndRt2, Rt1);
5624 return Builder.CreateBitCast(RtAndRt2, ConvertType(E->
getType()));
5627 if (BuiltinID == ARM::BI__builtin_arm_ldrexd ||
5628 ((BuiltinID == ARM::BI__builtin_arm_ldrex ||
5629 BuiltinID == ARM::BI__builtin_arm_ldaex) &&
5630 getContext().getTypeSize(E->
getType()) == 64) ||
5631 BuiltinID == ARM::BI__ldrexd) {
5634 switch (BuiltinID) {
5635 default: llvm_unreachable(
"unexpected builtin");
5636 case ARM::BI__builtin_arm_ldaex:
5637 F = CGM.getIntrinsic(Intrinsic::arm_ldaexd);
5639 case ARM::BI__builtin_arm_ldrexd:
5640 case ARM::BI__builtin_arm_ldrex:
5641 case ARM::BI__ldrexd:
5642 F = CGM.getIntrinsic(Intrinsic::arm_ldrexd);
5647 Value *Val = Builder.CreateCall(F, Builder.CreateBitCast(LdPtr, Int8PtrTy),
5650 Value *Val0 = Builder.CreateExtractValue(Val, 1);
5651 Value *Val1 = Builder.CreateExtractValue(Val, 0);
5652 Val0 = Builder.CreateZExt(Val0, Int64Ty);
5653 Val1 = Builder.CreateZExt(Val1, Int64Ty);
5655 Value *ShiftCst = llvm::ConstantInt::get(Int64Ty, 32);
5656 Val = Builder.CreateShl(Val0, ShiftCst,
"shl",
true );
5657 Val = Builder.CreateOr(Val, Val1);
5658 return Builder.CreateBitCast(Val, ConvertType(E->
getType()));
5661 if (BuiltinID == ARM::BI__builtin_arm_ldrex ||
5662 BuiltinID == ARM::BI__builtin_arm_ldaex) {
5668 getLLVMContext(), getContext().getTypeSize(Ty))->getPointerTo();
5669 LoadAddr = Builder.CreateBitCast(LoadAddr, PtrTy);
5671 Function *F = CGM.getIntrinsic(BuiltinID == ARM::BI__builtin_arm_ldaex
5672 ? Intrinsic::arm_ldaex
5673 : Intrinsic::arm_ldrex,
5675 Value *Val = Builder.CreateCall(F, LoadAddr,
"ldrex");
5677 if (RealResTy->isPointerTy())
5678 return Builder.CreateIntToPtr(Val, RealResTy);
5680 llvm::Type *IntResTy = llvm::IntegerType::get(
5681 getLLVMContext(), CGM.getDataLayout().getTypeSizeInBits(RealResTy));
5682 Val = Builder.CreateTruncOrBitCast(Val, IntResTy);
5683 return Builder.CreateBitCast(Val, RealResTy);
5687 if (BuiltinID == ARM::BI__builtin_arm_strexd ||
5688 ((BuiltinID == ARM::BI__builtin_arm_stlex ||
5689 BuiltinID == ARM::BI__builtin_arm_strex) &&
5691 Function *F = CGM.getIntrinsic(BuiltinID == ARM::BI__builtin_arm_stlex
5692 ? Intrinsic::arm_stlexd
5693 : Intrinsic::arm_strexd);
5694 llvm::Type *STy = llvm::StructType::get(Int32Ty, Int32Ty);
5698 Builder.CreateStore(Val, Tmp);
5700 Address LdPtr = Builder.CreateBitCast(Tmp,llvm::PointerType::getUnqual(STy));
5701 Val = Builder.CreateLoad(LdPtr);
5703 Value *Arg0 = Builder.CreateExtractValue(Val, 0);
5704 Value *Arg1 = Builder.CreateExtractValue(Val, 1);
5705 Value *StPtr = Builder.CreateBitCast(EmitScalarExpr(E->
getArg(1)), Int8PtrTy);
5706 return Builder.CreateCall(F, {Arg0, Arg1, StPtr},
"strexd");
5709 if (BuiltinID == ARM::BI__builtin_arm_strex ||
5710 BuiltinID == ARM::BI__builtin_arm_stlex) {
5715 llvm::Type *StoreTy = llvm::IntegerType::get(getLLVMContext(),
5716 getContext().getTypeSize(Ty));
5717 StoreAddr = Builder.CreateBitCast(StoreAddr, StoreTy->getPointerTo());
5719 if (StoreVal->getType()->isPointerTy())
5720 StoreVal = Builder.CreatePtrToInt(StoreVal, Int32Ty);
5724 CGM.getDataLayout().getTypeSizeInBits(StoreVal->getType()));
5725 StoreVal = Builder.CreateBitCast(StoreVal, IntTy);
5726 StoreVal = Builder.CreateZExtOrBitCast(StoreVal, Int32Ty);
5729 Function *F = CGM.getIntrinsic(BuiltinID == ARM::BI__builtin_arm_stlex
5730 ? Intrinsic::arm_stlex
5731 : Intrinsic::arm_strex,
5732 StoreAddr->getType());
5733 return Builder.CreateCall(F, {StoreVal, StoreAddr},
"strex");
5736 switch (BuiltinID) {
5737 case ARM::BI__iso_volatile_load8:
5738 case ARM::BI__iso_volatile_load16:
5739 case ARM::BI__iso_volatile_load32:
5740 case ARM::BI__iso_volatile_load64:
5741 return EmitISOVolatileLoad(E);
5742 case ARM::BI__iso_volatile_store8:
5743 case ARM::BI__iso_volatile_store16:
5744 case ARM::BI__iso_volatile_store32:
5745 case ARM::BI__iso_volatile_store64:
5746 return EmitISOVolatileStore(E);
5749 if (BuiltinID == ARM::BI__builtin_arm_clrex) {
5750 Function *F = CGM.getIntrinsic(Intrinsic::arm_clrex);
5751 return Builder.CreateCall(F);
5756 switch (BuiltinID) {
5757 case ARM::BI__builtin_arm_crc32b:
5758 CRCIntrinsicID = Intrinsic::arm_crc32b;
break;
5759 case ARM::BI__builtin_arm_crc32cb:
5760 CRCIntrinsicID = Intrinsic::arm_crc32cb;
break;
5761 case ARM::BI__builtin_arm_crc32h:
5762 CRCIntrinsicID = Intrinsic::arm_crc32h;
break;
5763 case ARM::BI__builtin_arm_crc32ch:
5764 CRCIntrinsicID = Intrinsic::arm_crc32ch;
break;
5765 case ARM::BI__builtin_arm_crc32w:
5766 case ARM::BI__builtin_arm_crc32d:
5767 CRCIntrinsicID = Intrinsic::arm_crc32w;
break;
5768 case ARM::BI__builtin_arm_crc32cw:
5769 case ARM::BI__builtin_arm_crc32cd:
5770 CRCIntrinsicID = Intrinsic::arm_crc32cw;
break;
5773 if (CRCIntrinsicID != Intrinsic::not_intrinsic) {
5779 if (BuiltinID == ARM::BI__builtin_arm_crc32d ||
5780 BuiltinID == ARM::BI__builtin_arm_crc32cd) {
5781 Value *C1 = llvm::ConstantInt::get(Int64Ty, 32);
5782 Value *Arg1a = Builder.CreateTruncOrBitCast(Arg1, Int32Ty);
5783 Value *Arg1b = Builder.CreateLShr(Arg1, C1);
5784 Arg1b = Builder.CreateTruncOrBitCast(Arg1b, Int32Ty);
5786 Function *F = CGM.getIntrinsic(CRCIntrinsicID);
5787 Value *Res = Builder.CreateCall(F, {Arg0, Arg1a});
5788 return Builder.CreateCall(F, {Res, Arg1b});
5790 Arg1 = Builder.CreateZExtOrBitCast(Arg1, Int32Ty);
5792 Function *F = CGM.getIntrinsic(CRCIntrinsicID);
5793 return Builder.CreateCall(F, {Arg0, Arg1});
5797 if (BuiltinID == ARM::BI__builtin_arm_rsr ||
5798 BuiltinID == ARM::BI__builtin_arm_rsr64 ||
5799 BuiltinID == ARM::BI__builtin_arm_rsrp ||
5800 BuiltinID == ARM::BI__builtin_arm_wsr ||
5801 BuiltinID == ARM::BI__builtin_arm_wsr64 ||
5802 BuiltinID == ARM::BI__builtin_arm_wsrp) {
5804 bool IsRead = BuiltinID == ARM::BI__builtin_arm_rsr ||
5805 BuiltinID == ARM::BI__builtin_arm_rsr64 ||
5806 BuiltinID == ARM::BI__builtin_arm_rsrp;
5808 bool IsPointerBuiltin = BuiltinID == ARM::BI__builtin_arm_rsrp ||
5809 BuiltinID == ARM::BI__builtin_arm_wsrp;
5811 bool Is64Bit = BuiltinID == ARM::BI__builtin_arm_rsr64 ||
5812 BuiltinID == ARM::BI__builtin_arm_wsr64;
5816 if (IsPointerBuiltin) {
5817 ValueType = VoidPtrTy;
5818 RegisterType = Int32Ty;
5819 }
else if (Is64Bit) {
5820 ValueType = RegisterType = Int64Ty;
5822 ValueType = RegisterType = Int32Ty;
5830 unsigned ICEArguments = 0;
5832 getContext().GetBuiltinType(BuiltinID, Error, &ICEArguments);
5835 auto getAlignmentValue32 = [&](
Address addr) ->
Value* {
5836 return Builder.getInt32(addr.getAlignment().getQuantity());
5843 unsigned NumArgs = E->
getNumArgs() - (HasExtraArg ? 1 : 0);
5844 for (
unsigned i = 0, e = NumArgs; i != e; i++) {
5846 switch (BuiltinID) {
5847 case NEON::BI__builtin_neon_vld1_v:
5848 case NEON::BI__builtin_neon_vld1q_v:
5849 case NEON::BI__builtin_neon_vld1q_lane_v:
5850 case NEON::BI__builtin_neon_vld1_lane_v:
5851 case NEON::BI__builtin_neon_vld1_dup_v:
5852 case NEON::BI__builtin_neon_vld1q_dup_v:
5853 case NEON::BI__builtin_neon_vst1_v:
5854 case NEON::BI__builtin_neon_vst1q_v:
5855 case NEON::BI__builtin_neon_vst1q_lane_v:
5856 case NEON::BI__builtin_neon_vst1_lane_v:
5857 case NEON::BI__builtin_neon_vst2_v:
5858 case NEON::BI__builtin_neon_vst2q_v:
5859 case NEON::BI__builtin_neon_vst2_lane_v:
5860 case NEON::BI__builtin_neon_vst2q_lane_v:
5861 case NEON::BI__builtin_neon_vst3_v:
5862 case NEON::BI__builtin_neon_vst3q_v:
5863 case NEON::BI__builtin_neon_vst3_lane_v:
5864 case NEON::BI__builtin_neon_vst3q_lane_v:
5865 case NEON::BI__builtin_neon_vst4_v:
5866 case NEON::BI__builtin_neon_vst4q_v:
5867 case NEON::BI__builtin_neon_vst4_lane_v:
5868 case NEON::BI__builtin_neon_vst4q_lane_v:
5871 PtrOp0 = EmitPointerWithAlignment(E->
getArg(0));
5877 switch (BuiltinID) {
5878 case NEON::BI__builtin_neon_vld2_v:
5879 case NEON::BI__builtin_neon_vld2q_v:
5880 case NEON::BI__builtin_neon_vld3_v:
5881 case NEON::BI__builtin_neon_vld3q_v:
5882 case NEON::BI__builtin_neon_vld4_v:
5883 case NEON::BI__builtin_neon_vld4q_v:
5884 case NEON::BI__builtin_neon_vld2_lane_v:
5885 case NEON::BI__builtin_neon_vld2q_lane_v:
5886 case NEON::BI__builtin_neon_vld3_lane_v:
5887 case NEON::BI__builtin_neon_vld3q_lane_v:
5888 case NEON::BI__builtin_neon_vld4_lane_v:
5889 case NEON::BI__builtin_neon_vld4q_lane_v:
5890 case NEON::BI__builtin_neon_vld2_dup_v:
5891 case NEON::BI__builtin_neon_vld2q_dup_v:
5892 case NEON::BI__builtin_neon_vld3_dup_v:
5893 case NEON::BI__builtin_neon_vld3q_dup_v:
5894 case NEON::BI__builtin_neon_vld4_dup_v:
5895 case NEON::BI__builtin_neon_vld4q_dup_v:
5898 PtrOp1 = EmitPointerWithAlignment(E->
getArg(1));
5904 if ((ICEArguments & (1 << i)) == 0) {
5905 Ops.push_back(EmitScalarExpr(E->
getArg(i)));
5909 llvm::APSInt Result;
5911 assert(IsConst &&
"Constant arg isn't actually constant?"); (void)IsConst;
5912 Ops.push_back(llvm::ConstantInt::get(getLLVMContext(), Result));
5916 switch (BuiltinID) {
5919 case NEON::BI__builtin_neon_vget_lane_i8:
5920 case NEON::BI__builtin_neon_vget_lane_i16:
5921 case NEON::BI__builtin_neon_vget_lane_i32:
5922 case NEON::BI__builtin_neon_vget_lane_i64:
5923 case NEON::BI__builtin_neon_vget_lane_f32:
5924 case NEON::BI__builtin_neon_vgetq_lane_i8:
5925 case NEON::BI__builtin_neon_vgetq_lane_i16:
5926 case NEON::BI__builtin_neon_vgetq_lane_i32:
5927 case NEON::BI__builtin_neon_vgetq_lane_i64:
5928 case NEON::BI__builtin_neon_vgetq_lane_f32:
5929 return Builder.CreateExtractElement(Ops[0], Ops[1],
"vget_lane");
5931 case NEON::BI__builtin_neon_vrndns_f32: {
5934 Function *F = CGM.getIntrinsic(Intrinsic::arm_neon_vrintn, Tys);
5935 return Builder.CreateCall(F, {Arg},
"vrndn"); }
5937 case NEON::BI__builtin_neon_vset_lane_i8:
5938 case NEON::BI__builtin_neon_vset_lane_i16:
5939 case NEON::BI__builtin_neon_vset_lane_i32:
5940 case NEON::BI__builtin_neon_vset_lane_i64:
5941 case NEON::BI__builtin_neon_vset_lane_f32:
5942 case NEON::BI__builtin_neon_vsetq_lane_i8:
5943 case NEON::BI__builtin_neon_vsetq_lane_i16:
5944 case NEON::BI__builtin_neon_vsetq_lane_i32:
5945 case NEON::BI__builtin_neon_vsetq_lane_i64:
5946 case NEON::BI__builtin_neon_vsetq_lane_f32:
5947 return Builder.CreateInsertElement(Ops[1], Ops[0], Ops[2],
"vset_lane");
5949 case NEON::BI__builtin_neon_vsha1h_u32:
5950 return EmitNeonCall(CGM.getIntrinsic(Intrinsic::arm_neon_sha1h), Ops,
5952 case NEON::BI__builtin_neon_vsha1cq_u32:
5953 return EmitNeonCall(CGM.getIntrinsic(Intrinsic::arm_neon_sha1c), Ops,
5955 case NEON::BI__builtin_neon_vsha1pq_u32:
5956 return EmitNeonCall(CGM.getIntrinsic(Intrinsic::arm_neon_sha1p), Ops,
5958 case NEON::BI__builtin_neon_vsha1mq_u32:
5959 return EmitNeonCall(CGM.getIntrinsic(Intrinsic::arm_neon_sha1m), Ops,
5964 case ARM::BI_MoveToCoprocessor:
5965 case ARM::BI_MoveToCoprocessor2: {
5966 Function *F = CGM.getIntrinsic(BuiltinID == ARM::BI_MoveToCoprocessor ?
5967 Intrinsic::arm_mcr : Intrinsic::arm_mcr2);
5968 return Builder.CreateCall(F, {Ops[1], Ops[2], Ops[0],
5969 Ops[3], Ops[4], Ops[5]});
5971 case ARM::BI_BitScanForward:
5972 case ARM::BI_BitScanForward64:
5973 return EmitMSVCBuiltinExpr(MSVCIntrin::_BitScanForward, E);
5974 case ARM::BI_BitScanReverse:
5975 case ARM::BI_BitScanReverse64:
5976 return EmitMSVCBuiltinExpr(MSVCIntrin::_BitScanReverse, E);
5978 case ARM::BI_InterlockedAnd64:
5979 return EmitMSVCBuiltinExpr(MSVCIntrin::_InterlockedAnd, E);
5980 case ARM::BI_InterlockedExchange64:
5981 return EmitMSVCBuiltinExpr(MSVCIntrin::_InterlockedExchange, E);
5982 case ARM::BI_InterlockedExchangeAdd64:
5983 return EmitMSVCBuiltinExpr(MSVCIntrin::_InterlockedExchangeAdd, E);
5984 case ARM::BI_InterlockedExchangeSub64:
5985 return EmitMSVCBuiltinExpr(MSVCIntrin::_InterlockedExchangeSub, E);
5986 case ARM::BI_InterlockedOr64:
5987 return EmitMSVCBuiltinExpr(MSVCIntrin::_InterlockedOr, E);
5988 case ARM::BI_InterlockedXor64:
5989 return EmitMSVCBuiltinExpr(MSVCIntrin::_InterlockedXor, E);
5990 case ARM::BI_InterlockedDecrement64:
5991 return EmitMSVCBuiltinExpr(MSVCIntrin::_InterlockedDecrement, E);
5992 case ARM::BI_InterlockedIncrement64:
5993 return EmitMSVCBuiltinExpr(MSVCIntrin::_InterlockedIncrement, E);
5997 assert(HasExtraArg);
5998 llvm::APSInt Result;
6003 if (BuiltinID == ARM::BI__builtin_arm_vcvtr_f ||
6004 BuiltinID == ARM::BI__builtin_arm_vcvtr_d) {
6007 if (BuiltinID == ARM::BI__builtin_arm_vcvtr_f)
6013 bool usgn = Result.getZExtValue() == 1;
6014 unsigned Int = usgn ? Intrinsic::arm_vcvtru : Intrinsic::arm_vcvtr;
6017 Function *F = CGM.getIntrinsic(Int, Ty);
6018 return Builder.CreateCall(F, Ops,
"vcvtr");
6023 bool usgn =
Type.isUnsigned();
6024 bool rightShift =
false;
6027 getTarget().hasLegalHalfType());
6034 auto IntrinsicMap = makeArrayRef(ARMSIMDIntrinsicMap);
6036 IntrinsicMap, BuiltinID, NEONSIMDIntrinsicsProvenSorted);
6038 return EmitCommonNeonBuiltinExpr(
6039 Builtin->BuiltinID, Builtin->LLVMIntrinsic, Builtin->AltLLVMIntrinsic,
6040 Builtin->NameHint, Builtin->TypeModifier, E, Ops, PtrOp0, PtrOp1, Arch);
6043 switch (BuiltinID) {
6044 default:
return nullptr;
6045 case NEON::BI__builtin_neon_vld1q_lane_v:
6048 if (VTy->getElementType()->isIntegerTy(64)) {
6050 Ops[1] = Builder.CreateBitCast(Ops[1], Ty);
6051 uint32_t Lane = cast<ConstantInt>(Ops[2])->getZExtValue();
6052 Value *SV = llvm::ConstantVector::get(ConstantInt::get(Int32Ty, 1-Lane));
6053 Ops[1] = Builder.CreateShuffleVector(Ops[1], Ops[1], SV);
6055 Ty = llvm::VectorType::get(VTy->getElementType(), 1);
6057 Function *F = CGM.getIntrinsic(Intrinsic::arm_neon_vld1, Tys);
6058 Value *Align = getAlignmentValue32(PtrOp0);
6059 Value *Ld = Builder.CreateCall(F, {Ops[0], Align});
6061 uint32_t Indices[] = {1 - Lane, Lane};
6062 SV = llvm::ConstantDataVector::get(getLLVMContext(), Indices);
6063 return Builder.CreateShuffleVector(Ops[1], Ld, SV,
"vld1q_lane");
6066 case NEON::BI__builtin_neon_vld1_lane_v: {
6067 Ops[1] = Builder.CreateBitCast(Ops[1], Ty);
6068 PtrOp0 = Builder.CreateElementBitCast(PtrOp0, VTy->
getElementType());
6069 Value *Ld = Builder.CreateLoad(PtrOp0);
6070 return Builder.CreateInsertElement(Ops[1], Ld, Ops[2],
"vld1_lane");
6072 case NEON::BI__builtin_neon_vqrshrn_n_v:
6074 usgn ? Intrinsic::arm_neon_vqrshiftnu : Intrinsic::arm_neon_vqrshiftns;
6075 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops,
"vqrshrn_n",
6077 case NEON::BI__builtin_neon_vqrshrun_n_v:
6078 return EmitNeonCall(CGM.getIntrinsic(Intrinsic::arm_neon_vqrshiftnsu, Ty),
6079 Ops,
"vqrshrun_n", 1,
true);
6080 case NEON::BI__builtin_neon_vqshrn_n_v:
6081 Int = usgn ? Intrinsic::arm_neon_vqshiftnu : Intrinsic::arm_neon_vqshiftns;
6082 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops,
"vqshrn_n",
6084 case NEON::BI__builtin_neon_vqshrun_n_v:
6085 return EmitNeonCall(CGM.getIntrinsic(Intrinsic::arm_neon_vqshiftnsu, Ty),
6086 Ops,
"vqshrun_n", 1,
true);
6087 case NEON::BI__builtin_neon_vrecpe_v:
6088 case NEON::BI__builtin_neon_vrecpeq_v:
6089 return EmitNeonCall(CGM.getIntrinsic(Intrinsic::arm_neon_vrecpe, Ty),
6091 case NEON::BI__builtin_neon_vrshrn_n_v:
6092 return EmitNeonCall(CGM.getIntrinsic(Intrinsic::arm_neon_vrshiftn, Ty),
6093 Ops,
"vrshrn_n", 1,
true);
6094 case NEON::BI__builtin_neon_vrsra_n_v:
6095 case NEON::BI__builtin_neon_vrsraq_n_v:
6096 Ops[0] = Builder.CreateBitCast(Ops[0], Ty);
6097 Ops[1] = Builder.CreateBitCast(Ops[1], Ty);
6098 Ops[2] = EmitNeonShiftVector(Ops[2], Ty,
true);
6099 Int = usgn ? Intrinsic::arm_neon_vrshiftu : Intrinsic::arm_neon_vrshifts;
6100 Ops[1] = Builder.CreateCall(CGM.getIntrinsic(Int, Ty), {Ops[1], Ops[2]});
6101 return Builder.CreateAdd(Ops[0], Ops[1],
"vrsra_n");
6102 case NEON::BI__builtin_neon_vsri_n_v:
6103 case NEON::BI__builtin_neon_vsriq_n_v:
6106 case NEON::BI__builtin_neon_vsli_n_v:
6107 case NEON::BI__builtin_neon_vsliq_n_v:
6108 Ops[2] = EmitNeonShiftVector(Ops[2], Ty, rightShift);
6109 return EmitNeonCall(CGM.getIntrinsic(Intrinsic::arm_neon_vshiftins, Ty),
6111 case NEON::BI__builtin_neon_vsra_n_v:
6112 case NEON::BI__builtin_neon_vsraq_n_v:
6113 Ops[0] = Builder.CreateBitCast(Ops[0], Ty);
6114 Ops[1] = EmitNeonRShiftImm(Ops[1], Ops[2], Ty, usgn,
"vsra_n");
6115 return Builder.CreateAdd(Ops[0], Ops[1]);
6116 case NEON::BI__builtin_neon_vst1q_lane_v:
6119 if (VTy->getElementType()->isIntegerTy(64)) {
6120 Ops[1] = Builder.CreateBitCast(Ops[1], Ty);
6121 Value *SV = llvm::ConstantVector::get(cast<llvm::Constant>(Ops[2]));
6122 Ops[1] = Builder.CreateShuffleVector(Ops[1], Ops[1], SV);
6123 Ops[2] = getAlignmentValue32(PtrOp0);
6124 llvm::Type *Tys[] = {Int8PtrTy, Ops[1]->getType()};
6125 return Builder.CreateCall(CGM.getIntrinsic(Intrinsic::arm_neon_vst1,
6129 case NEON::BI__builtin_neon_vst1_lane_v: {
6130 Ops[1] = Builder.CreateBitCast(Ops[1], Ty);
6131 Ops[1] = Builder.CreateExtractElement(Ops[1], Ops[2]);
6132 Ty = llvm::PointerType::getUnqual(Ops[1]->getType());
6133 auto St = Builder.CreateStore(Ops[1], Builder.CreateBitCast(PtrOp0, Ty));
6136 case NEON::BI__builtin_neon_vtbl1_v:
6137 return EmitNeonCall(CGM.getIntrinsic(Intrinsic::arm_neon_vtbl1),
6139 case NEON::BI__builtin_neon_vtbl2_v:
6140 return EmitNeonCall(CGM.getIntrinsic(Intrinsic::arm_neon_vtbl2),
6142 case NEON::BI__builtin_neon_vtbl3_v:
6143 return EmitNeonCall(CGM.getIntrinsic(Intrinsic::arm_neon_vtbl3),
6145 case NEON::BI__builtin_neon_vtbl4_v:
6146 return EmitNeonCall(CGM.getIntrinsic(Intrinsic::arm_neon_vtbl4),
6148 case NEON::BI__builtin_neon_vtbx1_v:
6149 return EmitNeonCall(CGM.getIntrinsic(Intrinsic::arm_neon_vtbx1),
6151 case NEON::BI__builtin_neon_vtbx2_v:
6152 return EmitNeonCall(CGM.getIntrinsic(Intrinsic::arm_neon_vtbx2),
6154 case NEON::BI__builtin_neon_vtbx3_v:
6155 return EmitNeonCall(CGM.getIntrinsic(Intrinsic::arm_neon_vtbx3),
6157 case NEON::BI__builtin_neon_vtbx4_v:
6158 return EmitNeonCall(CGM.getIntrinsic(Intrinsic::arm_neon_vtbx4),
6166 llvm::Triple::ArchType Arch) {
6167 unsigned int Int = 0;
6168 const char *s =
nullptr;
6170 switch (BuiltinID) {
6173 case NEON::BI__builtin_neon_vtbl1_v:
6174 case NEON::BI__builtin_neon_vqtbl1_v:
6175 case NEON::BI__builtin_neon_vqtbl1q_v:
6176 case NEON::BI__builtin_neon_vtbl2_v:
6177 case NEON::BI__builtin_neon_vqtbl2_v:
6178 case NEON::BI__builtin_neon_vqtbl2q_v:
6179 case NEON::BI__builtin_neon_vtbl3_v:
6180 case NEON::BI__builtin_neon_vqtbl3_v:
6181 case NEON::BI__builtin_neon_vqtbl3q_v:
6182 case NEON::BI__builtin_neon_vtbl4_v:
6183 case NEON::BI__builtin_neon_vqtbl4_v:
6184 case NEON::BI__builtin_neon_vqtbl4q_v:
6186 case NEON::BI__builtin_neon_vtbx1_v:
6187 case NEON::BI__builtin_neon_vqtbx1_v:
6188 case NEON::BI__builtin_neon_vqtbx1q_v:
6189 case NEON::BI__builtin_neon_vtbx2_v:
6190 case NEON::BI__builtin_neon_vqtbx2_v:
6191 case NEON::BI__builtin_neon_vqtbx2q_v:
6192 case NEON::BI__builtin_neon_vtbx3_v:
6193 case NEON::BI__builtin_neon_vqtbx3_v:
6194 case NEON::BI__builtin_neon_vqtbx3q_v:
6195 case NEON::BI__builtin_neon_vtbx4_v:
6196 case NEON::BI__builtin_neon_vqtbx4_v:
6197 case NEON::BI__builtin_neon_vqtbx4q_v:
6204 llvm::APSInt Result;
6219 switch (BuiltinID) {
6220 case NEON::BI__builtin_neon_vtbl1_v: {
6222 Ops[1], Ty, Intrinsic::aarch64_neon_tbl1,
6225 case NEON::BI__builtin_neon_vtbl2_v: {
6227 Ops[2], Ty, Intrinsic::aarch64_neon_tbl1,
6230 case NEON::BI__builtin_neon_vtbl3_v: {
6232 Ops[3], Ty, Intrinsic::aarch64_neon_tbl2,
6235 case NEON::BI__builtin_neon_vtbl4_v: {
6237 Ops[4], Ty, Intrinsic::aarch64_neon_tbl2,
6240 case NEON::BI__builtin_neon_vtbx1_v: {
6243 Ty, Intrinsic::aarch64_neon_tbl1,
"vtbl1");
6245 llvm::Constant *EightV = ConstantInt::get(Ty, 8);
6246 Value *CmpRes = Builder.CreateICmp(ICmpInst::ICMP_UGE, Ops[2], EightV);
6247 CmpRes = Builder.CreateSExt(CmpRes, Ty);
6249 Value *EltsFromInput = Builder.CreateAnd(CmpRes, Ops[0]);
6250 Value *EltsFromTbl = Builder.CreateAnd(Builder.CreateNot(CmpRes), TblRes);
6251 return Builder.CreateOr(EltsFromInput, EltsFromTbl,
"vtbx");
6253 case NEON::BI__builtin_neon_vtbx2_v: {
6255 Ops[3], Ty, Intrinsic::aarch64_neon_tbx1,
6258 case NEON::BI__builtin_neon_vtbx3_v: {
6261 Ty, Intrinsic::aarch64_neon_tbl2,
"vtbl2");
6263 llvm::Constant *TwentyFourV = ConstantInt::get(Ty, 24);
6264 Value *CmpRes = Builder.CreateICmp(ICmpInst::ICMP_UGE, Ops[4],
6266 CmpRes = Builder.CreateSExt(CmpRes, Ty);
6268 Value *EltsFromInput = Builder.CreateAnd(CmpRes, Ops[0]);
6269 Value *EltsFromTbl = Builder.CreateAnd(Builder.CreateNot(CmpRes), TblRes);
6270 return Builder.CreateOr(EltsFromInput, EltsFromTbl,
"vtbx");
6272 case NEON::BI__builtin_neon_vtbx4_v: {
6274 Ops[5], Ty, Intrinsic::aarch64_neon_tbx2,
6277 case NEON::BI__builtin_neon_vqtbl1_v:
6278 case NEON::BI__builtin_neon_vqtbl1q_v:
6279 Int = Intrinsic::aarch64_neon_tbl1; s =
"vtbl1";
break;
6280 case NEON::BI__builtin_neon_vqtbl2_v:
6281 case NEON::BI__builtin_neon_vqtbl2q_v: {
6282 Int = Intrinsic::aarch64_neon_tbl2; s =
"vtbl2";
break;
6283 case NEON::BI__builtin_neon_vqtbl3_v:
6284 case NEON::BI__builtin_neon_vqtbl3q_v:
6285 Int = Intrinsic::aarch64_neon_tbl3; s =
"vtbl3";
break;
6286 case NEON::BI__builtin_neon_vqtbl4_v:
6287 case NEON::BI__builtin_neon_vqtbl4q_v:
6288 Int = Intrinsic::aarch64_neon_tbl4; s =
"vtbl4";
break;
6289 case NEON::BI__builtin_neon_vqtbx1_v:
6290 case NEON::BI__builtin_neon_vqtbx1q_v:
6291 Int = Intrinsic::aarch64_neon_tbx1; s =
"vtbx1";
break;
6292 case NEON::BI__builtin_neon_vqtbx2_v:
6293 case NEON::BI__builtin_neon_vqtbx2q_v:
6294 Int = Intrinsic::aarch64_neon_tbx2; s =
"vtbx2";
break;
6295 case NEON::BI__builtin_neon_vqtbx3_v:
6296 case NEON::BI__builtin_neon_vqtbx3q_v:
6297 Int = Intrinsic::aarch64_neon_tbx3; s =
"vtbx3";
break;
6298 case NEON::BI__builtin_neon_vqtbx4_v:
6299 case NEON::BI__builtin_neon_vqtbx4q_v:
6300 Int = Intrinsic::aarch64_neon_tbx4; s =
"vtbx4";
break;
6312 llvm::Type *VTy = llvm::VectorType::get(Int16Ty, 4);
6313 Op = Builder.CreateBitCast(Op, Int16Ty);
6314 Value *V = UndefValue::get(VTy);
6315 llvm::Constant *CI = ConstantInt::get(SizeTy, 0);
6316 Op = Builder.CreateInsertElement(V, Op, CI);
6322 llvm::Triple::ArchType Arch) {
6323 unsigned HintID =
static_cast<unsigned>(-1);
6324 switch (BuiltinID) {
6326 case AArch64::BI__builtin_arm_nop:
6329 case AArch64::BI__builtin_arm_yield:
6330 case AArch64::BI__yield:
6333 case AArch64::BI__builtin_arm_wfe:
6334 case AArch64::BI__wfe:
6337 case AArch64::BI__builtin_arm_wfi:
6338 case AArch64::BI__wfi:
6341 case AArch64::BI__builtin_arm_sev:
6342 case AArch64::BI__sev:
6345 case AArch64::BI__builtin_arm_sevl:
6346 case AArch64::BI__sevl:
6351 if (HintID != static_cast<unsigned>(-1)) {
6352 Function *F = CGM.getIntrinsic(Intrinsic::aarch64_hint);
6353 return Builder.CreateCall(F, llvm::ConstantInt::get(Int32Ty, HintID));
6356 if (BuiltinID == AArch64::BI__builtin_arm_prefetch) {
6360 Value *RetentionPolicy = EmitScalarExpr(E->
getArg(3));
6363 Value *Locality =
nullptr;
6364 if (cast<llvm::ConstantInt>(RetentionPolicy)->isZero()) {
6366 Locality = llvm::ConstantInt::get(Int32Ty,
6367 -cast<llvm::ConstantInt>(CacheLevel)->
getValue() + 3);
6370 Locality = llvm::ConstantInt::get(Int32Ty, 0);
6376 return Builder.CreateCall(F, {Address, RW, Locality, IsData});
6379 if (BuiltinID == AArch64::BI__builtin_arm_rbit) {
6380 assert((getContext().getTypeSize(E->
getType()) == 32) &&
6381 "rbit of unusual size!");
6383 return Builder.CreateCall(
6384 CGM.getIntrinsic(Intrinsic::bitreverse, Arg->getType()), Arg,
"rbit");
6386 if (BuiltinID == AArch64::BI__builtin_arm_rbit64) {
6387 assert((getContext().getTypeSize(E->
getType()) == 64) &&
6388 "rbit of unusual size!");
6390 return Builder.CreateCall(
6391 CGM.getIntrinsic(Intrinsic::bitreverse, Arg->getType()), Arg,
"rbit");
6394 if (BuiltinID == AArch64::BI__clear_cache) {
6395 assert(E->
getNumArgs() == 2 &&
"__clear_cache takes 2 arguments");
6398 for (
unsigned i = 0; i < 2; i++)
6399 Ops[i] = EmitScalarExpr(E->
getArg(i));
6400 llvm::Type *Ty = CGM.getTypes().ConvertType(FD->getType());
6401 llvm::FunctionType *FTy = cast<llvm::FunctionType>(Ty);
6402 StringRef Name = FD->getName();
6403 return EmitNounwindRuntimeCall(CGM.CreateRuntimeFunction(FTy, Name), Ops);
6406 if ((BuiltinID == AArch64::BI__builtin_arm_ldrex ||
6407 BuiltinID == AArch64::BI__builtin_arm_ldaex) &&
6408 getContext().getTypeSize(E->
getType()) == 128) {
6409 Function *F = CGM.getIntrinsic(BuiltinID == AArch64::BI__builtin_arm_ldaex
6410 ? Intrinsic::aarch64_ldaxp
6411 : Intrinsic::aarch64_ldxp);
6414 Value *Val = Builder.CreateCall(F, Builder.CreateBitCast(LdPtr, Int8PtrTy),
6417 Value *Val0 = Builder.CreateExtractValue(Val, 1);
6418 Value *Val1 = Builder.CreateExtractValue(Val, 0);
6419 llvm::Type *Int128Ty = llvm::IntegerType::get(getLLVMContext(), 128);
6420 Val0 = Builder.CreateZExt(Val0, Int128Ty);
6421 Val1 = Builder.CreateZExt(Val1, Int128Ty);
6423 Value *ShiftCst = llvm::ConstantInt::get(Int128Ty, 64);
6424 Val = Builder.CreateShl(Val0, ShiftCst,
"shl",
true );
6425 Val = Builder.CreateOr(Val, Val1);
6426 return Builder.CreateBitCast(Val, ConvertType(E->
getType()));
6427 }
else if (BuiltinID == AArch64::BI__builtin_arm_ldrex ||
6428 BuiltinID == AArch64::BI__builtin_arm_ldaex) {
6434 getLLVMContext(), getContext().getTypeSize(Ty))->getPointerTo();
6435 LoadAddr = Builder.CreateBitCast(LoadAddr, PtrTy);
6437 Function *F = CGM.getIntrinsic(BuiltinID == AArch64::BI__builtin_arm_ldaex
6438 ? Intrinsic::aarch64_ldaxr
6439 : Intrinsic::aarch64_ldxr,
6441 Value *Val = Builder.CreateCall(F, LoadAddr,
"ldxr");
6443 if (RealResTy->isPointerTy())
6444 return Builder.CreateIntToPtr(Val, RealResTy);
6446 llvm::Type *IntResTy = llvm::IntegerType::get(
6447 getLLVMContext(), CGM.getDataLayout().getTypeSizeInBits(RealResTy));
6448 Val = Builder.CreateTruncOrBitCast(Val, IntResTy);
6449 return Builder.CreateBitCast(Val, RealResTy);
6452 if ((BuiltinID == AArch64::BI__builtin_arm_strex ||
6453 BuiltinID == AArch64::BI__builtin_arm_stlex) &&
6455 Function *F = CGM.getIntrinsic(BuiltinID == AArch64::BI__builtin_arm_stlex
6456 ? Intrinsic::aarch64_stlxp
6457 : Intrinsic::aarch64_stxp);
6458 llvm::Type *STy = llvm::StructType::get(Int64Ty, Int64Ty);
6463 Tmp = Builder.CreateBitCast(Tmp, llvm::PointerType::getUnqual(STy));
6466 Value *Arg0 = Builder.CreateExtractValue(Val, 0);
6467 Value *Arg1 = Builder.CreateExtractValue(Val, 1);
6468 Value *StPtr = Builder.CreateBitCast(EmitScalarExpr(E->
getArg(1)),
6470 return Builder.CreateCall(F, {Arg0, Arg1, StPtr},
"stxp");
6473 if (BuiltinID == AArch64::BI__builtin_arm_strex ||
6474 BuiltinID == AArch64::BI__builtin_arm_stlex) {
6479 llvm::Type *StoreTy = llvm::IntegerType::get(getLLVMContext(),
6480 getContext().getTypeSize(Ty));
6481 StoreAddr = Builder.CreateBitCast(StoreAddr, StoreTy->getPointerTo());
6483 if (StoreVal->getType()->isPointerTy())
6484 StoreVal = Builder.CreatePtrToInt(StoreVal, Int64Ty);
6488 CGM.getDataLayout().getTypeSizeInBits(StoreVal->getType()));
6489 StoreVal = Builder.CreateBitCast(StoreVal, IntTy);
6490 StoreVal = Builder.CreateZExtOrBitCast(StoreVal, Int64Ty);
6493 Function *F = CGM.getIntrinsic(BuiltinID == AArch64::BI__builtin_arm_stlex
6494 ? Intrinsic::aarch64_stlxr
6495 : Intrinsic::aarch64_stxr,
6496 StoreAddr->getType());
6497 return Builder.CreateCall(F, {StoreVal, StoreAddr},
"stxr");
6500 if (BuiltinID == AArch64::BI__builtin_arm_clrex) {
6501 Function *F = CGM.getIntrinsic(Intrinsic::aarch64_clrex);
6502 return Builder.CreateCall(F);
6507 switch (BuiltinID) {
6508 case AArch64::BI__builtin_arm_crc32b:
6509 CRCIntrinsicID = Intrinsic::aarch64_crc32b;
break;
6510 case AArch64::BI__builtin_arm_crc32cb:
6511 CRCIntrinsicID = Intrinsic::aarch64_crc32cb;
break;
6512 case AArch64::BI__builtin_arm_crc32h:
6513 CRCIntrinsicID = Intrinsic::aarch64_crc32h;
break;
6514 case AArch64::BI__builtin_arm_crc32ch:
6515 CRCIntrinsicID = Intrinsic::aarch64_crc32ch;
break;
6516 case AArch64::BI__builtin_arm_crc32w:
6517 CRCIntrinsicID = Intrinsic::aarch64_crc32w;
break;
6518 case AArch64::BI__builtin_arm_crc32cw:
6519 CRCIntrinsicID = Intrinsic::aarch64_crc32cw;
break;
6520 case AArch64::BI__builtin_arm_crc32d:
6521 CRCIntrinsicID = Intrinsic::aarch64_crc32x;
break;
6522 case AArch64::BI__builtin_arm_crc32cd:
6523 CRCIntrinsicID = Intrinsic::aarch64_crc32cx;
break;
6526 if (CRCIntrinsicID != Intrinsic::not_intrinsic) {
6529 Function *F = CGM.getIntrinsic(CRCIntrinsicID);
6531 llvm::Type *DataTy = F->getFunctionType()->getParamType(1);
6532 Arg1 = Builder.CreateZExtOrBitCast(Arg1, DataTy);
6534 return Builder.CreateCall(F, {Arg0, Arg1});
6537 if (BuiltinID == AArch64::BI__builtin_arm_rsr ||
6538 BuiltinID == AArch64::BI__builtin_arm_rsr64 ||
6539 BuiltinID == AArch64::BI__builtin_arm_rsrp ||
6540 BuiltinID == AArch64::BI__builtin_arm_wsr ||
6541 BuiltinID == AArch64::BI__builtin_arm_wsr64 ||
6542 BuiltinID == AArch64::BI__builtin_arm_wsrp) {
6544 bool IsRead = BuiltinID == AArch64::BI__builtin_arm_rsr ||
6545 BuiltinID == AArch64::BI__builtin_arm_rsr64 ||
6546 BuiltinID == AArch64::BI__builtin_arm_rsrp;
6548 bool IsPointerBuiltin = BuiltinID == AArch64::BI__builtin_arm_rsrp ||
6549 BuiltinID == AArch64::BI__builtin_arm_wsrp;
6551 bool Is64Bit = BuiltinID != AArch64::BI__builtin_arm_rsr &&
6552 BuiltinID != AArch64::BI__builtin_arm_wsr;
6556 if (IsPointerBuiltin) {
6557 ValueType = VoidPtrTy;
6558 }
else if (Is64Bit) {
6559 ValueType = Int64Ty;
6561 ValueType = Int32Ty;
6569 unsigned ICEArguments = 0;
6571 getContext().GetBuiltinType(BuiltinID, Error, &ICEArguments);
6575 for (
unsigned i = 0, e = E->
getNumArgs() - 1; i != e; i++) {
6576 if ((ICEArguments & (1 << i)) == 0) {
6577 Ops.push_back(EmitScalarExpr(E->
getArg(i)));
6581 llvm::APSInt Result;
6583 assert(IsConst &&
"Constant arg isn't actually constant?");
6585 Ops.push_back(llvm::ConstantInt::get(getLLVMContext(), Result));
6589 auto SISDMap = makeArrayRef(AArch64SISDIntrinsicMap);
6591 SISDMap, BuiltinID, AArch64SISDIntrinsicsProvenSorted);
6596 assert(Result &&
"SISD intrinsic should have been handled");
6600 llvm::APSInt Result;
6608 bool quad = Type.
isQuad();
6611 switch (BuiltinID) {
6613 case NEON::BI__builtin_neon_vabsh_f16:
6614 Ops.push_back(EmitScalarExpr(E->
getArg(0)));
6615 return EmitNeonCall(CGM.getIntrinsic(
Intrinsic::fabs, HalfTy), Ops,
"vabs");
6616 case NEON::BI__builtin_neon_vldrq_p128: {
6617 llvm::Type *Int128Ty = llvm::Type::getIntNTy(getLLVMContext(), 128);
6618 llvm::Type *Int128PTy = llvm::PointerType::get(Int128Ty, 0);
6619 Value *Ptr = Builder.CreateBitCast(EmitScalarExpr(E->
getArg(0)), Int128PTy);
6620 return Builder.CreateAlignedLoad(Int128Ty, Ptr,
6623 case NEON::BI__builtin_neon_vstrq_p128: {
6624 llvm::Type *Int128PTy = llvm::Type::getIntNPtrTy(getLLVMContext(), 128);
6625 Value *Ptr = Builder.CreateBitCast(Ops[0], Int128PTy);
6626 return Builder.CreateDefaultAlignedStore(EmitScalarExpr(E->
getArg(1)), Ptr);
6628 case NEON::BI__builtin_neon_vcvts_u32_f32:
6629 case NEON::BI__builtin_neon_vcvtd_u64_f64:
6632 case NEON::BI__builtin_neon_vcvts_s32_f32:
6633 case NEON::BI__builtin_neon_vcvtd_s64_f64: {
6634 Ops.push_back(EmitScalarExpr(E->
getArg(0)));
6635 bool Is64 = Ops[0]->getType()->getPrimitiveSizeInBits() == 64;
6638 Ops[0] = Builder.CreateBitCast(Ops[0], FTy);
6640 return Builder.CreateFPToUI(Ops[0], InTy);
6641 return Builder.CreateFPToSI(Ops[0], InTy);
6643 case NEON::BI__builtin_neon_vcvts_f32_u32:
6644 case NEON::BI__builtin_neon_vcvtd_f64_u64:
6647 case NEON::BI__builtin_neon_vcvts_f32_s32:
6648 case NEON::BI__builtin_neon_vcvtd_f64_s64: {
6649 Ops.push_back(EmitScalarExpr(E->
getArg(0)));
6650 bool Is64 = Ops[0]->getType()->getPrimitiveSizeInBits() == 64;
6653 Ops[0] = Builder.CreateBitCast(Ops[0], InTy);
6655 return Builder.CreateUIToFP(Ops[0], FTy);
6656 return Builder.CreateSIToFP(Ops[0], FTy);
6658 case NEON::BI__builtin_neon_vcvth_f16_u16:
6659 case NEON::BI__builtin_neon_vcvth_f16_u32:
6660 case NEON::BI__builtin_neon_vcvth_f16_u64:
6663 case NEON::BI__builtin_neon_vcvth_f16_s16:
6664 case NEON::BI__builtin_neon_vcvth_f16_s32:
6665 case NEON::BI__builtin_neon_vcvth_f16_s64: {
6666 Ops.push_back(EmitScalarExpr(E->
getArg(0)));
6669 if (Ops[0]->getType()->getPrimitiveSizeInBits() == 64)
6671 else if (Ops[0]->getType()->getPrimitiveSizeInBits() == 32)
6675 Ops[0] = Builder.CreateBitCast(Ops[0], InTy);
6677 return Builder.CreateUIToFP(Ops[0], FTy);
6678 return Builder.CreateSIToFP(Ops[0], FTy);
6680 case NEON::BI__builtin_neon_vcvth_u16_f16:
6683 case NEON::BI__builtin_neon_vcvth_s16_f16: {
6684 Ops.push_back(EmitScalarExpr(E->
getArg(0)));
6685 Ops[0] = Builder.CreateBitCast(Ops[0], HalfTy);
6687 return Builder.CreateFPToUI(Ops[0], Int16Ty);
6688 return Builder.CreateFPToSI(Ops[0], Int16Ty);
6690 case NEON::BI__builtin_neon_vcvth_u32_f16:
6693 case NEON::BI__builtin_neon_vcvth_s32_f16: {
6694 Ops.push_back(EmitScalarExpr(E->
getArg(0)));
6695 Ops[0] = Builder.CreateBitCast(Ops[0], HalfTy);
6697 return Builder.CreateFPToUI(Ops[0], Int32Ty);
6698 return Builder.CreateFPToSI(Ops[0], Int32Ty);
6700 case NEON::BI__builtin_neon_vcvth_u64_f16:
6703 case NEON::BI__builtin_neon_vcvth_s64_f16: {
6704 Ops.push_back(EmitScalarExpr(E->
getArg(0)));
6705 Ops[0] = Builder.CreateBitCast(Ops[0], HalfTy);
6707 return Builder.CreateFPToUI(Ops[0], Int64Ty);
6708 return Builder.CreateFPToSI(Ops[0], Int64Ty);
6710 case NEON::BI__builtin_neon_vcvtah_u16_f16:
6711 case NEON::BI__builtin_neon_vcvtmh_u16_f16:
6712 case NEON::BI__builtin_neon_vcvtnh_u16_f16:
6713 case NEON::BI__builtin_neon_vcvtph_u16_f16:
6714 case NEON::BI__builtin_neon_vcvtah_s16_f16:
6715 case NEON::BI__builtin_neon_vcvtmh_s16_f16:
6716 case NEON::BI__builtin_neon_vcvtnh_s16_f16:
6717 case NEON::BI__builtin_neon_vcvtph_s16_f16: {
6722 Ops.push_back(EmitScalarExpr(E->
getArg(0)));
6723 switch (BuiltinID) {
6724 default: llvm_unreachable(
"missing builtin ID in switch!");
6725 case NEON::BI__builtin_neon_vcvtah_u16_f16:
6726 Int = Intrinsic::aarch64_neon_fcvtau;
break;
6727 case NEON::BI__builtin_neon_vcvtmh_u16_f16:
6728 Int = Intrinsic::aarch64_neon_fcvtmu;
break;
6729 case NEON::BI__builtin_neon_vcvtnh_u16_f16:
6730 Int = Intrinsic::aarch64_neon_fcvtnu;
break;
6731 case NEON::BI__builtin_neon_vcvtph_u16_f16:
6732 Int = Intrinsic::aarch64_neon_fcvtpu;
break;
6733 case NEON::BI__builtin_neon_vcvtah_s16_f16:
6734 Int = Intrinsic::aarch64_neon_fcvtas;
break;
6735 case NEON::BI__builtin_neon_vcvtmh_s16_f16:
6736 Int = Intrinsic::aarch64_neon_fcvtms;
break;
6737 case NEON::BI__builtin_neon_vcvtnh_s16_f16:
6738 Int = Intrinsic::aarch64_neon_fcvtns;
break;
6739 case NEON::BI__builtin_neon_vcvtph_s16_f16:
6740 Int = Intrinsic::aarch64_neon_fcvtps;
break;
6742 Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops,
"fcvt");
6743 return Builder.CreateTrunc(Ops[0], Int16Ty);
6745 case NEON::BI__builtin_neon_vcaleh_f16:
6746 case NEON::BI__builtin_neon_vcalth_f16:
6747 case NEON::BI__builtin_neon_vcageh_f16:
6748 case NEON::BI__builtin_neon_vcagth_f16: {
6753 Ops.push_back(EmitScalarExpr(E->
getArg(1)));
6754 switch (BuiltinID) {
6755 default: llvm_unreachable(
"missing builtin ID in switch!");
6756 case NEON::BI__builtin_neon_vcageh_f16:
6757 Int = Intrinsic::aarch64_neon_facge;
break;
6758 case NEON::BI__builtin_neon_vcagth_f16:
6759 Int = Intrinsic::aarch64_neon_facgt;
break;
6760 case NEON::BI__builtin_neon_vcaleh_f16:
6761 Int = Intrinsic::aarch64_neon_facge; std::swap(Ops[0], Ops[1]);
break;
6762 case NEON::BI__builtin_neon_vcalth_f16:
6763 Int = Intrinsic::aarch64_neon_facgt; std::swap(Ops[0], Ops[1]);
break;
6765 Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops,
"facg");
6766 return Builder.CreateTrunc(Ops[0], Int16Ty);
6768 case NEON::BI__builtin_neon_vcvth_n_s16_f16:
6769 case NEON::BI__builtin_neon_vcvth_n_u16_f16: {
6774 Ops.push_back(EmitScalarExpr(E->
getArg(1)));
6775 switch (BuiltinID) {
6776 default: llvm_unreachable(
"missing builtin ID in switch!");
6777 case NEON::BI__builtin_neon_vcvth_n_s16_f16:
6778 Int = Intrinsic::aarch64_neon_vcvtfp2fxs;
break;
6779 case NEON::BI__builtin_neon_vcvth_n_u16_f16:
6780 Int = Intrinsic::aarch64_neon_vcvtfp2fxu;
break;
6782 Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops,
"fcvth_n");
6783 return Builder.CreateTrunc(Ops[0], Int16Ty);
6785 case NEON::BI__builtin_neon_vcvth_n_f16_s16:
6786 case NEON::BI__builtin_neon_vcvth_n_f16_u16: {
6791 Ops.push_back(EmitScalarExpr(E->
getArg(1)));
6792 switch (BuiltinID) {
6793 default: llvm_unreachable(
"missing builtin ID in switch!");
6794 case NEON::BI__builtin_neon_vcvth_n_f16_s16:
6795 Int = Intrinsic::aarch64_neon_vcvtfxs2fp;
6796 Ops[0] = Builder.CreateSExt(Ops[0], InTy,
"sext");
6798 case NEON::BI__builtin_neon_vcvth_n_f16_u16:
6799 Int = Intrinsic::aarch64_neon_vcvtfxu2fp;
6800 Ops[0] = Builder.CreateZExt(Ops[0], InTy);
6803 return EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops,
"fcvth_n");
6805 case NEON::BI__builtin_neon_vpaddd_s64: {
6806 llvm::Type *Ty = llvm::VectorType::get(Int64Ty, 2);
6809 Vec = Builder.CreateBitCast(Vec, Ty,
"v2i64");
6810 llvm::Value *Idx0 = llvm::ConstantInt::get(SizeTy, 0);
6811 llvm::Value *Idx1 = llvm::ConstantInt::get(SizeTy, 1);
6812 Value *Op0 = Builder.CreateExtractElement(Vec, Idx0,
"lane0");
6813 Value *Op1 = Builder.CreateExtractElement(Vec, Idx1,
"lane1");
6815 return Builder.CreateAdd(Op0, Op1,
"vpaddd");
6817 case NEON::BI__builtin_neon_vpaddd_f64: {
6819 llvm::VectorType::get(DoubleTy, 2);
6822 Vec = Builder.CreateBitCast(Vec, Ty,
"v2f64");
6823 llvm::Value *Idx0 = llvm::ConstantInt::get(SizeTy, 0);
6824 llvm::Value *Idx1 = llvm::ConstantInt::get(SizeTy, 1);
6825 Value *Op0 = Builder.CreateExtractElement(Vec, Idx0,
"lane0");
6826 Value *Op1 = Builder.CreateExtractElement(Vec, Idx1,
"lane1");
6828 return Builder.CreateFAdd(Op0, Op1,
"vpaddd");
6830 case NEON::BI__builtin_neon_vpadds_f32: {
6832 llvm::VectorType::get(FloatTy, 2);
6835 Vec = Builder.CreateBitCast(Vec, Ty,
"v2f32");
6836 llvm::Value *Idx0 = llvm::ConstantInt::get(SizeTy, 0);
6837 llvm::Value *Idx1 = llvm::ConstantInt::get(SizeTy, 1);
6838 Value *Op0 = Builder.CreateExtractElement(Vec, Idx0,
"lane0");
6839 Value *Op1 = Builder.CreateExtractElement(Vec, Idx1,
"lane1");
6841 return Builder.CreateFAdd(Op0, Op1,
"vpaddd");
6843 case NEON::BI__builtin_neon_vceqzd_s64:
6844 case NEON::BI__builtin_neon_vceqzd_f64:
6845 case NEON::BI__builtin_neon_vceqzs_f32:
6846 case NEON::BI__builtin_neon_vceqzh_f16:
6847 Ops.push_back(EmitScalarExpr(E->
getArg(0)));
6848 return EmitAArch64CompareBuiltinExpr(
6850 ICmpInst::FCMP_OEQ, ICmpInst::ICMP_EQ,
"vceqz");
6851 case NEON::BI__builtin_neon_vcgezd_s64:
6852 case NEON::BI__builtin_neon_vcgezd_f64:
6853 case NEON::BI__builtin_neon_vcgezs_f32:
6854 case NEON::BI__builtin_neon_vcgezh_f16:
6855 Ops.push_back(EmitScalarExpr(E->
getArg(0)));
6856 return EmitAArch64CompareBuiltinExpr(
6858 ICmpInst::FCMP_OGE, ICmpInst::ICMP_SGE,
"vcgez");
6859 case NEON::BI__builtin_neon_vclezd_s64:
6860 case NEON::BI__builtin_neon_vclezd_f64:
6861 case NEON::BI__builtin_neon_vclezs_f32:
6862 case NEON::BI__builtin_neon_vclezh_f16:
6863 Ops.push_back(EmitScalarExpr(E->
getArg(0)));
6864 return EmitAArch64CompareBuiltinExpr(
6866 ICmpInst::FCMP_OLE, ICmpInst::ICMP_SLE,
"vclez");
6867 case NEON::BI__builtin_neon_vcgtzd_s64:
6868 case NEON::BI__builtin_neon_vcgtzd_f64:
6869 case NEON::BI__builtin_neon_vcgtzs_f32:
6870 case NEON::BI__builtin_neon_vcgtzh_f16:
6871 Ops.push_back(EmitScalarExpr(E->
getArg(0)));
6872 return EmitAArch64CompareBuiltinExpr(
6874 ICmpInst::FCMP_OGT, ICmpInst::ICMP_SGT,
"vcgtz");
6875 case NEON::BI__builtin_neon_vcltzd_s64:
6876 case NEON::BI__builtin_neon_vcltzd_f64:
6877 case NEON::BI__builtin_neon_vcltzs_f32:
6878 case NEON::BI__builtin_neon_vcltzh_f16:
6879 Ops.push_back(EmitScalarExpr(E->
getArg(0)));
6880 return EmitAArch64CompareBuiltinExpr(
6882 ICmpInst::FCMP_OLT, ICmpInst::ICMP_SLT,
"vcltz");
6884 case NEON::BI__builtin_neon_vceqzd_u64: {
6885 Ops.push_back(EmitScalarExpr(E->
getArg(0)));
6886 Ops[0] = Builder.CreateBitCast(Ops[0], Int64Ty);
6888 Builder.CreateICmpEQ(Ops[0], llvm::Constant::getNullValue(Int64Ty));
6889 return Builder.CreateSExt(Ops[0], Int64Ty,
"vceqzd");
6891 case NEON::BI__builtin_neon_vceqd_f64:
6892 case NEON::BI__builtin_neon_vcled_f64:
6893 case NEON::BI__builtin_neon_vcltd_f64:
6894 case NEON::BI__builtin_neon_vcged_f64:
6895 case NEON::BI__builtin_neon_vcgtd_f64: {
6896 llvm::CmpInst::Predicate
P;
6897 switch (BuiltinID) {
6898 default: llvm_unreachable(
"missing builtin ID in switch!");
6899 case NEON::BI__builtin_neon_vceqd_f64: P = llvm::FCmpInst::FCMP_OEQ;
break;
6900 case NEON::BI__builtin_neon_vcled_f64: P = llvm::FCmpInst::FCMP_OLE;
break;
6901 case NEON::BI__builtin_neon_vcltd_f64: P = llvm::FCmpInst::FCMP_OLT;
break;
6902 case NEON::BI__builtin_neon_vcged_f64: P = llvm::FCmpInst::FCMP_OGE;
break;
6903 case NEON::BI__builtin_neon_vcgtd_f64: P = llvm::FCmpInst::FCMP_OGT;
break;
6905 Ops.push_back(EmitScalarExpr(E->
getArg(1)));
6906 Ops[0] = Builder.CreateBitCast(Ops[0], DoubleTy);
6907 Ops[1] = Builder.CreateBitCast(Ops[1], DoubleTy);
6908 Ops[0] = Builder.CreateFCmp(P, Ops[0], Ops[1]);
6909 return Builder.CreateSExt(Ops[0], Int64Ty,
"vcmpd");
6911 case NEON::BI__builtin_neon_vceqs_f32:
6912 case NEON::BI__builtin_neon_vcles_f32:
6913 case NEON::BI__builtin_neon_vclts_f32:
6914 case NEON::BI__builtin_neon_vcges_f32:
6915 case NEON::BI__builtin_neon_vcgts_f32: {
6916 llvm::CmpInst::Predicate
P;
6917 switch (BuiltinID) {
6918 default: llvm_unreachable(
"missing builtin ID in switch!");
6919 case NEON::BI__builtin_neon_vceqs_f32: P = llvm::FCmpInst::FCMP_OEQ;
break;
6920 case NEON::BI__builtin_neon_vcles_f32: P = llvm::FCmpInst::FCMP_OLE;
break;
6921 case NEON::BI__builtin_neon_vclts_f32: P = llvm::FCmpInst::FCMP_OLT;
break;
6922 case NEON::BI__builtin_neon_vcges_f32: P = llvm::FCmpInst::FCMP_OGE;
break;
6923 case NEON::BI__builtin_neon_vcgts_f32: P = llvm::FCmpInst::FCMP_OGT;
break;
6925 Ops.push_back(EmitScalarExpr(E->
getArg(1)));
6926 Ops[0] = Builder.CreateBitCast(Ops[0], FloatTy);
6927 Ops[1] = Builder.CreateBitCast(Ops[1], FloatTy);
6928 Ops[0] = Builder.CreateFCmp(P, Ops[0], Ops[1]);
6929 return Builder.CreateSExt(Ops[0], Int32Ty,
"vcmpd");
6931 case NEON::BI__builtin_neon_vceqh_f16:
6932 case NEON::BI__builtin_neon_vcleh_f16:
6933 case NEON::BI__builtin_neon_vclth_f16:
6934 case NEON::BI__builtin_neon_vcgeh_f16:
6935 case NEON::BI__builtin_neon_vcgth_f16: {
6936 llvm::CmpInst::Predicate
P;
6937 switch (BuiltinID) {
6938 default: llvm_unreachable(
"missing builtin ID in switch!");
6939 case NEON::BI__builtin_neon_vceqh_f16: P = llvm::FCmpInst::FCMP_OEQ;
break;
6940 case NEON::BI__builtin_neon_vcleh_f16: P = llvm::FCmpInst::FCMP_OLE;
break;
6941 case NEON::BI__builtin_neon_vclth_f16: P = llvm::FCmpInst::FCMP_OLT;
break;
6942 case NEON::BI__builtin_neon_vcgeh_f16: P = llvm::FCmpInst::FCMP_OGE;
break;
6943 case NEON::BI__builtin_neon_vcgth_f16: P = llvm::FCmpInst::FCMP_OGT;
break;
6945 Ops.push_back(EmitScalarExpr(E->
getArg(1)));
6946 Ops[0] = Builder.CreateBitCast(Ops[0], HalfTy);
6947 Ops[1] = Builder.CreateBitCast(Ops[1], HalfTy);
6948 Ops[0] = Builder.CreateFCmp(P, Ops[0], Ops[1]);
6949 return Builder.CreateSExt(Ops[0], Int16Ty,
"vcmpd");
6951 case NEON::BI__builtin_neon_vceqd_s64:
6952 case NEON::BI__builtin_neon_vceqd_u64:
6953 case NEON::BI__builtin_neon_vcgtd_s64:
6954 case NEON::BI__builtin_neon_vcgtd_u64:
6955 case NEON::BI__builtin_neon_vcltd_s64:
6956 case NEON::BI__builtin_neon_vcltd_u64:
6957 case NEON::BI__builtin_neon_vcged_u64:
6958 case NEON::BI__builtin_neon_vcged_s64:
6959 case NEON::BI__builtin_neon_vcled_u64:
6960 case NEON::BI__builtin_neon_vcled_s64: {
6961 llvm::CmpInst::Predicate
P;
6962 switch (BuiltinID) {
6963 default: llvm_unreachable(
"missing builtin ID in switch!");
6964 case NEON::BI__builtin_neon_vceqd_s64:
6965 case NEON::BI__builtin_neon_vceqd_u64:P = llvm::ICmpInst::ICMP_EQ;
break;
6966 case NEON::BI__builtin_neon_vcgtd_s64:P = llvm::ICmpInst::ICMP_SGT;
break;
6967 case NEON::BI__builtin_neon_vcgtd_u64:P = llvm::ICmpInst::ICMP_UGT;
break;
6968 case NEON::BI__builtin_neon_vcltd_s64:P = llvm::ICmpInst::ICMP_SLT;
break;
6969 case NEON::BI__builtin_neon_vcltd_u64:P = llvm::ICmpInst::ICMP_ULT;
break;
6970 case NEON::BI__builtin_neon_vcged_u64:P = llvm::ICmpInst::ICMP_UGE;
break;
6971 case NEON::BI__builtin_neon_vcged_s64:P = llvm::ICmpInst::ICMP_SGE;
break;
6972 case NEON::BI__builtin_neon_vcled_u64:P = llvm::ICmpInst::ICMP_ULE;
break;
6973 case NEON::BI__builtin_neon_vcled_s64:P = llvm::ICmpInst::ICMP_SLE;
break;
6975 Ops.push_back(EmitScalarExpr(E->
getArg(1)));
6976 Ops[0] = Builder.CreateBitCast(Ops[0], Int64Ty);
6977 Ops[1] = Builder.CreateBitCast(Ops[1], Int64Ty);
6978 Ops[0] = Builder.CreateICmp(P, Ops[0], Ops[1]);
6979 return Builder.CreateSExt(Ops[0], Int64Ty,
"vceqd");
6981 case NEON::BI__builtin_neon_vtstd_s64:
6982 case NEON::BI__builtin_neon_vtstd_u64: {
6983 Ops.push_back(EmitScalarExpr(E->
getArg(1)));
6984 Ops[0] = Builder.CreateBitCast(Ops[0], Int64Ty);
6985 Ops[1] = Builder.CreateBitCast(Ops[1], Int64Ty);
6986 Ops[0] = Builder.CreateAnd(Ops[0], Ops[1]);
6987 Ops[0] = Builder.CreateICmp(ICmpInst::ICMP_NE, Ops[0],
6988 llvm::Constant::getNullValue(Int64Ty));
6989 return Builder.CreateSExt(Ops[0], Int64Ty,
"vtstd");
6991 case NEON::BI__builtin_neon_vset_lane_i8:
6992 case NEON::BI__builtin_neon_vset_lane_i16:
6993 case NEON::BI__builtin_neon_vset_lane_i32:
6994 case NEON::BI__builtin_neon_vset_lane_i64:
6995 case NEON::BI__builtin_neon_vset_lane_f32:
6996 case NEON::BI__builtin_neon_vsetq_lane_i8:
6997 case NEON::BI__builtin_neon_vsetq_lane_i16:
6998 case NEON::BI__builtin_neon_vsetq_lane_i32:
6999 case NEON::BI__builtin_neon_vsetq_lane_i64:
7000 case NEON::BI__builtin_neon_vsetq_lane_f32:
7001 Ops.push_back(EmitScalarExpr(E->
getArg(2)));
7002 return Builder.CreateInsertElement(Ops[1], Ops[0], Ops[2],
"vset_lane");
7003 case NEON::BI__builtin_neon_vset_lane_f64:
7005 Ops[1] = Builder.CreateBitCast(Ops[1],
7006 llvm::VectorType::get(DoubleTy, 1));
7007 Ops.push_back(EmitScalarExpr(E->
getArg(2)));
7008 return Builder.CreateInsertElement(Ops[1], Ops[0], Ops[2],
"vset_lane");
7009 case NEON::BI__builtin_neon_vsetq_lane_f64:
7011 Ops[1] = Builder.CreateBitCast(Ops[1],
7012 llvm::VectorType::get(DoubleTy, 2));
7013 Ops.push_back(EmitScalarExpr(E->
getArg(2)));
7014 return Builder.CreateInsertElement(Ops[1], Ops[0], Ops[2],
"vset_lane");
7016 case NEON::BI__builtin_neon_vget_lane_i8:
7017 case NEON::BI__builtin_neon_vdupb_lane_i8:
7018 Ops[0] = Builder.CreateBitCast(Ops[0], llvm::VectorType::get(Int8Ty, 8));
7019 return Builder.CreateExtractElement(Ops[0], EmitScalarExpr(E->
getArg(1)),
7021 case NEON::BI__builtin_neon_vgetq_lane_i8:
7022 case NEON::BI__builtin_neon_vdupb_laneq_i8:
7023 Ops[0] = Builder.CreateBitCast(Ops[0], llvm::VectorType::get(Int8Ty, 16));
7024 return Builder.CreateExtractElement(Ops[0], EmitScalarExpr(E->
getArg(1)),
7026 case NEON::BI__builtin_neon_vget_lane_i16:
7027 case NEON::BI__builtin_neon_vduph_lane_i16:
7028 Ops[0] = Builder.CreateBitCast(Ops[0], llvm::VectorType::get(Int16Ty, 4));
7029 return Builder.CreateExtractElement(Ops[0], EmitScalarExpr(E->
getArg(1)),
7031 case NEON::BI__builtin_neon_vgetq_lane_i16:
7032 case NEON::BI__builtin_neon_vduph_laneq_i16:
7033 Ops[0] = Builder.CreateBitCast(Ops[0], llvm::VectorType::get(Int16Ty, 8));
7034 return Builder.CreateExtractElement(Ops[0], EmitScalarExpr(E->
getArg(1)),
7036 case NEON::BI__builtin_neon_vget_lane_i32:
7037 case NEON::BI__builtin_neon_vdups_lane_i32:
7038 Ops[0] = Builder.CreateBitCast(Ops[0], llvm::VectorType::get(Int32Ty, 2));
7039 return Builder.CreateExtractElement(Ops[0], EmitScalarExpr(E->
getArg(1)),
7041 case NEON::BI__builtin_neon_vdups_lane_f32:
7042 Ops[0] = Builder.CreateBitCast(Ops[0],
7043 llvm::VectorType::get(FloatTy, 2));
7044 return Builder.CreateExtractElement(Ops[0], EmitScalarExpr(E->
getArg(1)),
7046 case NEON::BI__builtin_neon_vgetq_lane_i32:
7047 case NEON::BI__builtin_neon_vdups_laneq_i32:
7048 Ops[0] = Builder.CreateBitCast(Ops[0], llvm::VectorType::get(Int32Ty, 4));
7049 return Builder.CreateExtractElement(Ops[0], EmitScalarExpr(E->
getArg(1)),
7051 case NEON::BI__builtin_neon_vget_lane_i64:
7052 case NEON::BI__builtin_neon_vdupd_lane_i64:
7053 Ops[0] = Builder.CreateBitCast(Ops[0], llvm::VectorType::get(Int64Ty, 1));
7054 return Builder.CreateExtractElement(Ops[0], EmitScalarExpr(E->
getArg(1)),
7056 case NEON::BI__builtin_neon_vdupd_lane_f64:
7057 Ops[0] = Builder.CreateBitCast(Ops[0],
7058 llvm::VectorType::get(DoubleTy, 1));
7059 return Builder.CreateExtractElement(Ops[0], EmitScalarExpr(E->
getArg(1)),
7061 case NEON::BI__builtin_neon_vgetq_lane_i64:
7062 case NEON::BI__builtin_neon_vdupd_laneq_i64:
7063 Ops[0] = Builder.CreateBitCast(Ops[0], llvm::VectorType::get(Int64Ty, 2));
7064 return Builder.CreateExtractElement(Ops[0], EmitScalarExpr(E->
getArg(1)),
7066 case NEON::BI__builtin_neon_vget_lane_f32:
7067 Ops[0] = Builder.CreateBitCast(Ops[0],
7068 llvm::VectorType::get(FloatTy, 2));
7069 return Builder.CreateExtractElement(Ops[0], EmitScalarExpr(E->
getArg(1)),
7071 case NEON::BI__builtin_neon_vget_lane_f64:
7072 Ops[0] = Builder.CreateBitCast(Ops[0],
7073 llvm::VectorType::get(DoubleTy, 1));
7074 return Builder.CreateExtractElement(Ops[0], EmitScalarExpr(E->
getArg(1)),
7076 case NEON::BI__builtin_neon_vgetq_lane_f32:
7077 case NEON::BI__builtin_neon_vdups_laneq_f32:
7078 Ops[0] = Builder.CreateBitCast(Ops[0],
7079 llvm::VectorType::get(FloatTy, 4));
7080 return Builder.CreateExtractElement(Ops[0], EmitScalarExpr(E->
getArg(1)),
7082 case NEON::BI__builtin_neon_vgetq_lane_f64:
7083 case NEON::BI__builtin_neon_vdupd_laneq_f64:
7084 Ops[0] = Builder.CreateBitCast(Ops[0],
7085 llvm::VectorType::get(DoubleTy, 2));
7086 return Builder.CreateExtractElement(Ops[0], EmitScalarExpr(E->
getArg(1)),
7088 case NEON::BI__builtin_neon_vaddh_f16:
7089 Ops.push_back(EmitScalarExpr(E->
getArg(1)));
7090 return Builder.CreateFAdd(Ops[0], Ops[1],
"vaddh");
7091 case NEON::BI__builtin_neon_vsubh_f16:
7092 Ops.push_back(EmitScalarExpr(E->
getArg(1)));
7093 return Builder.CreateFSub(Ops[0], Ops[1],
"vsubh");
7094 case NEON::BI__builtin_neon_vmulh_f16:
7095 Ops.push_back(EmitScalarExpr(E->
getArg(1)));
7096 return Builder.CreateFMul(Ops[0], Ops[1],
"vmulh");
7097 case NEON::BI__builtin_neon_vdivh_f16:
7098 Ops.push_back(EmitScalarExpr(E->
getArg(1)));
7099 return Builder.CreateFDiv(Ops[0], Ops[1],
"vdivh");
7100 case NEON::BI__builtin_neon_vfmah_f16: {
7103 return Builder.CreateCall(F,
7104 {EmitScalarExpr(E->
getArg(1)), EmitScalarExpr(E->
getArg(2)), Ops[0]});
7106 case NEON::BI__builtin_neon_vfmsh_f16: {
7108 Value *Zero = llvm::ConstantFP::getZeroValueForNegation(HalfTy);
7109 Value* Sub = Builder.CreateFSub(Zero, EmitScalarExpr(E->
getArg(1)),
"vsubh");
7111 return Builder.CreateCall(F, {Sub, EmitScalarExpr(E->
getArg(2)), Ops[0]});
7113 case NEON::BI__builtin_neon_vaddd_s64:
7114 case NEON::BI__builtin_neon_vaddd_u64:
7115 return Builder.CreateAdd(Ops[0], EmitScalarExpr(E->
getArg(1)),
"vaddd");
7116 case NEON::BI__builtin_neon_vsubd_s64:
7117 case NEON::BI__builtin_neon_vsubd_u64:
7118 return Builder.CreateSub(Ops[0], EmitScalarExpr(E->
getArg(1)),
"vsubd");
7119 case NEON::BI__builtin_neon_vqdmlalh_s16:
7120 case NEON::BI__builtin_neon_vqdmlslh_s16: {
7122 ProductOps.push_back(vectorWrapScalar16(Ops[1]));
7123 ProductOps.push_back(vectorWrapScalar16(EmitScalarExpr(E->
getArg(2))));
7124 llvm::Type *VTy = llvm::VectorType::get(Int32Ty, 4);
7125 Ops[1] = EmitNeonCall(CGM.getIntrinsic(Intrinsic::aarch64_neon_sqdmull, VTy),
7126 ProductOps,
"vqdmlXl");
7127 Constant *CI = ConstantInt::get(SizeTy, 0);
7128 Ops[1] = Builder.CreateExtractElement(Ops[1], CI,
"lane0");
7130 unsigned AccumInt = BuiltinID == NEON::BI__builtin_neon_vqdmlalh_s16
7131 ? Intrinsic::aarch64_neon_sqadd
7132 : Intrinsic::aarch64_neon_sqsub;
7133 return EmitNeonCall(CGM.getIntrinsic(AccumInt, Int32Ty), Ops,
"vqdmlXl");
7135 case NEON::BI__builtin_neon_vqshlud_n_s64: {
7136 Ops.push_back(EmitScalarExpr(E->
getArg(1)));
7137 Ops[1] = Builder.CreateZExt(Ops[1], Int64Ty);
7138 return EmitNeonCall(CGM.getIntrinsic(Intrinsic::aarch64_neon_sqshlu, Int64Ty),
7141 case NEON::BI__builtin_neon_vqshld_n_u64:
7142 case NEON::BI__builtin_neon_vqshld_n_s64: {
7143 unsigned Int = BuiltinID == NEON::BI__builtin_neon_vqshld_n_u64
7144 ? Intrinsic::aarch64_neon_uqshl
7145 : Intrinsic::aarch64_neon_sqshl;
7146 Ops.push_back(EmitScalarExpr(E->
getArg(1)));
7147 Ops[1] = Builder.CreateZExt(Ops[1], Int64Ty);
7148 return EmitNeonCall(CGM.getIntrinsic(Int, Int64Ty), Ops,
"vqshl_n");
7150 case NEON::BI__builtin_neon_vrshrd_n_u64:
7151 case NEON::BI__builtin_neon_vrshrd_n_s64: {
7152 unsigned Int = BuiltinID == NEON::BI__builtin_neon_vrshrd_n_u64
7153 ? Intrinsic::aarch64_neon_urshl
7154 : Intrinsic::aarch64_neon_srshl;
7155 Ops.push_back(EmitScalarExpr(E->
getArg(1)));
7156 int SV = cast<ConstantInt>(Ops[1])->getSExtValue();
7157 Ops[1] = ConstantInt::get(Int64Ty, -SV);
7158 return EmitNeonCall(CGM.getIntrinsic(Int, Int64Ty), Ops,
"vrshr_n");
7160 case NEON::BI__builtin_neon_vrsrad_n_u64:
7161 case NEON::BI__builtin_neon_vrsrad_n_s64: {
7162 unsigned Int = BuiltinID == NEON::BI__builtin_neon_vrsrad_n_u64
7163 ? Intrinsic::aarch64_neon_urshl
7164 : Intrinsic::aarch64_neon_srshl;
7165 Ops[1] = Builder.CreateBitCast(Ops[1], Int64Ty);
7166 Ops.push_back(Builder.CreateNeg(EmitScalarExpr(E->
getArg(2))));
7167 Ops[1] = Builder.CreateCall(CGM.getIntrinsic(Int, Int64Ty),
7168 {Ops[1], Builder.CreateSExt(Ops[2], Int64Ty)});
7169 return Builder.CreateAdd(Ops[0], Builder.CreateBitCast(Ops[1], Int64Ty));
7171 case NEON::BI__builtin_neon_vshld_n_s64:
7172 case NEON::BI__builtin_neon_vshld_n_u64: {
7173 llvm::ConstantInt *Amt = cast<ConstantInt>(EmitScalarExpr(E->
getArg(1)));
7174 return Builder.CreateShl(
7175 Ops[0], ConstantInt::get(Int64Ty, Amt->getZExtValue()),
"shld_n");
7177 case NEON::BI__builtin_neon_vshrd_n_s64: {
7178 llvm::ConstantInt *Amt = cast<ConstantInt>(EmitScalarExpr(E->
getArg(1)));
7179 return Builder.CreateAShr(
7180 Ops[0], ConstantInt::get(Int64Ty,
std::min(static_cast<uint64_t>(63),
7181 Amt->getZExtValue())),
7184 case NEON::BI__builtin_neon_vshrd_n_u64: {
7185 llvm::ConstantInt *Amt = cast<ConstantInt>(EmitScalarExpr(E->
getArg(1)));
7186 uint64_t ShiftAmt = Amt->getZExtValue();
7189 return ConstantInt::get(Int64Ty, 0);
7190 return Builder.CreateLShr(Ops[0], ConstantInt::get(Int64Ty, ShiftAmt),
7193 case NEON::BI__builtin_neon_vsrad_n_s64: {
7194 llvm::ConstantInt *Amt = cast<ConstantInt>(EmitScalarExpr(E->
getArg(2)));
7195 Ops[1] = Builder.CreateAShr(
7196 Ops[1], ConstantInt::get(Int64Ty,
std::min(static_cast<uint64_t>(63),
7197 Amt->getZExtValue())),
7199 return Builder.CreateAdd(Ops[0], Ops[1]);
7201 case NEON::BI__builtin_neon_vsrad_n_u64: {
7202 llvm::ConstantInt *Amt = cast<ConstantInt>(EmitScalarExpr(E->
getArg(2)));
7203 uint64_t ShiftAmt = Amt->getZExtValue();
7208 Ops[1] = Builder.CreateLShr(Ops[1], ConstantInt::get(Int64Ty, ShiftAmt),
7210 return Builder.CreateAdd(Ops[0], Ops[1]);
7212 case NEON::BI__builtin_neon_vqdmlalh_lane_s16:
7213 case NEON::BI__builtin_neon_vqdmlalh_laneq_s16:
7214 case NEON::BI__builtin_neon_vqdmlslh_lane_s16:
7215 case NEON::BI__builtin_neon_vqdmlslh_laneq_s16: {
7216 Ops[2] = Builder.CreateExtractElement(Ops[2], EmitScalarExpr(E->
getArg(3)),
7219 ProductOps.push_back(vectorWrapScalar16(Ops[1]));
7220 ProductOps.push_back(vectorWrapScalar16(Ops[2]));
7221 llvm::Type *VTy = llvm::VectorType::get(Int32Ty, 4);
7222 Ops[1] = EmitNeonCall(CGM.getIntrinsic(Intrinsic::aarch64_neon_sqdmull, VTy),
7223 ProductOps,
"vqdmlXl");
7224 Constant *CI = ConstantInt::get(SizeTy, 0);
7225 Ops[1] = Builder.CreateExtractElement(Ops[1], CI,
"lane0");
7228 unsigned AccInt = (BuiltinID == NEON::BI__builtin_neon_vqdmlalh_lane_s16 ||
7229 BuiltinID == NEON::BI__builtin_neon_vqdmlalh_laneq_s16)
7230 ? Intrinsic::aarch64_neon_sqadd
7231 : Intrinsic::aarch64_neon_sqsub;
7232 return EmitNeonCall(CGM.getIntrinsic(AccInt, Int32Ty), Ops,
"vqdmlXl");
7234 case NEON::BI__builtin_neon_vqdmlals_s32:
7235 case NEON::BI__builtin_neon_vqdmlsls_s32: {
7237 ProductOps.push_back(Ops[1]);
7238 ProductOps.push_back(EmitScalarExpr(E->
getArg(2)));
7240 EmitNeonCall(CGM.getIntrinsic(Intrinsic::aarch64_neon_sqdmulls_scalar),
7241 ProductOps,
"vqdmlXl");
7243 unsigned AccumInt = BuiltinID == NEON::BI__builtin_neon_vqdmlals_s32
7244 ? Intrinsic::aarch64_neon_sqadd
7245 : Intrinsic::aarch64_neon_sqsub;
7246 return EmitNeonCall(CGM.getIntrinsic(AccumInt, Int64Ty), Ops,
"vqdmlXl");
7248 case NEON::BI__builtin_neon_vqdmlals_lane_s32:
7249 case NEON::BI__builtin_neon_vqdmlals_laneq_s32:
7250 case NEON::BI__builtin_neon_vqdmlsls_lane_s32:
7251 case NEON::BI__builtin_neon_vqdmlsls_laneq_s32: {
7252 Ops[2] = Builder.CreateExtractElement(Ops[2], EmitScalarExpr(E->
getArg(3)),
7255 ProductOps.push_back(Ops[1]);
7256 ProductOps.push_back(Ops[2]);
7258 EmitNeonCall(CGM.getIntrinsic(Intrinsic::aarch64_neon_sqdmulls_scalar),
7259 ProductOps,
"vqdmlXl");
7262 unsigned AccInt = (BuiltinID == NEON::BI__builtin_neon_vqdmlals_lane_s32 ||
7263 BuiltinID == NEON::BI__builtin_neon_vqdmlals_laneq_s32)
7264 ? Intrinsic::aarch64_neon_sqadd
7265 : Intrinsic::aarch64_neon_sqsub;
7266 return EmitNeonCall(CGM.getIntrinsic(AccInt, Int64Ty), Ops,
"vqdmlXl");
7278 AArch64SIMDIntrinsicsProvenSorted);
7281 return EmitCommonNeonBuiltinExpr(
7282 Builtin->BuiltinID, Builtin->LLVMIntrinsic, Builtin->AltLLVMIntrinsic,
7283 Builtin->NameHint, Builtin->TypeModifier, E, Ops,
7290 switch (BuiltinID) {
7291 default:
return nullptr;
7292 case NEON::BI__builtin_neon_vbsl_v:
7293 case NEON::BI__builtin_neon_vbslq_v: {
7294 llvm::Type *BitTy = llvm::VectorType::getInteger(VTy);
7295 Ops[0] = Builder.CreateBitCast(Ops[0], BitTy,
"vbsl");
7296 Ops[1] = Builder.CreateBitCast(Ops[1], BitTy,
"vbsl");
7297 Ops[2] = Builder.CreateBitCast(Ops[2], BitTy,
"vbsl");
7299 Ops[1] = Builder.CreateAnd(Ops[0], Ops[1],
"vbsl");
7300 Ops[2] = Builder.CreateAnd(Builder.CreateNot(Ops[0]), Ops[2],
"vbsl");
7301 Ops[0] = Builder.CreateOr(Ops[1], Ops[2],
"vbsl");
7302 return Builder.CreateBitCast(Ops[0], Ty);
7304 case NEON::BI__builtin_neon_vfma_lane_v:
7305 case NEON::BI__builtin_neon_vfmaq_lane_v: {
7308 Value *Addend = Ops[0];
7309 Value *Multiplicand = Ops[1];
7310 Value *LaneSource = Ops[2];
7311 Ops[0] = Multiplicand;
7312 Ops[1] = LaneSource;
7316 llvm::Type *SourceTy = BuiltinID == NEON::BI__builtin_neon_vfmaq_lane_v ?
7317 llvm::VectorType::get(VTy->getElementType(), VTy->getNumElements() / 2) :
7319 llvm::Constant *cst = cast<Constant>(Ops[3]);
7320 Value *SV = llvm::ConstantVector::getSplat(VTy->getNumElements(), cst);
7321 Ops[1] = Builder.CreateBitCast(Ops[1], SourceTy);
7322 Ops[1] = Builder.CreateShuffleVector(Ops[1], Ops[1], SV,
"lane");
7326 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops,
"fmla");
7328 case NEON::BI__builtin_neon_vfma_laneq_v: {
7329 llvm::VectorType *VTy = cast<llvm::VectorType>(Ty);
7331 if (VTy && VTy->getElementType() == DoubleTy) {
7332 Ops[0] = Builder.CreateBitCast(Ops[0], DoubleTy);
7333 Ops[1] = Builder.CreateBitCast(Ops[1], DoubleTy);
7336 Ops[2] = Builder.CreateBitCast(Ops[2], VTy);
7337 Ops[2] = Builder.CreateExtractElement(Ops[2], Ops[3],
"extract");
7339 Value *Result = Builder.CreateCall(F, {Ops[1], Ops[2], Ops[0]});
7340 return Builder.CreateBitCast(Result, Ty);
7343 Ops[0] = Builder.CreateBitCast(Ops[0], Ty);
7344 Ops[1] = Builder.CreateBitCast(Ops[1], Ty);
7346 llvm::Type *STy = llvm::VectorType::get(VTy->getElementType(),
7347 VTy->getNumElements() * 2);
7348 Ops[2] = Builder.CreateBitCast(Ops[2], STy);
7349 Value* SV = llvm::ConstantVector::getSplat(VTy->getNumElements(),
7350 cast<ConstantInt>(Ops[3]));
7351 Ops[2] = Builder.CreateShuffleVector(Ops[2], Ops[2], SV,
"lane");
7353 return Builder.CreateCall(F, {Ops[2], Ops[1], Ops[0]});
7355 case NEON::BI__builtin_neon_vfmaq_laneq_v: {
7357 Ops[0] = Builder.CreateBitCast(Ops[0], Ty);
7358 Ops[1] = Builder.CreateBitCast(Ops[1], Ty);
7360 Ops[2] = Builder.CreateBitCast(Ops[2], Ty);
7361 Ops[2] = EmitNeonSplat(Ops[2], cast<ConstantInt>(Ops[3]));
7362 return Builder.CreateCall(F, {Ops[2], Ops[1], Ops[0]});
7364 case NEON::BI__builtin_neon_vfmah_lane_f16:
7365 case NEON::BI__builtin_neon_vfmas_lane_f32:
7366 case NEON::BI__builtin_neon_vfmah_laneq_f16:
7367 case NEON::BI__builtin_neon_vfmas_laneq_f32:
7368 case NEON::BI__builtin_neon_vfmad_lane_f64:
7369 case NEON::BI__builtin_neon_vfmad_laneq_f64: {
7370 Ops.push_back(EmitScalarExpr(E->
getArg(3)));
7373 Ops[2] = Builder.CreateExtractElement(Ops[2], Ops[3],
"extract");
7374 return Builder.CreateCall(F, {Ops[1], Ops[2], Ops[0]});
7376 case NEON::BI__builtin_neon_vmull_v:
7378 Int = usgn ? Intrinsic::aarch64_neon_umull : Intrinsic::aarch64_neon_smull;
7379 if (Type.
isPoly()) Int = Intrinsic::aarch64_neon_pmull;
7380 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops,
"vmull");
7381 case NEON::BI__builtin_neon_vmax_v:
7382 case NEON::BI__builtin_neon_vmaxq_v:
7384 Int = usgn ? Intrinsic::aarch64_neon_umax : Intrinsic::aarch64_neon_smax;
7385 if (Ty->isFPOrFPVectorTy()) Int = Intrinsic::aarch64_neon_fmax;
7386 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops,
"vmax");
7387 case NEON::BI__builtin_neon_vmaxh_f16: {
7388 Ops.push_back(EmitScalarExpr(E->
getArg(1)));
7389 Int = Intrinsic::aarch64_neon_fmax;
7390 return EmitNeonCall(CGM.getIntrinsic(Int, HalfTy), Ops,
"vmax");
7392 case NEON::BI__builtin_neon_vmin_v:
7393 case NEON::BI__builtin_neon_vminq_v:
7395 Int = usgn ? Intrinsic::aarch64_neon_umin : Intrinsic::aarch64_neon_smin;
7396 if (Ty->isFPOrFPVectorTy()) Int = Intrinsic::aarch64_neon_fmin;
7397 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops,
"vmin");
7398 case NEON::BI__builtin_neon_vminh_f16: {
7399 Ops.push_back(EmitScalarExpr(E->
getArg(1)));
7400 Int = Intrinsic::aarch64_neon_fmin;
7401 return EmitNeonCall(CGM.getIntrinsic(Int, HalfTy), Ops,
"vmin");
7403 case NEON::BI__builtin_neon_vabd_v:
7404 case NEON::BI__builtin_neon_vabdq_v:
7406 Int = usgn ? Intrinsic::aarch64_neon_uabd : Intrinsic::aarch64_neon_sabd;
7407 if (Ty->isFPOrFPVectorTy()) Int = Intrinsic::aarch64_neon_fabd;
7408 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops,
"vabd");
7409 case NEON::BI__builtin_neon_vpadal_v:
7410 case NEON::BI__builtin_neon_vpadalq_v: {
7411 unsigned ArgElts = VTy->getNumElements();
7412 llvm::IntegerType *EltTy = cast<IntegerType>(VTy->getElementType());
7413 unsigned BitWidth = EltTy->getBitWidth();
7415 llvm::IntegerType::get(getLLVMContext(), BitWidth/2), 2*ArgElts);
7417 Int = usgn ? Intrinsic::aarch64_neon_uaddlp : Intrinsic::aarch64_neon_saddlp;
7419 TmpOps.push_back(Ops[1]);
7420 Function *F = CGM.getIntrinsic(Int, Tys);
7421 llvm::Value *tmp = EmitNeonCall(F, TmpOps,
"vpadal");
7422 llvm::Value *addend = Builder.CreateBitCast(Ops[0], tmp->getType());
7423 return Builder.CreateAdd(tmp, addend);
7425 case NEON::BI__builtin_neon_vpmin_v:
7426 case NEON::BI__builtin_neon_vpminq_v:
7428 Int = usgn ? Intrinsic::aarch64_neon_uminp : Intrinsic::aarch64_neon_sminp;
7429 if (Ty->isFPOrFPVectorTy()) Int = Intrinsic::aarch64_neon_fminp;
7430 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops,
"vpmin");
7431 case NEON::BI__builtin_neon_vpmax_v:
7432 case NEON::BI__builtin_neon_vpmaxq_v:
7434 Int = usgn ? Intrinsic::aarch64_neon_umaxp : Intrinsic::aarch64_neon_smaxp;
7435 if (Ty->isFPOrFPVectorTy()) Int = Intrinsic::aarch64_neon_fmaxp;
7436 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops,
"vpmax");
7437 case NEON::BI__builtin_neon_vminnm_v:
7438 case NEON::BI__builtin_neon_vminnmq_v:
7439 Int = Intrinsic::aarch64_neon_fminnm;
7440 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops,
"vminnm");
7441 case NEON::BI__builtin_neon_vminnmh_f16:
7442 Ops.push_back(EmitScalarExpr(E->
getArg(1)));
7443 Int = Intrinsic::aarch64_neon_fminnm;
7444 return EmitNeonCall(CGM.getIntrinsic(Int, HalfTy), Ops,
"vminnm");
7445 case NEON::BI__builtin_neon_vmaxnm_v:
7446 case NEON::BI__builtin_neon_vmaxnmq_v:
7447 Int = Intrinsic::aarch64_neon_fmaxnm;
7448 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops,
"vmaxnm");
7449 case NEON::BI__builtin_neon_vmaxnmh_f16:
7450 Ops.push_back(EmitScalarExpr(E->
getArg(1)));
7451 Int = Intrinsic::aarch64_neon_fmaxnm;
7452 return EmitNeonCall(CGM.getIntrinsic(Int, HalfTy), Ops,
"vmaxnm");
7453 case NEON::BI__builtin_neon_vrecpss_f32: {
7454 Ops.push_back(EmitScalarExpr(E->
getArg(1)));
7455 return EmitNeonCall(CGM.getIntrinsic(Intrinsic::aarch64_neon_frecps, FloatTy),
7458 case NEON::BI__builtin_neon_vrecpsd_f64:
7459 Ops.push_back(EmitScalarExpr(E->
getArg(1)));
7460 return EmitNeonCall(CGM.getIntrinsic(Intrinsic::aarch64_neon_frecps, DoubleTy),
7462 case NEON::BI__builtin_neon_vrecpsh_f16:
7463 Ops.push_back(EmitScalarExpr(E->
getArg(1)));
7464 return EmitNeonCall(CGM.getIntrinsic(Intrinsic::aarch64_neon_frecps, HalfTy),
7466 case NEON::BI__builtin_neon_vqshrun_n_v:
7467 Int = Intrinsic::aarch64_neon_sqshrun;
7468 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops,
"vqshrun_n");
7469 case NEON::BI__builtin_neon_vqrshrun_n_v:
7470 Int = Intrinsic::aarch64_neon_sqrshrun;
7471 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops,
"vqrshrun_n");
7472 case NEON::BI__builtin_neon_vqshrn_n_v:
7473 Int = usgn ? Intrinsic::aarch64_neon_uqshrn : Intrinsic::aarch64_neon_sqshrn;
7474 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops,
"vqshrn_n");
7475 case NEON::BI__builtin_neon_vrshrn_n_v:
7476 Int = Intrinsic::aarch64_neon_rshrn;
7477 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops,
"vrshrn_n");
7478 case NEON::BI__builtin_neon_vqrshrn_n_v:
7479 Int = usgn ? Intrinsic::aarch64_neon_uqrshrn : Intrinsic::aarch64_neon_sqrshrn;
7480 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops,
"vqrshrn_n");
7481 case NEON::BI__builtin_neon_vrndah_f16: {
7482 Ops.push_back(EmitScalarExpr(E->
getArg(0)));
7484 return EmitNeonCall(CGM.getIntrinsic(Int, HalfTy), Ops,
"vrnda");
7486 case NEON::BI__builtin_neon_vrnda_v:
7487 case NEON::BI__builtin_neon_vrndaq_v: {
7489 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops,
"vrnda");
7491 case NEON::BI__builtin_neon_vrndih_f16: {
7492 Ops.push_back(EmitScalarExpr(E->
getArg(0)));
7494 return EmitNeonCall(CGM.getIntrinsic(Int, HalfTy), Ops,
"vrndi");
7496 case NEON::BI__builtin_neon_vrndmh_f16: {
7497 Ops.push_back(EmitScalarExpr(E->
getArg(0)));
7499 return EmitNeonCall(CGM.getIntrinsic(Int, HalfTy), Ops,
"vrndm");
7501 case NEON::BI__builtin_neon_vrndm_v:
7502 case NEON::BI__builtin_neon_vrndmq_v: {
7504 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops,
"vrndm");
7506 case NEON::BI__builtin_neon_vrndnh_f16: {
7507 Ops.push_back(EmitScalarExpr(E->
getArg(0)));
7508 Int = Intrinsic::aarch64_neon_frintn;
7509 return EmitNeonCall(CGM.getIntrinsic(Int, HalfTy), Ops,
"vrndn");
7511 case NEON::BI__builtin_neon_vrndn_v:
7512 case NEON::BI__builtin_neon_vrndnq_v: {
7513 Int = Intrinsic::aarch64_neon_frintn;
7514 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops,
"vrndn");
7516 case NEON::BI__builtin_neon_vrndns_f32: {
7517 Ops.push_back(EmitScalarExpr(E->
getArg(0)));
7518 Int = Intrinsic::aarch64_neon_frintn;
7519 return EmitNeonCall(CGM.getIntrinsic(Int, FloatTy), Ops,
"vrndn");
7521 case NEON::BI__builtin_neon_vrndph_f16: {
7522 Ops.push_back(EmitScalarExpr(E->
getArg(0)));
7524 return EmitNeonCall(CGM.getIntrinsic(Int, HalfTy), Ops,
"vrndp");
7526 case NEON::BI__builtin_neon_vrndp_v:
7527 case NEON::BI__builtin_neon_vrndpq_v: {
7529 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops,
"vrndp");
7531 case NEON::BI__builtin_neon_vrndxh_f16: {
7532 Ops.push_back(EmitScalarExpr(E->
getArg(0)));
7534 return EmitNeonCall(CGM.getIntrinsic(Int, HalfTy), Ops,
"vrndx");
7536 case NEON::BI__builtin_neon_vrndx_v:
7537 case NEON::BI__builtin_neon_vrndxq_v: {
7539 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops,
"vrndx");
7541 case NEON::BI__builtin_neon_vrndh_f16: {
7542 Ops.push_back(EmitScalarExpr(E->
getArg(0)));
7544 return EmitNeonCall(CGM.getIntrinsic(Int, HalfTy), Ops,
"vrndz");
7546 case NEON::BI__builtin_neon_vrnd_v:
7547 case NEON::BI__builtin_neon_vrndq_v: {
7549 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops,
"vrndz");
7551 case NEON::BI__builtin_neon_vcvt_f64_v:
7552 case NEON::BI__builtin_neon_vcvtq_f64_v:
7553 Ops[0] = Builder.CreateBitCast(Ops[0], Ty);
7555 return usgn ? Builder.CreateUIToFP(Ops[0], Ty,
"vcvt")
7556 : Builder.CreateSIToFP(Ops[0], Ty,
"vcvt");
7557 case NEON::BI__builtin_neon_vcvt_f64_f32: {
7559 "unexpected vcvt_f64_f32 builtin");
7561 Ops[0] = Builder.CreateBitCast(Ops[0],
GetNeonType(
this, SrcFlag));
7563 return Builder.CreateFPExt(Ops[0], Ty,
"vcvt");
7565 case NEON::BI__builtin_neon_vcvt_f32_f64: {
7567 "unexpected vcvt_f32_f64 builtin");
7569 Ops[0] = Builder.CreateBitCast(Ops[0],
GetNeonType(
this, SrcFlag));
7571 return Builder.CreateFPTrunc(Ops[0], Ty,
"vcvt");
7573 case NEON::BI__builtin_neon_vcvt_s32_v:
7574 case NEON::BI__builtin_neon_vcvt_u32_v:
7575 case NEON::BI__builtin_neon_vcvt_s64_v:
7576 case NEON::BI__builtin_neon_vcvt_u64_v:
7577 case NEON::BI__builtin_neon_vcvt_s16_v:
7578 case NEON::BI__builtin_neon_vcvt_u16_v:
7579 case NEON::BI__builtin_neon_vcvtq_s32_v:
7580 case NEON::BI__builtin_neon_vcvtq_u32_v:
7581 case NEON::BI__builtin_neon_vcvtq_s64_v:
7582 case NEON::BI__builtin_neon_vcvtq_u64_v:
7583 case NEON::BI__builtin_neon_vcvtq_s16_v:
7584 case NEON::BI__builtin_neon_vcvtq_u16_v: {
7587 return Builder.CreateFPToUI(Ops[0], Ty);
7588 return Builder.CreateFPToSI(Ops[0], Ty);
7590 case NEON::BI__builtin_neon_vcvta_s16_v:
7591 case NEON::BI__builtin_neon_vcvta_u16_v:
7592 case NEON::BI__builtin_neon_vcvta_s32_v:
7593 case NEON::BI__builtin_neon_vcvtaq_s16_v:
7594 case NEON::BI__builtin_neon_vcvtaq_s32_v:
7595 case NEON::BI__builtin_neon_vcvta_u32_v:
7596 case NEON::BI__builtin_neon_vcvtaq_u16_v:
7597 case NEON::BI__builtin_neon_vcvtaq_u32_v:
7598 case NEON::BI__builtin_neon_vcvta_s64_v:
7599 case NEON::BI__builtin_neon_vcvtaq_s64_v:
7600 case NEON::BI__builtin_neon_vcvta_u64_v:
7601 case NEON::BI__builtin_neon_vcvtaq_u64_v: {
7602 Int = usgn ? Intrinsic::aarch64_neon_fcvtau : Intrinsic::aarch64_neon_fcvtas;
7604 return EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops,
"vcvta");
7606 case NEON::BI__builtin_neon_vcvtm_s16_v:
7607 case NEON::BI__builtin_neon_vcvtm_s32_v:
7608 case NEON::BI__builtin_neon_vcvtmq_s16_v:
7609 case NEON::BI__builtin_neon_vcvtmq_s32_v:
7610 case NEON::BI__builtin_neon_vcvtm_u16_v:
7611 case NEON::BI__builtin_neon_vcvtm_u32_v:
7612 case NEON::BI__builtin_neon_vcvtmq_u16_v:
7613 case NEON::BI__builtin_neon_vcvtmq_u32_v:
7614 case NEON::BI__builtin_neon_vcvtm_s64_v:
7615 case NEON::BI__builtin_neon_vcvtmq_s64_v:
7616 case NEON::BI__builtin_neon_vcvtm_u64_v:
7617 case NEON::BI__builtin_neon_vcvtmq_u64_v: {
7618 Int = usgn ? Intrinsic::aarch64_neon_fcvtmu : Intrinsic::aarch64_neon_fcvtms;
7620 return EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops,
"vcvtm");
7622 case NEON::BI__builtin_neon_vcvtn_s16_v:
7623 case NEON::BI__builtin_neon_vcvtn_s32_v:
7624 case NEON::BI__builtin_neon_vcvtnq_s16_v:
7625 case NEON::BI__builtin_neon_vcvtnq_s32_v:
7626 case NEON::BI__builtin_neon_vcvtn_u16_v:
7627 case NEON::BI__builtin_neon_vcvtn_u32_v:
7628 case NEON::BI__builtin_neon_vcvtnq_u16_v:
7629 case NEON::BI__builtin_neon_vcvtnq_u32_v:
7630 case NEON::BI__builtin_neon_vcvtn_s64_v:
7631 case NEON::BI__builtin_neon_vcvtnq_s64_v:
7632 case NEON::BI__builtin_neon_vcvtn_u64_v:
7633 case NEON::BI__builtin_neon_vcvtnq_u64_v: {
7634 Int = usgn ? Intrinsic::aarch64_neon_fcvtnu : Intrinsic::aarch64_neon_fcvtns;
7636 return EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops,
"vcvtn");
7638 case NEON::BI__builtin_neon_vcvtp_s16_v:
7639 case NEON::BI__builtin_neon_vcvtp_s32_v:
7640 case NEON::BI__builtin_neon_vcvtpq_s16_v:
7641 case NEON::BI__builtin_neon_vcvtpq_s32_v:
7642 case NEON::BI__builtin_neon_vcvtp_u16_v:
7643 case NEON::BI__builtin_neon_vcvtp_u32_v:
7644 case NEON::BI__builtin_neon_vcvtpq_u16_v:
7645 case NEON::BI__builtin_neon_vcvtpq_u32_v:
7646 case NEON::BI__builtin_neon_vcvtp_s64_v:
7647 case NEON::BI__builtin_neon_vcvtpq_s64_v:
7648 case NEON::BI__builtin_neon_vcvtp_u64_v:
7649 case NEON::BI__builtin_neon_vcvtpq_u64_v: {
7650 Int = usgn ? Intrinsic::aarch64_neon_fcvtpu : Intrinsic::aarch64_neon_fcvtps;
7652 return EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops,
"vcvtp");
7654 case NEON::BI__builtin_neon_vmulx_v:
7655 case NEON::BI__builtin_neon_vmulxq_v: {
7656 Int = Intrinsic::aarch64_neon_fmulx;
7657 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops,
"vmulx");
7659 case NEON::BI__builtin_neon_vmulxh_lane_f16:
7660 case NEON::BI__builtin_neon_vmulxh_laneq_f16: {
7663 Ops.push_back(EmitScalarExpr(E->
getArg(2)));
7664 Ops[1] = Builder.CreateExtractElement(Ops[1], Ops[2],
"extract");
7666 Int = Intrinsic::aarch64_neon_fmulx;
7667 return EmitNeonCall(CGM.getIntrinsic(Int, HalfTy), Ops,
"vmulx");
7669 case NEON::BI__builtin_neon_vmul_lane_v:
7670 case NEON::BI__builtin_neon_vmul_laneq_v: {
7673 if (BuiltinID == NEON::BI__builtin_neon_vmul_laneq_v)
7675 Ops[0] = Builder.CreateBitCast(Ops[0], DoubleTy);
7678 Ops[1] = Builder.CreateBitCast(Ops[1], VTy);
7679 Ops[1] = Builder.CreateExtractElement(Ops[1], Ops[2],
"extract");
7680 Value *Result = Builder.CreateFMul(Ops[0], Ops[1]);
7681 return Builder.CreateBitCast(Result, Ty);
7683 case NEON::BI__builtin_neon_vnegd_s64:
7684 return Builder.CreateNeg(EmitScalarExpr(E->
getArg(0)),
"vnegd");
7685 case NEON::BI__builtin_neon_vnegh_f16:
7686 return Builder.CreateFNeg(EmitScalarExpr(E->
getArg(0)),
"vnegh");
7687 case NEON::BI__builtin_neon_vpmaxnm_v:
7688 case NEON::BI__builtin_neon_vpmaxnmq_v: {
7689 Int = Intrinsic::aarch64_neon_fmaxnmp;
7690 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops,
"vpmaxnm");
7692 case NEON::BI__builtin_neon_vpminnm_v:
7693 case NEON::BI__builtin_neon_vpminnmq_v: {
7694 Int = Intrinsic::aarch64_neon_fminnmp;
7695 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops,
"vpminnm");
7697 case NEON::BI__builtin_neon_vsqrth_f16: {
7698 Ops.push_back(EmitScalarExpr(E->
getArg(0)));
7700 return EmitNeonCall(CGM.getIntrinsic(Int, HalfTy), Ops,
"vsqrt");
7702 case NEON::BI__builtin_neon_vsqrt_v:
7703 case NEON::BI__builtin_neon_vsqrtq_v: {
7705 Ops[0] = Builder.CreateBitCast(Ops[0], Ty);
7706 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops,
"vsqrt");
7708 case NEON::BI__builtin_neon_vrbit_v:
7709 case NEON::BI__builtin_neon_vrbitq_v: {
7710 Int = Intrinsic::aarch64_neon_rbit;
7711 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops,
"vrbit");
7713 case NEON::BI__builtin_neon_vaddv_u8:
7717 case NEON::BI__builtin_neon_vaddv_s8: {
7718 Int = usgn ? Intrinsic::aarch64_neon_uaddv : Intrinsic::aarch64_neon_saddv;
7720 VTy = llvm::VectorType::get(Int8Ty, 8);
7722 Ops.push_back(EmitScalarExpr(E->
getArg(0)));
7723 Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops,
"vaddv");
7724 return Builder.CreateTrunc(Ops[0], Int8Ty);
7726 case NEON::BI__builtin_neon_vaddv_u16:
7729 case NEON::BI__builtin_neon_vaddv_s16: {
7730 Int = usgn ? Intrinsic::aarch64_neon_uaddv : Intrinsic::aarch64_neon_saddv;
7732 VTy = llvm::VectorType::get(Int16Ty, 4);
7734 Ops.push_back(EmitScalarExpr(E->
getArg(0)));
7735 Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops,
"vaddv");
7736 return Builder.CreateTrunc(Ops[0], Int16Ty);
7738 case NEON::BI__builtin_neon_vaddvq_u8:
7741 case NEON::BI__builtin_neon_vaddvq_s8: {
7742 Int = usgn ? Intrinsic::aarch64_neon_uaddv : Intrinsic::aarch64_neon_saddv;
7744 VTy = llvm::VectorType::get(Int8Ty, 16);
7746 Ops.push_back(EmitScalarExpr(E->
getArg(0)));
7747 Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops,
"vaddv");
7748 return Builder.CreateTrunc(Ops[0], Int8Ty);
7750 case NEON::BI__builtin_neon_vaddvq_u16:
7753 case NEON::BI__builtin_neon_vaddvq_s16: {
7754 Int = usgn ? Intrinsic::aarch64_neon_uaddv : Intrinsic::aarch64_neon_saddv;
7756 VTy = llvm::VectorType::get(Int16Ty, 8);
7758 Ops.push_back(EmitScalarExpr(E->
getArg(0)));
7759 Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops,
"vaddv");
7760 return Builder.CreateTrunc(Ops[0], Int16Ty);
7762 case NEON::BI__builtin_neon_vmaxv_u8: {
7763 Int = Intrinsic::aarch64_neon_umaxv;
7765 VTy = llvm::VectorType::get(Int8Ty, 8);
7767 Ops.push_back(EmitScalarExpr(E->
getArg(0)));
7768 Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops,
"vmaxv");
7769 return Builder.CreateTrunc(Ops[0], Int8Ty);
7771 case NEON::BI__builtin_neon_vmaxv_u16: {
7772 Int = Intrinsic::aarch64_neon_umaxv;
7774 VTy = llvm::VectorType::get(Int16Ty, 4);
7776 Ops.push_back(EmitScalarExpr(E->
getArg(0)));
7777 Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops,
"vmaxv");
7778 return Builder.CreateTrunc(Ops[0], Int16Ty);
7780 case NEON::BI__builtin_neon_vmaxvq_u8: {
7781 Int = Intrinsic::aarch64_neon_umaxv;
7783 VTy = llvm::VectorType::get(Int8Ty, 16);
7785 Ops.push_back(EmitScalarExpr(E->
getArg(0)));
7786 Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops,
"vmaxv");
7787 return Builder.CreateTrunc(Ops[0], Int8Ty);
7789 case NEON::BI__builtin_neon_vmaxvq_u16: {
7790 Int = Intrinsic::aarch64_neon_umaxv;
7792 VTy = llvm::VectorType::get(Int16Ty, 8);
7794 Ops.push_back(EmitScalarExpr(E->
getArg(0)));
7795 Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops,
"vmaxv");
7796 return Builder.CreateTrunc(Ops[0], Int16Ty);
7798 case NEON::BI__builtin_neon_vmaxv_s8: {
7799 Int = Intrinsic::aarch64_neon_smaxv;
7801 VTy = llvm::VectorType::get(Int8Ty, 8);
7803 Ops.push_back(EmitScalarExpr(E->
getArg(0)));
7804 Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops,
"vmaxv");
7805 return Builder.CreateTrunc(Ops[0], Int8Ty);
7807 case NEON::BI__builtin_neon_vmaxv_s16: {
7808 Int = Intrinsic::aarch64_neon_smaxv;
7810 VTy = llvm::VectorType::get(Int16Ty, 4);
7812 Ops.push_back(EmitScalarExpr(E->
getArg(0)));
7813 Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops,
"vmaxv");
7814 return Builder.CreateTrunc(Ops[0], Int16Ty);
7816 case NEON::BI__builtin_neon_vmaxvq_s8: {
7817 Int = Intrinsic::aarch64_neon_smaxv;
7819 VTy = llvm::VectorType::get(Int8Ty, 16);
7821 Ops.push_back(EmitScalarExpr(E->
getArg(0)));
7822 Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops,
"vmaxv");
7823 return Builder.CreateTrunc(Ops[0], Int8Ty);
7825 case NEON::BI__builtin_neon_vmaxvq_s16: {
7826 Int = Intrinsic::aarch64_neon_smaxv;
7828 VTy = llvm::VectorType::get(Int16Ty, 8);
7830 Ops.push_back(EmitScalarExpr(E->
getArg(0)));
7831 Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops,
"vmaxv");
7832 return Builder.CreateTrunc(Ops[0], Int16Ty);
7834 case NEON::BI__builtin_neon_vmaxv_f16: {
7835 Int = Intrinsic::aarch64_neon_fmaxv;
7837 VTy = llvm::VectorType::get(HalfTy, 4);
7839 Ops.push_back(EmitScalarExpr(E->
getArg(0)));
7840 Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops,
"vmaxv");
7841 return Builder.CreateTrunc(Ops[0], HalfTy);
7843 case NEON::BI__builtin_neon_vmaxvq_f16: {
7844 Int = Intrinsic::aarch64_neon_fmaxv;
7846 VTy = llvm::VectorType::get(HalfTy, 8);
7848 Ops.push_back(EmitScalarExpr(E->
getArg(0)));
7849 Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops,
"vmaxv");
7850 return Builder.CreateTrunc(Ops[0], HalfTy);
7852 case NEON::BI__builtin_neon_vminv_u8: {
7853 Int = Intrinsic::aarch64_neon_uminv;
7855 VTy = llvm::VectorType::get(Int8Ty, 8);
7857 Ops.push_back(EmitScalarExpr(E->
getArg(0)));
7858 Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops,
"vminv");
7859 return Builder.CreateTrunc(Ops[0], Int8Ty);
7861 case NEON::BI__builtin_neon_vminv_u16: {
7862 Int = Intrinsic::aarch64_neon_uminv;
7864 VTy = llvm::VectorType::get(Int16Ty, 4);
7866 Ops.push_back(EmitScalarExpr(E->
getArg(0)));
7867 Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops,
"vminv");
7868 return Builder.CreateTrunc(Ops[0], Int16Ty);
7870 case NEON::BI__builtin_neon_vminvq_u8: {
7871 Int = Intrinsic::aarch64_neon_uminv;
7873 VTy = llvm::VectorType::get(Int8Ty, 16);
7875 Ops.push_back(EmitScalarExpr(E->
getArg(0)));
7876 Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops,
"vminv");
7877 return Builder.CreateTrunc(Ops[0], Int8Ty);
7879 case NEON::BI__builtin_neon_vminvq_u16: {
7880 Int = Intrinsic::aarch64_neon_uminv;
7882 VTy = llvm::VectorType::get(Int16Ty, 8);
7884 Ops.push_back(EmitScalarExpr(E->
getArg(0)));
7885 Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops,
"vminv");
7886 return Builder.CreateTrunc(Ops[0], Int16Ty);
7888 case NEON::BI__builtin_neon_vminv_s8: {
7889 Int = Intrinsic::aarch64_neon_sminv;
7891 VTy = llvm::VectorType::get(Int8Ty, 8);
7893 Ops.push_back(EmitScalarExpr(E->
getArg(0)));
7894 Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops,
"vminv");
7895 return Builder.CreateTrunc(Ops[0], Int8Ty);
7897 case NEON::BI__builtin_neon_vminv_s16: {
7898 Int = Intrinsic::aarch64_neon_sminv;
7900 VTy = llvm::VectorType::get(Int16Ty, 4);
7902 Ops.push_back(EmitScalarExpr(E->
getArg(0)));
7903 Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops,
"vminv");
7904 return Builder.CreateTrunc(Ops[0], Int16Ty);
7906 case NEON::BI__builtin_neon_vminvq_s8: {
7907 Int = Intrinsic::aarch64_neon_sminv;
7909 VTy = llvm::VectorType::get(Int8Ty, 16);
7911 Ops.push_back(EmitScalarExpr(E->
getArg(0)));
7912 Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops,
"vminv");
7913 return Builder.CreateTrunc(Ops[0], Int8Ty);
7915 case NEON::BI__builtin_neon_vminvq_s16: {
7916 Int = Intrinsic::aarch64_neon_sminv;
7918 VTy = llvm::VectorType::get(Int16Ty, 8);
7920 Ops.push_back(EmitScalarExpr(E->
getArg(0)));
7921 Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops,
"vminv");
7922 return Builder.CreateTrunc(Ops[0], Int16Ty);
7924 case NEON::BI__builtin_neon_vminv_f16: {
7925 Int = Intrinsic::aarch64_neon_fminv;
7927 VTy = llvm::VectorType::get(HalfTy, 4);
7929 Ops.push_back(EmitScalarExpr(E->
getArg(0)));
7930 Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops,
"vminv");
7931 return Builder.CreateTrunc(Ops[0], HalfTy);
7933 case NEON::BI__builtin_neon_vminvq_f16: {
7934 Int = Intrinsic::aarch64_neon_fminv;
7936 VTy = llvm::VectorType::get(HalfTy, 8);
7938 Ops.push_back(EmitScalarExpr(E->
getArg(0)));
7939 Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops,
"vminv");
7940 return Builder.CreateTrunc(Ops[0], HalfTy);
7942 case NEON::BI__builtin_neon_vmaxnmv_f16: {
7943 Int = Intrinsic::aarch64_neon_fmaxnmv;
7945 VTy = llvm::VectorType::get(HalfTy, 4);
7947 Ops.push_back(EmitScalarExpr(E->
getArg(0)));
7948 Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops,
"vmaxnmv");
7949 return Builder.CreateTrunc(Ops[0], HalfTy);
7951 case NEON::BI__builtin_neon_vmaxnmvq_f16: {
7952 Int = Intrinsic::aarch64_neon_fmaxnmv;
7954 VTy = llvm::VectorType::get(HalfTy, 8);
7956 Ops.push_back(EmitScalarExpr(E->
getArg(0)));
7957 Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops,
"vmaxnmv");
7958 return Builder.CreateTrunc(Ops[0], HalfTy);
7960 case NEON::BI__builtin_neon_vminnmv_f16: {
7961 Int = Intrinsic::aarch64_neon_fminnmv;
7963 VTy = llvm::VectorType::get(HalfTy, 4);
7965 Ops.push_back(EmitScalarExpr(E->
getArg(0)));
7966 Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops,
"vminnmv");
7967 return Builder.CreateTrunc(Ops[0], HalfTy);
7969 case NEON::BI__builtin_neon_vminnmvq_f16: {
7970 Int = Intrinsic::aarch64_neon_fminnmv;
7972 VTy = llvm::VectorType::get(HalfTy, 8);
7974 Ops.push_back(EmitScalarExpr(E->
getArg(0)));
7975 Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops,
"vminnmv");
7976 return Builder.CreateTrunc(Ops[0], HalfTy);
7978 case NEON::BI__builtin_neon_vmul_n_f64: {
7979 Ops[0] = Builder.CreateBitCast(Ops[0], DoubleTy);
7980 Value *RHS = Builder.CreateBitCast(EmitScalarExpr(E->
getArg(1)), DoubleTy);
7981 return Builder.CreateFMul(Ops[0], RHS);
7983 case NEON::BI__builtin_neon_vaddlv_u8: {
7984 Int = Intrinsic::aarch64_neon_uaddlv;
7986 VTy = llvm::VectorType::get(Int8Ty, 8);
7988 Ops.push_back(EmitScalarExpr(E->
getArg(0)));
7989 Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops,
"vaddlv");
7990 return Builder.CreateTrunc(Ops[0], Int16Ty);
7992 case NEON::BI__builtin_neon_vaddlv_u16: {
7993 Int = Intrinsic::aarch64_neon_uaddlv;
7995 VTy = llvm::VectorType::get(Int16Ty, 4);
7997 Ops.push_back(EmitScalarExpr(E->
getArg(0)));
7998 return EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops,
"vaddlv");
8000 case NEON::BI__builtin_neon_vaddlvq_u8: {
8001 Int = Intrinsic::aarch64_neon_uaddlv;
8003 VTy = llvm::VectorType::get(Int8Ty, 16);
8005 Ops.push_back(EmitScalarExpr(E->
getArg(0)));
8006 Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops,
"vaddlv");
8007 return Builder.CreateTrunc(Ops[0], Int16Ty);
8009 case NEON::BI__builtin_neon_vaddlvq_u16: {
8010 Int = Intrinsic::aarch64_neon_uaddlv;
8012 VTy = llvm::VectorType::get(Int16Ty, 8);
8014 Ops.push_back(EmitScalarExpr(E->
getArg(0)));
8015 return EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops,
"vaddlv");
8017 case NEON::BI__builtin_neon_vaddlv_s8: {
8018 Int = Intrinsic::aarch64_neon_saddlv;
8020 VTy = llvm::VectorType::get(Int8Ty, 8);
8022 Ops.push_back(EmitScalarExpr(E->
getArg(0)));
8023 Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops,
"vaddlv");
8024 return Builder.CreateTrunc(Ops[0], Int16Ty);
8026 case NEON::BI__builtin_neon_vaddlv_s16: {
8027 Int = Intrinsic::aarch64_neon_saddlv;
8029 VTy = llvm::VectorType::get(Int16Ty, 4);
8031 Ops.push_back(EmitScalarExpr(E->
getArg(0)));
8032 return EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops,
"vaddlv");
8034 case NEON::BI__builtin_neon_vaddlvq_s8: {
8035 Int = Intrinsic::aarch64_neon_saddlv;
8037 VTy = llvm::VectorType::get(Int8Ty, 16);
8039 Ops.push_back(EmitScalarExpr(E->
getArg(0)));
8040 Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops,
"vaddlv");
8041 return Builder.CreateTrunc(Ops[0], Int16Ty);
8043 case NEON::BI__builtin_neon_vaddlvq_s16: {
8044 Int = Intrinsic::aarch64_neon_saddlv;
8046 VTy = llvm::VectorType::get(Int16Ty, 8);
8048 Ops.push_back(EmitScalarExpr(E->
getArg(0)));
8049 return EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops,
"vaddlv");
8051 case NEON::BI__builtin_neon_vsri_n_v:
8052 case NEON::BI__builtin_neon_vsriq_n_v: {
8053 Int = Intrinsic::aarch64_neon_vsri;
8054 llvm::Function *Intrin = CGM.getIntrinsic(Int, Ty);
8055 return EmitNeonCall(Intrin, Ops,
"vsri_n");
8057 case NEON::BI__builtin_neon_vsli_n_v:
8058 case NEON::BI__builtin_neon_vsliq_n_v: {
8059 Int = Intrinsic::aarch64_neon_vsli;
8060 llvm::Function *Intrin = CGM.getIntrinsic(Int, Ty);
8061 return EmitNeonCall(Intrin, Ops,
"vsli_n");
8063 case NEON::BI__builtin_neon_vsra_n_v:
8064 case NEON::BI__builtin_neon_vsraq_n_v:
8065 Ops[0] = Builder.CreateBitCast(Ops[0], Ty);
8066 Ops[1] = EmitNeonRShiftImm(Ops[1], Ops[2], Ty, usgn,
"vsra_n");
8067 return Builder.CreateAdd(Ops[0], Ops[1]);
8068 case NEON::BI__builtin_neon_vrsra_n_v:
8069 case NEON::BI__builtin_neon_vrsraq_n_v: {
8070 Int = usgn ? Intrinsic::aarch64_neon_urshl : Intrinsic::aarch64_neon_srshl;
8072 TmpOps.push_back(Ops[1]);
8073 TmpOps.push_back(Ops[2]);
8074 Function* F = CGM.getIntrinsic(Int, Ty);
8075 llvm::Value *tmp = EmitNeonCall(F, TmpOps,
"vrshr_n", 1,
true);
8076 Ops[0] = Builder.CreateBitCast(Ops[0], VTy);
8077 return Builder.CreateAdd(Ops[0], tmp);
8079 case NEON::BI__builtin_neon_vld1_v:
8080 case NEON::BI__builtin_neon_vld1q_v: {
8081 Ops[0] = Builder.CreateBitCast(Ops[0], llvm::PointerType::getUnqual(VTy));
8083 BuiltinID == NEON::BI__builtin_neon_vld1_v ? 8 : 16);
8084 return Builder.CreateAlignedLoad(VTy, Ops[0], Alignment);
8086 case NEON::BI__builtin_neon_vst1_v:
8087 case NEON::BI__builtin_neon_vst1q_v:
8088 Ops[0] = Builder.CreateBitCast(Ops[0], llvm::PointerType::getUnqual(VTy));
8089 Ops[1] = Builder.CreateBitCast(Ops[1], VTy);
8090 return Builder.CreateDefaultAlignedStore(Ops[1], Ops[0]);
8091 case NEON::BI__builtin_neon_vld1_lane_v:
8092 case NEON::BI__builtin_neon_vld1q_lane_v: {
8093 Ops[1] = Builder.CreateBitCast(Ops[1], Ty);
8094 Ty = llvm::PointerType::getUnqual(VTy->getElementType());
8095 Ops[0] = Builder.CreateBitCast(Ops[0], Ty);
8097 BuiltinID == NEON::BI__builtin_neon_vld1_lane_v ? 8 : 16);
8099 Builder.CreateAlignedLoad(VTy->getElementType(), Ops[0], Alignment);
8100 return Builder.CreateInsertElement(Ops[1], Ops[0], Ops[2],
"vld1_lane");
8102 case NEON::BI__builtin_neon_vld1_dup_v:
8103 case NEON::BI__builtin_neon_vld1q_dup_v: {
8104 Value *V = UndefValue::get(Ty);
8105 Ty = llvm::PointerType::getUnqual(VTy->getElementType());
8106 Ops[0] = Builder.CreateBitCast(Ops[0], Ty);
8108 BuiltinID == NEON::BI__builtin_neon_vld1_dup_v ? 8 : 16);
8110 Builder.CreateAlignedLoad(VTy->getElementType(), Ops[0], Alignment);
8111 llvm::Constant *CI = ConstantInt::get(Int32Ty, 0);
8112 Ops[0] = Builder.CreateInsertElement(V, Ops[0], CI);
8113 return EmitNeonSplat(Ops[0], CI);
8115 case NEON::BI__builtin_neon_vst1_lane_v:
8116 case NEON::BI__builtin_neon_vst1q_lane_v:
8117 Ops[1] = Builder.CreateBitCast(Ops[1], Ty);
8118 Ops[1] = Builder.CreateExtractElement(Ops[1], Ops[2]);
8119 Ty = llvm::PointerType::getUnqual(Ops[1]->getType());
8120 return Builder.CreateDefaultAlignedStore(Ops[1],
8121 Builder.CreateBitCast(Ops[0], Ty));
8122 case NEON::BI__builtin_neon_vld2_v:
8123 case NEON::BI__builtin_neon_vld2q_v: {
8124 llvm::Type *PTy = llvm::PointerType::getUnqual(VTy);
8125 Ops[1] = Builder.CreateBitCast(Ops[1], PTy);
8127 Function *F = CGM.getIntrinsic(Intrinsic::aarch64_neon_ld2, Tys);
8128 Ops[1] = Builder.CreateCall(F, Ops[1],
"vld2");
8129 Ops[0] = Builder.CreateBitCast(Ops[0],
8130 llvm::PointerType::getUnqual(Ops[1]->getType()));
8131 return Builder.CreateDefaultAlignedStore(Ops[1], Ops[0]);
8133 case NEON::BI__builtin_neon_vld3_v:
8134 case NEON::BI__builtin_neon_vld3q_v: {
8135 llvm::Type *PTy = llvm::PointerType::getUnqual(VTy);
8136 Ops[1] = Builder.CreateBitCast(Ops[1], PTy);
8138 Function *F = CGM.getIntrinsic(Intrinsic::aarch64_neon_ld3, Tys);
8139 Ops[1] = Builder.CreateCall(F, Ops[1],
"vld3");
8140 Ops[0] = Builder.CreateBitCast(Ops[0],
8141 llvm::PointerType::getUnqual(Ops[1]->getType()));
8142 return Builder.CreateDefaultAlignedStore(Ops[1], Ops[0]);
8144 case NEON::BI__builtin_neon_vld4_v:
8145 case NEON::BI__builtin_neon_vld4q_v: {
8146 llvm::Type *PTy = llvm::PointerType::getUnqual(VTy);
8147 Ops[1] = Builder.CreateBitCast(Ops[1], PTy);
8149 Function *F = CGM.getIntrinsic(Intrinsic::aarch64_neon_ld4, Tys);
8150 Ops[1] = Builder.CreateCall(F, Ops[1],
"vld4");
8151 Ops[0] = Builder.CreateBitCast(Ops[0],
8152 llvm::PointerType::getUnqual(Ops[1]->getType()));
8153 return Builder.CreateDefaultAlignedStore(Ops[1], Ops[0]);
8155 case NEON::BI__builtin_neon_vld2_dup_v:
8156 case NEON::BI__builtin_neon_vld2q_dup_v: {
8158 llvm::PointerType::getUnqual(VTy->getElementType());
8159 Ops[1] = Builder.CreateBitCast(Ops[1], PTy);
8161 Function *F = CGM.getIntrinsic(Intrinsic::aarch64_neon_ld2r, Tys);
8162 Ops[1] = Builder.CreateCall(F, Ops[1],
"vld2");
8163 Ops[0] = Builder.CreateBitCast(Ops[0],
8164 llvm::PointerType::getUnqual(Ops[1]->getType()));
8165 return Builder.CreateDefaultAlignedStore(Ops[1], Ops[0]);
8167 case NEON::BI__builtin_neon_vld3_dup_v:
8168 case NEON::BI__builtin_neon_vld3q_dup_v: {
8170 llvm::PointerType::getUnqual(VTy->getElementType());
8171 Ops[1] = Builder.CreateBitCast(Ops[1], PTy);
8173 Function *F = CGM.getIntrinsic(Intrinsic::aarch64_neon_ld3r, Tys);
8174 Ops[1] = Builder.CreateCall(F, Ops[1],
"vld3");
8175 Ops[0] = Builder.CreateBitCast(Ops[0],
8176 llvm::PointerType::getUnqual(Ops[1]->getType()));
8177 return Builder.CreateDefaultAlignedStore(Ops[1], Ops[0]);
8179 case NEON::BI__builtin_neon_vld4_dup_v:
8180 case NEON::BI__builtin_neon_vld4q_dup_v: {
8182 llvm::PointerType::getUnqual(VTy->getElementType());
8183 Ops[1] = Builder.CreateBitCast(Ops[1], PTy);
8185 Function *F = CGM.getIntrinsic(Intrinsic::aarch64_neon_ld4r, Tys);
8186 Ops[1] = Builder.CreateCall(F, Ops[1],
"vld4");
8187 Ops[0] = Builder.CreateBitCast(Ops[0],
8188 llvm::PointerType::getUnqual(Ops[1]->getType()));
8189 return Builder.CreateDefaultAlignedStore(Ops[1], Ops[0]);
8191 case NEON::BI__builtin_neon_vld2_lane_v:
8192 case NEON::BI__builtin_neon_vld2q_lane_v: {
8193 llvm::Type *Tys[2] = { VTy, Ops[1]->getType() };
8194 Function *F = CGM.getIntrinsic(Intrinsic::aarch64_neon_ld2lane, Tys);
8195 Ops.push_back(Ops[1]);
8196 Ops.erase(Ops.begin()+1);
8197 Ops[1] = Builder.CreateBitCast(Ops[1], Ty);
8198 Ops[2] = Builder.CreateBitCast(Ops[2], Ty);
8199 Ops[3] = Builder.CreateZExt(Ops[3], Int64Ty);
8200 Ops[1] = Builder.CreateCall(F, makeArrayRef(Ops).slice(1),
"vld2_lane");
8201 Ty = llvm::PointerType::getUnqual(Ops[1]->getType());
8202 Ops[0] = Builder.CreateBitCast(Ops[0], Ty);
8203 return Builder.CreateDefaultAlignedStore(Ops[1], Ops[0]);
8205 case NEON::BI__builtin_neon_vld3_lane_v:
8206 case NEON::BI__builtin_neon_vld3q_lane_v: {
8207 llvm::Type *Tys[2] = { VTy, Ops[1]->getType() };
8208 Function *F = CGM.getIntrinsic(Intrinsic::aarch64_neon_ld3lane, Tys);
8209 Ops.push_back(Ops[1]);
8210 Ops.erase(Ops.begin()+1);
8211 Ops[1] = Builder.CreateBitCast(Ops[1], Ty);
8212 Ops[2] = Builder.CreateBitCast(Ops[2], Ty);
8213 Ops[3] = Builder.CreateBitCast(Ops[3], Ty);
8214 Ops[4] = Builder.CreateZExt(Ops[4], Int64Ty);
8215 Ops[1] = Builder.CreateCall(F, makeArrayRef(Ops).slice(1),
"vld3_lane");
8216 Ty = llvm::PointerType::getUnqual(Ops[1]->getType());
8217 Ops[0] = Builder.CreateBitCast(Ops[0], Ty);
8218 return Builder.CreateDefaultAlignedStore(Ops[1], Ops[0]);
8220 case NEON::BI__builtin_neon_vld4_lane_v:
8221 case NEON::BI__builtin_neon_vld4q_lane_v: {
8222 llvm::Type *Tys[2] = { VTy, Ops[1]->getType() };
8223 Function *F = CGM.getIntrinsic(Intrinsic::aarch64_neon_ld4lane, Tys);
8224 Ops.push_back(Ops[1]);
8225 Ops.erase(Ops.begin()+1);
8226 Ops[1] = Builder.CreateBitCast(Ops[1], Ty);
8227 Ops[2] = Builder.CreateBitCast(Ops[2], Ty);
8228 Ops[3] = Builder.CreateBitCast(Ops[3], Ty);
8229 Ops[4] = Builder.CreateBitCast(Ops[4], Ty);
8230 Ops[5] = Builder.CreateZExt(Ops[5], Int64Ty);
8231 Ops[1] = Builder.CreateCall(F, makeArrayRef(Ops).slice(1),
"vld4_lane");
8232 Ty = llvm::PointerType::getUnqual(Ops[1]->getType());
8233 Ops[0] = Builder.CreateBitCast(Ops[0], Ty);
8234 return Builder.CreateDefaultAlignedStore(Ops[1], Ops[0]);
8236 case NEON::BI__builtin_neon_vst2_v:
8237 case NEON::BI__builtin_neon_vst2q_v: {
8238 Ops.push_back(Ops[0]);
8239 Ops.erase(Ops.begin());
8240 llvm::Type *Tys[2] = { VTy, Ops[2]->getType() };
8241 return EmitNeonCall(CGM.getIntrinsic(Intrinsic::aarch64_neon_st2, Tys),
8244 case NEON::BI__builtin_neon_vst2_lane_v:
8245 case NEON::BI__builtin_neon_vst2q_lane_v: {
8246 Ops.push_back(Ops[0]);
8247 Ops.erase(Ops.begin());
8248 Ops[2] = Builder.CreateZExt(Ops[2], Int64Ty);
8249 llvm::Type *Tys[2] = { VTy, Ops[3]->getType() };
8250 return EmitNeonCall(CGM.getIntrinsic(Intrinsic::aarch64_neon_st2lane, Tys),
8253 case NEON::BI__builtin_neon_vst3_v:
8254 case NEON::BI__builtin_neon_vst3q_v: {
8255 Ops.push_back(Ops[0]);
8256 Ops.erase(Ops.begin());
8257 llvm::Type *Tys[2] = { VTy, Ops[3]->getType() };
8258 return EmitNeonCall(CGM.getIntrinsic(Intrinsic::aarch64_neon_st3, Tys),
8261 case NEON::BI__builtin_neon_vst3_lane_v:
8262 case NEON::BI__builtin_neon_vst3q_lane_v: {
8263 Ops.push_back(Ops[0]);
8264 Ops.erase(Ops.begin());
8265 Ops[3] = Builder.CreateZExt(Ops[3], Int64Ty);
8266 llvm::Type *Tys[2] = { VTy, Ops[4]->getType() };
8267 return EmitNeonCall(CGM.getIntrinsic(Intrinsic::aarch64_neon_st3lane, Tys),
8270 case NEON::BI__builtin_neon_vst4_v:
8271 case NEON::BI__builtin_neon_vst4q_v: {
8272 Ops.push_back(Ops[0]);
8273 Ops.erase(Ops.begin());
8274 llvm::Type *Tys[2] = { VTy, Ops[4]->getType() };
8275 return EmitNeonCall(CGM.getIntrinsic(Intrinsic::aarch64_neon_st4, Tys),
8278 case NEON::BI__builtin_neon_vst4_lane_v:
8279 case NEON::BI__builtin_neon_vst4q_lane_v: {
8280 Ops.push_back(Ops[0]);
8281 Ops.erase(Ops.begin());
8282 Ops[4] = Builder.CreateZExt(Ops[4], Int64Ty);
8283 llvm::Type *Tys[2] = { VTy, Ops[5]->getType() };
8284 return EmitNeonCall(CGM.getIntrinsic(Intrinsic::aarch64_neon_st4lane, Tys),
8287 case NEON::BI__builtin_neon_vtrn_v:
8288 case NEON::BI__builtin_neon_vtrnq_v: {
8289 Ops[0] = Builder.CreateBitCast(Ops[0], llvm::PointerType::getUnqual(Ty));
8290 Ops[1] = Builder.CreateBitCast(Ops[1], Ty);
8291 Ops[2] = Builder.CreateBitCast(Ops[2], Ty);
8292 Value *SV =
nullptr;
8294 for (
unsigned vi = 0; vi != 2; ++vi) {
8296 for (
unsigned i = 0, e = VTy->getNumElements(); i != e; i += 2) {
8297 Indices.push_back(i+vi);
8298 Indices.push_back(i+e+vi);
8300 Value *Addr = Builder.CreateConstInBoundsGEP1_32(Ty, Ops[0], vi);
8301 SV = Builder.CreateShuffleVector(Ops[1], Ops[2], Indices,
"vtrn");
8302 SV = Builder.CreateDefaultAlignedStore(SV, Addr);
8306 case NEON::BI__builtin_neon_vuzp_v:
8307 case NEON::BI__builtin_neon_vuzpq_v: {
8308 Ops[0] = Builder.CreateBitCast(Ops[0], llvm::PointerType::getUnqual(Ty));
8309 Ops[1] = Builder.CreateBitCast(Ops[1], Ty);
8310 Ops[2] = Builder.CreateBitCast(Ops[2], Ty);
8311 Value *SV =
nullptr;
8313 for (
unsigned vi = 0; vi != 2; ++vi) {
8315 for (
unsigned i = 0, e = VTy->getNumElements(); i != e; ++i)
8316 Indices.push_back(2*i+vi);
8318 Value *Addr = Builder.CreateConstInBoundsGEP1_32(Ty, Ops[0], vi);
8319 SV = Builder.CreateShuffleVector(Ops[1], Ops[2], Indices,
"vuzp");
8320 SV = Builder.CreateDefaultAlignedStore(SV, Addr);
8324 case NEON::BI__builtin_neon_vzip_v:
8325 case NEON::BI__builtin_neon_vzipq_v: {
8326 Ops[0] = Builder.CreateBitCast(Ops[0], llvm::PointerType::getUnqual(Ty));
8327 Ops[1] = Builder.CreateBitCast(Ops[1], Ty);
8328 Ops[2] = Builder.CreateBitCast(Ops[2], Ty);
8329 Value *SV =
nullptr;
8331 for (
unsigned vi = 0; vi != 2; ++vi) {
8333 for (
unsigned i = 0, e = VTy->getNumElements(); i != e; i += 2) {
8334 Indices.push_back((i + vi*e) >> 1);
8335 Indices.push_back(((i + vi*e) >> 1)+e);
8337 Value *Addr = Builder.CreateConstInBoundsGEP1_32(Ty, Ops[0], vi);
8338 SV = Builder.CreateShuffleVector(Ops[1], Ops[2], Indices,
"vzip");
8339 SV = Builder.CreateDefaultAlignedStore(SV, Addr);
8343 case NEON::BI__builtin_neon_vqtbl1q_v: {
8344 return EmitNeonCall(CGM.getIntrinsic(Intrinsic::aarch64_neon_tbl1, Ty),
8347 case NEON::BI__builtin_neon_vqtbl2q_v: {
8348 return EmitNeonCall(CGM.getIntrinsic(Intrinsic::aarch64_neon_tbl2, Ty),
8351 case NEON::BI__builtin_neon_vqtbl3q_v: {
8352 return EmitNeonCall(CGM.getIntrinsic(Intrinsic::aarch64_neon_tbl3, Ty),
8355 case NEON::BI__builtin_neon_vqtbl4q_v: {
8356 return EmitNeonCall(CGM.getIntrinsic(Intrinsic::aarch64_neon_tbl4, Ty),
8359 case NEON::BI__builtin_neon_vqtbx1q_v: {
8360 return EmitNeonCall(CGM.getIntrinsic(Intrinsic::aarch64_neon_tbx1, Ty),
8363 case NEON::BI__builtin_neon_vqtbx2q_v: {
8364 return EmitNeonCall(CGM.getIntrinsic(Intrinsic::aarch64_neon_tbx2, Ty),
8367 case NEON::BI__builtin_neon_vqtbx3q_v: {
8368 return EmitNeonCall(CGM.getIntrinsic(Intrinsic::aarch64_neon_tbx3, Ty),
8371 case NEON::BI__builtin_neon_vqtbx4q_v: {
8372 return EmitNeonCall(CGM.getIntrinsic(Intrinsic::aarch64_neon_tbx4, Ty),
8375 case NEON::BI__builtin_neon_vsqadd_v:
8376 case NEON::BI__builtin_neon_vsqaddq_v: {
8377 Int = Intrinsic::aarch64_neon_usqadd;
8378 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops,
"vsqadd");
8380 case NEON::BI__builtin_neon_vuqadd_v:
8381 case NEON::BI__builtin_neon_vuqaddq_v: {
8382 Int = Intrinsic::aarch64_neon_suqadd;
8383 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops,
"vuqadd");
8385 case AArch64::BI__iso_volatile_load8:
8386 case AArch64::BI__iso_volatile_load16:
8387 case AArch64::BI__iso_volatile_load32:
8388 case AArch64::BI__iso_volatile_load64:
8389 return EmitISOVolatileLoad(E);
8390 case AArch64::BI__iso_volatile_store8:
8391 case AArch64::BI__iso_volatile_store16:
8392 case AArch64::BI__iso_volatile_store32:
8393 case AArch64::BI__iso_volatile_store64:
8394 return EmitISOVolatileStore(E);
8395 case AArch64::BI_BitScanForward:
8396 case AArch64::BI_BitScanForward64:
8397 return EmitMSVCBuiltinExpr(MSVCIntrin::_BitScanForward, E);
8398 case AArch64::BI_BitScanReverse:
8399 case AArch64::BI_BitScanReverse64:
8400 return EmitMSVCBuiltinExpr(MSVCIntrin::_BitScanReverse, E);
8401 case AArch64::BI_InterlockedAnd64:
8402 return EmitMSVCBuiltinExpr(MSVCIntrin::_InterlockedAnd, E);
8403 case AArch64::BI_InterlockedExchange64:
8404 return EmitMSVCBuiltinExpr(MSVCIntrin::_InterlockedExchange, E);
8405 case AArch64::BI_InterlockedExchangeAdd64:
8406 return EmitMSVCBuiltinExpr(MSVCIntrin::_InterlockedExchangeAdd, E);
8407 case AArch64::BI_InterlockedExchangeSub64:
8408 return EmitMSVCBuiltinExpr(MSVCIntrin::_InterlockedExchangeSub, E);
8409 case AArch64::BI_InterlockedOr64:
8410 return EmitMSVCBuiltinExpr(MSVCIntrin::_InterlockedOr, E);
8411 case AArch64::BI_InterlockedXor64:
8412 return EmitMSVCBuiltinExpr(MSVCIntrin::_InterlockedXor, E);
8413 case AArch64::BI_InterlockedDecrement64:
8414 return EmitMSVCBuiltinExpr(MSVCIntrin::_InterlockedDecrement, E);
8415 case AArch64::BI_InterlockedIncrement64:
8416 return EmitMSVCBuiltinExpr(MSVCIntrin::_InterlockedIncrement, E);
8422 assert((Ops.size() & (Ops.size() - 1)) == 0 &&
8423 "Not a power-of-two sized vector!");
8424 bool AllConstants =
true;
8425 for (
unsigned i = 0, e = Ops.size(); i != e && AllConstants; ++i)
8426 AllConstants &= isa<Constant>(Ops[i]);
8431 for (
unsigned i = 0, e = Ops.size(); i != e; ++i)
8432 CstOps.push_back(cast<Constant>(Ops[i]));
8433 return llvm::ConstantVector::get(CstOps);
8438 llvm::UndefValue::get(llvm::VectorType::get(Ops[0]->getType(), Ops.size()));
8440 for (
unsigned i = 0, e = Ops.size(); i != e; ++i)
8441 Result = Builder.CreateInsertElement(Result, Ops[i], Builder.getInt32(i));
8450 llvm::VectorType *MaskTy = llvm::VectorType::get(CGF.
Builder.getInt1Ty(),
8451 cast<IntegerType>(Mask->getType())->getBitWidth());
8457 uint32_t Indices[4];
8458 for (
unsigned i = 0; i != NumElts; ++i)
8460 MaskVec = CGF.
Builder.CreateShuffleVector(MaskVec, MaskVec,
8461 makeArrayRef(Indices, NumElts),
8472 llvm::PointerType::getUnqual(Ops[1]->getType()));
8475 Ops[1]->getType()->getVectorNumElements());
8477 return CGF.
Builder.CreateMaskedStore(Ops[1], Ptr, Align, MaskVec);
8484 llvm::PointerType::getUnqual(Ops[1]->getType()));
8487 Ops[1]->getType()->getVectorNumElements());
8489 return CGF.
Builder.CreateMaskedLoad(Ptr, Align, MaskVec, Ops[1]);
8495 llvm::Type *PtrTy = ResultTy->getVectorElementType();
8499 llvm::PointerType::getUnqual(PtrTy));
8502 ResultTy->getVectorNumElements());
8504 llvm::Function *F = CGF.
CGM.
getIntrinsic(Intrinsic::masked_expandload,
8506 return CGF.
Builder.CreateCall(F, { Ptr, MaskVec, Ops[1] });
8512 llvm::Type *PtrTy = ResultTy->getVectorElementType();
8516 llvm::PointerType::getUnqual(PtrTy));
8519 ResultTy->getVectorNumElements());
8521 llvm::Function *F = CGF.
CGM.
getIntrinsic(Intrinsic::masked_compressstore,
8523 return CGF.
Builder.CreateCall(F, { Ops[1], Ptr, MaskVec });
8528 bool InvertLHS =
false) {
8533 LHS = CGF.
Builder.CreateNot(LHS);
8543 if (
const auto *C = dyn_cast<Constant>(Mask))
8544 if (C->isAllOnesValue())
8547 Mask =
getMaskVecValue(CGF, Mask, Op0->getType()->getVectorNumElements());
8549 return CGF.
Builder.CreateSelect(Mask, Op0, Op1);
8555 if (
const auto *C = dyn_cast<Constant>(Mask))
8556 if (C->isAllOnesValue())
8559 llvm::VectorType *MaskTy =
8560 llvm::VectorType::get(CGF.
Builder.getInt1Ty(),
8561 Mask->getType()->getIntegerBitWidth());
8563 Mask = CGF.
Builder.CreateExtractElement(Mask, (uint64_t)0);
8564 return CGF.
Builder.CreateSelect(Mask, Op0, Op1);
8568 unsigned NumElts,
Value *MaskIn) {
8570 const auto *C = dyn_cast<Constant>(MaskIn);
8571 if (!C || !C->isAllOnesValue())
8576 uint32_t Indices[8];
8577 for (
unsigned i = 0; i != NumElts; ++i)
8579 for (
unsigned i = NumElts; i != 8; ++i)
8580 Indices[i] = i % NumElts + NumElts;
8581 Cmp = CGF.
Builder.CreateShuffleVector(
8582 Cmp, llvm::Constant::getNullValue(Cmp->getType()), Indices);
8592 assert((Ops.size() == 2 || Ops.size() == 4) &&
8593 "Unexpected number of arguments");
8594 unsigned NumElts = Ops[0]->getType()->getVectorNumElements();
8598 Cmp = Constant::getNullValue(
8599 llvm::VectorType::get(CGF.
Builder.getInt1Ty(), NumElts));
8600 }
else if (CC == 7) {
8601 Cmp = Constant::getAllOnesValue(
8602 llvm::VectorType::get(CGF.
Builder.getInt1Ty(), NumElts));
8604 ICmpInst::Predicate Pred;
8606 default: llvm_unreachable(
"Unknown condition code");
8607 case 0: Pred = ICmpInst::ICMP_EQ;
break;
8608 case 1: Pred = Signed ? ICmpInst::ICMP_SLT : ICmpInst::ICMP_ULT;
break;
8609 case 2: Pred = Signed ? ICmpInst::ICMP_SLE : ICmpInst::ICMP_ULE;
break;
8610 case 4: Pred = ICmpInst::ICMP_NE;
break;
8611 case 5: Pred = Signed ? ICmpInst::ICMP_SGE : ICmpInst::ICMP_UGE;
break;
8612 case 6: Pred = Signed ? ICmpInst::ICMP_SGT : ICmpInst::ICMP_UGT;
break;
8614 Cmp = CGF.
Builder.CreateICmp(Pred, Ops[0], Ops[1]);
8617 Value *MaskIn =
nullptr;
8618 if (Ops.size() == 4)
8625 Value *Zero = Constant::getNullValue(In->getType());
8632 Value *Zero = llvm::Constant::getNullValue(Ty);
8634 Value *Cmp = CGF.
Builder.CreateICmp(ICmpInst::ICMP_SGT, Ops[0], Zero);
8635 Value *Res = CGF.
Builder.CreateSelect(Cmp, Ops[0], Sub);
8641 Value *Cmp = CGF.
Builder.CreateICmp(Pred, Ops[0], Ops[1]);
8642 Value *Res = CGF.
Builder.CreateSelect(Cmp, Ops[0], Ops[1]);
8644 assert(Ops.size() == 2);
8650 unsigned BuiltinID,
bool IsAddSub) {
8652 bool Subtract =
false;
8654 switch (BuiltinID) {
8656 case clang::X86::BI__builtin_ia32_vfmsubps512_mask3:
8659 case clang::X86::BI__builtin_ia32_vfmaddps512_mask:
8660 case clang::X86::BI__builtin_ia32_vfmaddps512_maskz:
8661 case clang::X86::BI__builtin_ia32_vfmaddps512_mask3:
8662 IID = llvm::Intrinsic::x86_avx512_vfmadd_ps_512;
break;
8663 case clang::X86::BI__builtin_ia32_vfmsubpd512_mask3:
8666 case clang::X86::BI__builtin_ia32_vfmaddpd512_mask:
8667 case clang::X86::BI__builtin_ia32_vfmaddpd512_maskz:
8668 case clang::X86::BI__builtin_ia32_vfmaddpd512_mask3:
8669 IID = llvm::Intrinsic::x86_avx512_vfmadd_pd_512;
break;
8670 case clang::X86::BI__builtin_ia32_vfmsubaddps512_mask3:
8673 case clang::X86::BI__builtin_ia32_vfmaddsubps512_mask:
8674 case clang::X86::BI__builtin_ia32_vfmaddsubps512_maskz:
8675 case clang::X86::BI__builtin_ia32_vfmaddsubps512_mask3:
8676 IID = llvm::Intrinsic::x86_avx512_vfmaddsub_ps_512;
8678 case clang::X86::BI__builtin_ia32_vfmsubaddpd512_mask3:
8681 case clang::X86::BI__builtin_ia32_vfmaddsubpd512_mask:
8682 case clang::X86::BI__builtin_ia32_vfmaddsubpd512_maskz:
8683 case clang::X86::BI__builtin_ia32_vfmaddsubpd512_mask3:
8684 IID = llvm::Intrinsic::x86_avx512_vfmaddsub_pd_512;
8693 C = CGF.
Builder.CreateFNeg(C);
8698 if (IID != Intrinsic::not_intrinsic &&
8699 cast<llvm::ConstantInt>(Ops.back())->getZExtValue() != (uint64_t)4) {
8701 Res = CGF.
Builder.CreateCall(Intr, {A, B, C, Ops.back() });
8705 Res = CGF.
Builder.CreateCall(FMA, {A, B, C} );
8709 unsigned NumElts = Ty->getVectorNumElements();
8711 for (
unsigned i = 0; i != NumElts; ++i)
8712 Indices[i] = i + (i % 2) * NumElts;
8715 Value *FMSub = CGF.
Builder.CreateCall(FMA, {A, B, NegC} );
8716 Res = CGF.
Builder.CreateShuffleVector(FMSub, Res, Indices);
8721 Value *MaskFalseVal =
nullptr;
8722 switch (BuiltinID) {
8723 case clang::X86::BI__builtin_ia32_vfmaddps512_mask:
8724 case clang::X86::BI__builtin_ia32_vfmaddpd512_mask:
8725 case clang::X86::BI__builtin_ia32_vfmaddsubps512_mask:
8726 case clang::X86::BI__builtin_ia32_vfmaddsubpd512_mask:
8727 MaskFalseVal = Ops[0];
8729 case clang::X86::BI__builtin_ia32_vfmaddps512_maskz:
8730 case clang::X86::BI__builtin_ia32_vfmaddpd512_maskz:
8731 case clang::X86::BI__builtin_ia32_vfmaddsubps512_maskz:
8732 case clang::X86::BI__builtin_ia32_vfmaddsubpd512_maskz:
8733 MaskFalseVal = Constant::getNullValue(Ops[0]->getType());
8735 case clang::X86::BI__builtin_ia32_vfmsubps512_mask3:
8736 case clang::X86::BI__builtin_ia32_vfmaddps512_mask3:
8737 case clang::X86::BI__builtin_ia32_vfmsubpd512_mask3:
8738 case clang::X86::BI__builtin_ia32_vfmaddpd512_mask3:
8739 case clang::X86::BI__builtin_ia32_vfmsubaddps512_mask3:
8740 case clang::X86::BI__builtin_ia32_vfmaddsubps512_mask3:
8741 case clang::X86::BI__builtin_ia32_vfmsubaddpd512_mask3:
8742 case clang::X86::BI__builtin_ia32_vfmaddsubpd512_mask3:
8743 MaskFalseVal = Ops[2];
8755 Value *Upper,
bool ZeroMask =
false,
unsigned PTIdx = 0,
8756 bool NegAcc =
false) {
8759 Rnd = cast<llvm::ConstantInt>(Ops[4])->getZExtValue();
8762 Ops[2] = CGF.
Builder.CreateFNeg(Ops[2]);
8764 Ops[0] = CGF.
Builder.CreateExtractElement(Ops[0], (uint64_t)0);
8765 Ops[1] = CGF.
Builder.CreateExtractElement(Ops[1], (uint64_t)0);
8766 Ops[2] = CGF.
Builder.CreateExtractElement(Ops[2], (uint64_t)0);
8769 Intrinsic::ID IID = Ops[0]->getType()->getPrimitiveSizeInBits() == 32 ?
8770 Intrinsic::x86_avx512_vfmadd_f32 :
8771 Intrinsic::x86_avx512_vfmadd_f64;
8773 {Ops[0], Ops[1], Ops[2], Ops[4]});
8776 Res = CGF.
Builder.CreateCall(FMA, Ops.slice(0, 3));
8779 if (Ops.size() > 3) {
8780 Value *PassThru = ZeroMask ? Constant::getNullValue(Res->getType())
8786 if (NegAcc && PTIdx == 2)
8787 PassThru = CGF.
Builder.CreateExtractElement(Upper, (uint64_t)0);
8791 return CGF.
Builder.CreateInsertElement(Upper, Res, (uint64_t)0);
8798 Ty = llvm::VectorType::get(CGF.
Int64Ty,
8799 Ty->getPrimitiveSizeInBits() / 64);
8805 Constant *ShiftAmt = ConstantInt::get(Ty, 32);
8806 LHS = CGF.
Builder.CreateShl(LHS, ShiftAmt);
8807 LHS = CGF.
Builder.CreateAShr(LHS, ShiftAmt);
8808 RHS = CGF.
Builder.CreateShl(RHS, ShiftAmt);
8809 RHS = CGF.
Builder.CreateAShr(RHS, ShiftAmt);
8812 Constant *Mask = ConstantInt::get(Ty, 0xffffffff);
8813 LHS = CGF.
Builder.CreateAnd(LHS, Mask);
8814 RHS = CGF.
Builder.CreateAnd(RHS, Mask);
8817 return CGF.
Builder.CreateMul(LHS, RHS);
8827 unsigned VecWidth = Ty->getPrimitiveSizeInBits();
8828 unsigned EltWidth = Ty->getScalarSizeInBits();
8830 if (VecWidth == 128 && EltWidth == 32)
8831 IID = Intrinsic::x86_avx512_pternlog_d_128;
8832 else if (VecWidth == 256 && EltWidth == 32)
8833 IID = Intrinsic::x86_avx512_pternlog_d_256;
8834 else if (VecWidth == 512 && EltWidth == 32)
8835 IID = Intrinsic::x86_avx512_pternlog_d_512;
8836 else if (VecWidth == 128 && EltWidth == 64)
8837 IID = Intrinsic::x86_avx512_pternlog_q_128;
8838 else if (VecWidth == 256 && EltWidth == 64)
8839 IID = Intrinsic::x86_avx512_pternlog_q_256;
8840 else if (VecWidth == 512 && EltWidth == 64)
8841 IID = Intrinsic::x86_avx512_pternlog_q_512;
8843 llvm_unreachable(
"Unexpected intrinsic");
8847 Value *PassThru = ZeroMask ? ConstantAggregateZero::get(Ty) : Ops[0];
8853 unsigned NumberOfElements = DstTy->getVectorNumElements();
8855 return CGF.
Builder.CreateSExt(Mask, DstTy,
"vpmovm2");
8860 StringRef CPUStr = cast<clang::StringLiteral>(CPUExpr)->getString();
8861 return EmitX86CpuIs(CPUStr);
8864 Value *CodeGenFunction::EmitX86CpuIs(StringRef CPUStr) {
8874 llvm::Type *STy = llvm::StructType::get(Int32Ty, Int32Ty, Int32Ty,
8875 llvm::ArrayType::get(Int32Ty, 1));
8878 llvm::Constant *CpuModel = CGM.CreateRuntimeVariable(STy,
"__cpu_model");
8884 std::tie(Index, Value) = StringSwitch<std::pair<unsigned, unsigned>>(CPUStr)
8886 .Case(STRING, {0u,
static_cast<unsigned>(llvm::X86::ENUM)})
8887 #define X86_CPU_TYPE_COMPAT_WITH_ALIAS(ARCHNAME, ENUM, STR, ALIAS) \ 8888 .Cases(STR, ALIAS, {1u, static_cast<unsigned>(llvm::X86::ENUM)}) 8889 #define X86_CPU_TYPE_COMPAT(ARCHNAME, ENUM, STR) \ 8890 .Case(STR, {1u, static_cast<unsigned>(llvm::X86::ENUM)}) 8891 #define X86_CPU_SUBTYPE_COMPAT(ARCHNAME, ENUM, STR) \ 8892 .Case(STR, {2u, static_cast<unsigned>(llvm::X86::ENUM)}) 8893 #include "llvm/Support/X86TargetParser.def" 8895 assert(Value != 0 &&
"Invalid CPUStr passed to CpuIs");
8898 llvm::Value *Idxs[] = {ConstantInt::get(Int32Ty, 0),
8899 ConstantInt::get(Int32Ty, Index)};
8900 llvm::Value *CpuValue = Builder.CreateGEP(STy, CpuModel, Idxs);
8904 return Builder.CreateICmpEQ(CpuValue,
8905 llvm::ConstantInt::get(Int32Ty, Value));
8908 Value *CodeGenFunction::EmitX86CpuSupports(
const CallExpr *E) {
8910 StringRef FeatureStr = cast<StringLiteral>(FeatureExpr)->getString();
8911 return EmitX86CpuSupports(FeatureStr);
8917 uint32_t FeaturesMask = 0;
8918 for (
const StringRef &FeatureStr : FeatureStrs) {
8920 StringSwitch<unsigned>(FeatureStr)
8922 #include "llvm/Support/X86TargetParser.def" 8924 FeaturesMask |= (1U << Feature);
8926 return FeaturesMask;
8930 return EmitX86CpuSupports(GetX86CpuSupportsMask(FeatureStrs));
8933 llvm::Value *CodeGenFunction::EmitX86CpuSupports(uint32_t FeaturesMask) {
8940 llvm::Type *STy = llvm::StructType::get(Int32Ty, Int32Ty, Int32Ty,
8941 llvm::ArrayType::get(Int32Ty, 1));
8944 llvm::Constant *CpuModel = CGM.CreateRuntimeVariable(STy,
"__cpu_model");
8948 Value *Idxs[] = {ConstantInt::get(Int32Ty, 0), ConstantInt::get(Int32Ty, 3),
8949 ConstantInt::get(Int32Ty, 0)};
8950 Value *CpuFeatures = Builder.CreateGEP(STy, CpuModel, Idxs);
8955 Value *Bitset = Builder.CreateAnd(
8956 Features, llvm::ConstantInt::get(Int32Ty, FeaturesMask));
8957 return Builder.CreateICmpNE(Bitset, llvm::ConstantInt::get(Int32Ty, 0));
8960 Value *CodeGenFunction::EmitX86CpuInit() {
8961 llvm::FunctionType *FTy = llvm::FunctionType::get(VoidTy,
8963 llvm::Constant *Func = CGM.CreateRuntimeFunction(FTy,
"__cpu_indicator_init");
8964 return Builder.CreateCall(Func);
8969 if (BuiltinID == X86::BI__builtin_cpu_is)
8970 return EmitX86CpuIs(E);
8971 if (BuiltinID == X86::BI__builtin_cpu_supports)
8972 return EmitX86CpuSupports(E);
8973 if (BuiltinID == X86::BI__builtin_cpu_init)
8974 return EmitX86CpuInit();
8979 unsigned ICEArguments = 0;
8981 getContext().GetBuiltinType(BuiltinID, Error, &ICEArguments);
8984 for (
unsigned i = 0, e = E->
getNumArgs(); i != e; i++) {
8986 if ((ICEArguments & (1 << i)) == 0) {
8987 Ops.push_back(EmitScalarExpr(E->
getArg(i)));
8993 llvm::APSInt Result;
8995 assert(IsConst &&
"Constant arg isn't actually constant?"); (void)IsConst;
8996 Ops.push_back(llvm::ConstantInt::get(getLLVMContext(), Result));
9005 auto getCmpIntrinsicCall = [
this, &Ops](
Intrinsic::ID ID,
unsigned Imm) {
9006 Ops.push_back(llvm::ConstantInt::get(Int8Ty, Imm));
9007 llvm::Function *F = CGM.getIntrinsic(ID);
9008 return Builder.CreateCall(F, Ops);
9016 auto getVectorFCmpIR = [
this, &Ops](CmpInst::Predicate Pred) {
9017 Value *Cmp = Builder.CreateFCmp(Pred, Ops[0], Ops[1]);
9018 llvm::VectorType *FPVecTy = cast<llvm::VectorType>(Ops[0]->getType());
9019 llvm::VectorType *IntVecTy = llvm::VectorType::getInteger(FPVecTy);
9020 Value *Sext = Builder.CreateSExt(Cmp, IntVecTy);
9021 return Builder.CreateBitCast(Sext, FPVecTy);
9024 switch (BuiltinID) {
9025 default:
return nullptr;
9026 case X86::BI_mm_prefetch: {
9028 ConstantInt *C = cast<ConstantInt>(Ops[1]);
9029 Value *RW = ConstantInt::get(Int32Ty, (C->getZExtValue() >> 2) & 0x1);
9030 Value *Locality = ConstantInt::get(Int32Ty, C->getZExtValue() & 0x3);
9031 Value *Data = ConstantInt::get(Int32Ty, 1);
9033 return Builder.CreateCall(F, {Address, RW, Locality, Data});
9035 case X86::BI_mm_clflush: {
9036 return Builder.CreateCall(CGM.getIntrinsic(Intrinsic::x86_sse2_clflush),
9039 case X86::BI_mm_lfence: {
9040 return Builder.CreateCall(CGM.getIntrinsic(Intrinsic::x86_sse2_lfence));
9042 case X86::BI_mm_mfence: {
9043 return Builder.CreateCall(CGM.getIntrinsic(Intrinsic::x86_sse2_mfence));
9045 case X86::BI_mm_sfence: {
9046 return Builder.CreateCall(CGM.getIntrinsic(Intrinsic::x86_sse_sfence));
9048 case X86::BI_mm_pause: {
9049 return Builder.CreateCall(CGM.getIntrinsic(Intrinsic::x86_sse2_pause));
9051 case X86::BI__rdtsc: {
9052 return Builder.CreateCall(CGM.getIntrinsic(Intrinsic::x86_rdtsc));
9054 case X86::BI__builtin_ia32_undef128:
9055 case X86::BI__builtin_ia32_undef256:
9056 case X86::BI__builtin_ia32_undef512:
9062 return llvm::Constant::getNullValue(ConvertType(E->
getType()));
9063 case X86::BI__builtin_ia32_vec_init_v8qi:
9064 case X86::BI__builtin_ia32_vec_init_v4hi:
9065 case X86::BI__builtin_ia32_vec_init_v2si:
9066 return Builder.CreateBitCast(BuildVector(Ops),
9067 llvm::Type::getX86_MMXTy(getLLVMContext()));
9068 case X86::BI__builtin_ia32_vec_ext_v2si:
9069 case X86::BI__builtin_ia32_vec_ext_v16qi:
9070 case X86::BI__builtin_ia32_vec_ext_v8hi:
9071 case X86::BI__builtin_ia32_vec_ext_v4si:
9072 case X86::BI__builtin_ia32_vec_ext_v4sf:
9073 case X86::BI__builtin_ia32_vec_ext_v2di:
9074 case X86::BI__builtin_ia32_vec_ext_v32qi:
9075 case X86::BI__builtin_ia32_vec_ext_v16hi:
9076 case X86::BI__builtin_ia32_vec_ext_v8si:
9077 case X86::BI__builtin_ia32_vec_ext_v4di: {
9078 unsigned NumElts = Ops[0]->getType()->getVectorNumElements();
9079 uint64_t Index = cast<ConstantInt>(Ops[1])->getZExtValue();
9080 Index &= NumElts - 1;
9083 return Builder.CreateExtractElement(Ops[0], Index);
9085 case X86::BI__builtin_ia32_vec_set_v16qi:
9086 case X86::BI__builtin_ia32_vec_set_v8hi:
9087 case X86::BI__builtin_ia32_vec_set_v4si:
9088 case X86::BI__builtin_ia32_vec_set_v2di:
9089 case X86::BI__builtin_ia32_vec_set_v32qi:
9090 case X86::BI__builtin_ia32_vec_set_v16hi:
9091 case X86::BI__builtin_ia32_vec_set_v8si:
9092 case X86::BI__builtin_ia32_vec_set_v4di: {
9093 unsigned NumElts = Ops[0]->getType()->getVectorNumElements();
9094 unsigned Index = cast<ConstantInt>(Ops[2])->getZExtValue();
9095 Index &= NumElts - 1;
9098 return Builder.CreateInsertElement(Ops[0], Ops[1], Index);
9100 case X86::BI_mm_setcsr:
9101 case X86::BI__builtin_ia32_ldmxcsr: {
9103 Builder.CreateStore(Ops[0], Tmp);
9104 return Builder.CreateCall(CGM.getIntrinsic(Intrinsic::x86_sse_ldmxcsr),
9105 Builder.CreateBitCast(Tmp.
getPointer(), Int8PtrTy));
9107 case X86::BI_mm_getcsr:
9108 case X86::BI__builtin_ia32_stmxcsr: {
9110 Builder.CreateCall(CGM.getIntrinsic(Intrinsic::x86_sse_stmxcsr),
9111 Builder.CreateBitCast(Tmp.
getPointer(), Int8PtrTy));
9112 return Builder.CreateLoad(Tmp,
"stmxcsr");
9114 case X86::BI__builtin_ia32_xsave:
9115 case X86::BI__builtin_ia32_xsave64:
9116 case X86::BI__builtin_ia32_xrstor:
9117 case X86::BI__builtin_ia32_xrstor64:
9118 case X86::BI__builtin_ia32_xsaveopt:
9119 case X86::BI__builtin_ia32_xsaveopt64:
9120 case X86::BI__builtin_ia32_xrstors:
9121 case X86::BI__builtin_ia32_xrstors64:
9122 case X86::BI__builtin_ia32_xsavec:
9123 case X86::BI__builtin_ia32_xsavec64:
9124 case X86::BI__builtin_ia32_xsaves:
9125 case X86::BI__builtin_ia32_xsaves64: {
9127 #define INTRINSIC_X86_XSAVE_ID(NAME) \ 9128 case X86::BI__builtin_ia32_##NAME: \ 9129 ID = Intrinsic::x86_##NAME; \ 9131 switch (BuiltinID) {
9132 default: llvm_unreachable(
"Unsupported intrinsic!");
9146 #undef INTRINSIC_X86_XSAVE_ID 9147 Value *Mhi = Builder.CreateTrunc(
9148 Builder.CreateLShr(Ops[1], ConstantInt::get(Int64Ty, 32)), Int32Ty);
9149 Value *Mlo = Builder.CreateTrunc(Ops[1], Int32Ty);
9152 return Builder.CreateCall(CGM.getIntrinsic(ID), Ops);
9154 case X86::BI__builtin_ia32_storedqudi128_mask:
9155 case X86::BI__builtin_ia32_storedqusi128_mask:
9156 case X86::BI__builtin_ia32_storedquhi128_mask:
9157 case X86::BI__builtin_ia32_storedquqi128_mask:
9158 case X86::BI__builtin_ia32_storeupd128_mask:
9159 case X86::BI__builtin_ia32_storeups128_mask:
9160 case X86::BI__builtin_ia32_storedqudi256_mask:
9161 case X86::BI__builtin_ia32_storedqusi256_mask:
9162 case X86::BI__builtin_ia32_storedquhi256_mask:
9163 case X86::BI__builtin_ia32_storedquqi256_mask:
9164 case X86::BI__builtin_ia32_storeupd256_mask:
9165 case X86::BI__builtin_ia32_storeups256_mask:
9166 case X86::BI__builtin_ia32_storedqudi512_mask:
9167 case X86::BI__builtin_ia32_storedqusi512_mask:
9168 case X86::BI__builtin_ia32_storedquhi512_mask:
9169 case X86::BI__builtin_ia32_storedquqi512_mask:
9170 case X86::BI__builtin_ia32_storeupd512_mask:
9171 case X86::BI__builtin_ia32_storeups512_mask:
9174 case X86::BI__builtin_ia32_storess128_mask:
9175 case X86::BI__builtin_ia32_storesd128_mask: {
9178 case X86::BI__builtin_ia32_vpopcntb_128:
9179 case X86::BI__builtin_ia32_vpopcntd_128:
9180 case X86::BI__builtin_ia32_vpopcntq_128:
9181 case X86::BI__builtin_ia32_vpopcntw_128:
9182 case X86::BI__builtin_ia32_vpopcntb_256:
9183 case X86::BI__builtin_ia32_vpopcntd_256:
9184 case X86::BI__builtin_ia32_vpopcntq_256:
9185 case X86::BI__builtin_ia32_vpopcntw_256:
9186 case X86::BI__builtin_ia32_vpopcntb_512:
9187 case X86::BI__builtin_ia32_vpopcntd_512:
9188 case X86::BI__builtin_ia32_vpopcntq_512:
9189 case X86::BI__builtin_ia32_vpopcntw_512: {
9191 llvm::Function *F = CGM.getIntrinsic(Intrinsic::ctpop, ResultType);
9192 return Builder.CreateCall(F, Ops);
9194 case X86::BI__builtin_ia32_cvtmask2b128:
9195 case X86::BI__builtin_ia32_cvtmask2b256:
9196 case X86::BI__builtin_ia32_cvtmask2b512:
9197 case X86::BI__builtin_ia32_cvtmask2w128:
9198 case X86::BI__builtin_ia32_cvtmask2w256:
9199 case X86::BI__builtin_ia32_cvtmask2w512:
9200 case X86::BI__builtin_ia32_cvtmask2d128:
9201 case X86::BI__builtin_ia32_cvtmask2d256:
9202 case X86::BI__builtin_ia32_cvtmask2d512:
9203 case X86::BI__builtin_ia32_cvtmask2q128:
9204 case X86::BI__builtin_ia32_cvtmask2q256:
9205 case X86::BI__builtin_ia32_cvtmask2q512:
9208 case X86::BI__builtin_ia32_cvtb2mask128:
9209 case X86::BI__builtin_ia32_cvtb2mask256:
9210 case X86::BI__builtin_ia32_cvtb2mask512:
9211 case X86::BI__builtin_ia32_cvtw2mask128:
9212 case X86::BI__builtin_ia32_cvtw2mask256:
9213 case X86::BI__builtin_ia32_cvtw2mask512:
9214 case X86::BI__builtin_ia32_cvtd2mask128:
9215 case X86::BI__builtin_ia32_cvtd2mask256:
9216 case X86::BI__builtin_ia32_cvtd2mask512:
9217 case X86::BI__builtin_ia32_cvtq2mask128:
9218 case X86::BI__builtin_ia32_cvtq2mask256:
9219 case X86::BI__builtin_ia32_cvtq2mask512:
9222 case X86::BI__builtin_ia32_vfmaddss3:
9223 case X86::BI__builtin_ia32_vfmaddsd3:
9224 case X86::BI__builtin_ia32_vfmaddss3_mask:
9225 case X86::BI__builtin_ia32_vfmaddsd3_mask:
9227 case X86::BI__builtin_ia32_vfmaddss:
9228 case X86::BI__builtin_ia32_vfmaddsd:
9230 Constant::getNullValue(Ops[0]->getType()));
9231 case X86::BI__builtin_ia32_vfmaddss3_maskz:
9232 case X86::BI__builtin_ia32_vfmaddsd3_maskz:
9234 case X86::BI__builtin_ia32_vfmaddss3_mask3:
9235 case X86::BI__builtin_ia32_vfmaddsd3_mask3:
9237 case X86::BI__builtin_ia32_vfmsubss3_mask3:
9238 case X86::BI__builtin_ia32_vfmsubsd3_mask3:
9241 case X86::BI__builtin_ia32_vfmaddps:
9242 case X86::BI__builtin_ia32_vfmaddpd:
9243 case X86::BI__builtin_ia32_vfmaddps256:
9244 case X86::BI__builtin_ia32_vfmaddpd256:
9245 case X86::BI__builtin_ia32_vfmaddps512_mask:
9246 case X86::BI__builtin_ia32_vfmaddps512_maskz:
9247 case X86::BI__builtin_ia32_vfmaddps512_mask3:
9248 case X86::BI__builtin_ia32_vfmsubps512_mask3:
9249 case X86::BI__builtin_ia32_vfmaddpd512_mask:
9250 case X86::BI__builtin_ia32_vfmaddpd512_maskz:
9251 case X86::BI__builtin_ia32_vfmaddpd512_mask3:
9252 case X86::BI__builtin_ia32_vfmsubpd512_mask3:
9254 case X86::BI__builtin_ia32_vfmaddsubps:
9255 case X86::BI__builtin_ia32_vfmaddsubpd:
9256 case X86::BI__builtin_ia32_vfmaddsubps256:
9257 case X86::BI__builtin_ia32_vfmaddsubpd256:
9258 case X86::BI__builtin_ia32_vfmaddsubps512_mask:
9259 case X86::BI__builtin_ia32_vfmaddsubps512_maskz:
9260 case X86::BI__builtin_ia32_vfmaddsubps512_mask3:
9261 case X86::BI__builtin_ia32_vfmsubaddps512_mask3:
9262 case X86::BI__builtin_ia32_vfmaddsubpd512_mask:
9263 case X86::BI__builtin_ia32_vfmaddsubpd512_maskz:
9264 case X86::BI__builtin_ia32_vfmaddsubpd512_mask3:
9265 case X86::BI__builtin_ia32_vfmsubaddpd512_mask3:
9268 case X86::BI__builtin_ia32_movdqa32store128_mask:
9269 case X86::BI__builtin_ia32_movdqa64store128_mask:
9270 case X86::BI__builtin_ia32_storeaps128_mask:
9271 case X86::BI__builtin_ia32_storeapd128_mask:
9272 case X86::BI__builtin_ia32_movdqa32store256_mask:
9273 case X86::BI__builtin_ia32_movdqa64store256_mask:
9274 case X86::BI__builtin_ia32_storeaps256_mask:
9275 case X86::BI__builtin_ia32_storeapd256_mask:
9276 case X86::BI__builtin_ia32_movdqa32store512_mask:
9277 case X86::BI__builtin_ia32_movdqa64store512_mask:
9278 case X86::BI__builtin_ia32_storeaps512_mask:
9279 case X86::BI__builtin_ia32_storeapd512_mask: {
9281 getContext().getTypeAlignInChars(E->
getArg(1)->
getType()).getQuantity();
9284 case X86::BI__builtin_ia32_loadups128_mask:
9285 case X86::BI__builtin_ia32_loadups256_mask:
9286 case X86::BI__builtin_ia32_loadups512_mask:
9287 case X86::BI__builtin_ia32_loadupd128_mask:
9288 case X86::BI__builtin_ia32_loadupd256_mask:
9289 case X86::BI__builtin_ia32_loadupd512_mask:
9290 case X86::BI__builtin_ia32_loaddquqi128_mask:
9291 case X86::BI__builtin_ia32_loaddquqi256_mask:
9292 case X86::BI__builtin_ia32_loaddquqi512_mask:
9293 case X86::BI__builtin_ia32_loaddquhi128_mask:
9294 case X86::BI__builtin_ia32_loaddquhi256_mask:
9295 case X86::BI__builtin_ia32_loaddquhi512_mask:
9296 case X86::BI__builtin_ia32_loaddqusi128_mask:
9297 case X86::BI__builtin_ia32_loaddqusi256_mask:
9298 case X86::BI__builtin_ia32_loaddqusi512_mask:
9299 case X86::BI__builtin_ia32_loaddqudi128_mask:
9300 case X86::BI__builtin_ia32_loaddqudi256_mask:
9301 case X86::BI__builtin_ia32_loaddqudi512_mask:
9304 case X86::BI__builtin_ia32_loadss128_mask:
9305 case X86::BI__builtin_ia32_loadsd128_mask:
9308 case X86::BI__builtin_ia32_loadaps128_mask:
9309 case X86::BI__builtin_ia32_loadaps256_mask:
9310 case X86::BI__builtin_ia32_loadaps512_mask:
9311 case X86::BI__builtin_ia32_loadapd128_mask:
9312 case X86::BI__builtin_ia32_loadapd256_mask:
9313 case X86::BI__builtin_ia32_loadapd512_mask:
9314 case X86::BI__builtin_ia32_movdqa32load128_mask:
9315 case X86::BI__builtin_ia32_movdqa32load256_mask:
9316 case X86::BI__builtin_ia32_movdqa32load512_mask:
9317 case X86::BI__builtin_ia32_movdqa64load128_mask:
9318 case X86::BI__builtin_ia32_movdqa64load256_mask:
9319 case X86::BI__builtin_ia32_movdqa64load512_mask: {
9321 getContext().getTypeAlignInChars(E->
getArg(1)->
getType()).getQuantity();
9325 case X86::BI__builtin_ia32_expandloaddf128_mask:
9326 case X86::BI__builtin_ia32_expandloaddf256_mask:
9327 case X86::BI__builtin_ia32_expandloaddf512_mask:
9328 case X86::BI__builtin_ia32_expandloadsf128_mask:
9329 case X86::BI__builtin_ia32_expandloadsf256_mask:
9330 case X86::BI__builtin_ia32_expandloadsf512_mask:
9331 case X86::BI__builtin_ia32_expandloaddi128_mask:
9332 case X86::BI__builtin_ia32_expandloaddi256_mask:
9333 case X86::BI__builtin_ia32_expandloaddi512_mask:
9334 case X86::BI__builtin_ia32_expandloadsi128_mask:
9335 case X86::BI__builtin_ia32_expandloadsi256_mask:
9336 case X86::BI__builtin_ia32_expandloadsi512_mask:
9337 case X86::BI__builtin_ia32_expandloadhi128_mask:
9338 case X86::BI__builtin_ia32_expandloadhi256_mask:
9339 case X86::BI__builtin_ia32_expandloadhi512_mask:
9340 case X86::BI__builtin_ia32_expandloadqi128_mask:
9341 case X86::BI__builtin_ia32_expandloadqi256_mask:
9342 case X86::BI__builtin_ia32_expandloadqi512_mask:
9345 case X86::BI__builtin_ia32_compressstoredf128_mask:
9346 case X86::BI__builtin_ia32_compressstoredf256_mask:
9347 case X86::BI__builtin_ia32_compressstoredf512_mask:
9348 case X86::BI__builtin_ia32_compressstoresf128_mask:
9349 case X86::BI__builtin_ia32_compressstoresf256_mask:
9350 case X86::BI__builtin_ia32_compressstoresf512_mask:
9351 case X86::BI__builtin_ia32_compressstoredi128_mask:
9352 case X86::BI__builtin_ia32_compressstoredi256_mask:
9353 case X86::BI__builtin_ia32_compressstoredi512_mask:
9354 case X86::BI__builtin_ia32_compressstoresi128_mask:
9355 case X86::BI__builtin_ia32_compressstoresi256_mask:
9356 case X86::BI__builtin_ia32_compressstoresi512_mask:
9357 case X86::BI__builtin_ia32_compressstorehi128_mask:
9358 case X86::BI__builtin_ia32_compressstorehi256_mask:
9359 case X86::BI__builtin_ia32_compressstorehi512_mask:
9360 case X86::BI__builtin_ia32_compressstoreqi128_mask:
9361 case X86::BI__builtin_ia32_compressstoreqi256_mask:
9362 case X86::BI__builtin_ia32_compressstoreqi512_mask:
9365 case X86::BI__builtin_ia32_storehps:
9366 case X86::BI__builtin_ia32_storelps: {
9367 llvm::Type *PtrTy = llvm::PointerType::getUnqual(Int64Ty);
9368 llvm::Type *VecTy = llvm::VectorType::get(Int64Ty, 2);
9371 Ops[1] = Builder.CreateBitCast(Ops[1], VecTy,
"cast");
9374 unsigned Index = BuiltinID == X86::BI__builtin_ia32_storelps ? 0 : 1;
9375 Ops[1] = Builder.CreateExtractElement(Ops[1], Index,
"extract");
9378 Ops[0] = Builder.CreateBitCast(Ops[0], PtrTy);
9379 return Builder.CreateDefaultAlignedStore(Ops[1], Ops[0]);
9381 case X86::BI__builtin_ia32_vextractf128_pd256:
9382 case X86::BI__builtin_ia32_vextractf128_ps256:
9383 case X86::BI__builtin_ia32_vextractf128_si256:
9384 case X86::BI__builtin_ia32_extract128i256:
9385 case X86::BI__builtin_ia32_extractf64x4_mask:
9386 case X86::BI__builtin_ia32_extractf32x4_mask:
9387 case X86::BI__builtin_ia32_extracti64x4_mask:
9388 case X86::BI__builtin_ia32_extracti32x4_mask:
9389 case X86::BI__builtin_ia32_extractf32x8_mask:
9390 case X86::BI__builtin_ia32_extracti32x8_mask:
9391 case X86::BI__builtin_ia32_extractf32x4_256_mask:
9392 case X86::BI__builtin_ia32_extracti32x4_256_mask:
9393 case X86::BI__builtin_ia32_extractf64x2_256_mask:
9394 case X86::BI__builtin_ia32_extracti64x2_256_mask:
9395 case X86::BI__builtin_ia32_extractf64x2_512_mask:
9396 case X86::BI__builtin_ia32_extracti64x2_512_mask: {
9398 unsigned NumElts = DstTy->getVectorNumElements();
9399 unsigned SrcNumElts = Ops[0]->getType()->getVectorNumElements();
9400 unsigned SubVectors = SrcNumElts / NumElts;
9401 unsigned Index = cast<ConstantInt>(Ops[1])->getZExtValue();
9402 assert(llvm::isPowerOf2_32(SubVectors) &&
"Expected power of 2 subvectors");
9403 Index &= SubVectors - 1;
9406 uint32_t Indices[16];
9407 for (
unsigned i = 0; i != NumElts; ++i)
9408 Indices[i] = i + Index;
9410 Value *Res = Builder.CreateShuffleVector(Ops[0],
9411 UndefValue::get(Ops[0]->getType()),
9412 makeArrayRef(Indices, NumElts),
9415 if (Ops.size() == 4)
9420 case X86::BI__builtin_ia32_vinsertf128_pd256:
9421 case X86::BI__builtin_ia32_vinsertf128_ps256:
9422 case X86::BI__builtin_ia32_vinsertf128_si256:
9423 case X86::BI__builtin_ia32_insert128i256:
9424 case X86::BI__builtin_ia32_insertf64x4:
9425 case X86::BI__builtin_ia32_insertf32x4:
9426 case X86::BI__builtin_ia32_inserti64x4:
9427 case X86::BI__builtin_ia32_inserti32x4:
9428 case X86::BI__builtin_ia32_insertf32x8:
9429 case X86::BI__builtin_ia32_inserti32x8:
9430 case X86::BI__builtin_ia32_insertf32x4_256:
9431 case X86::BI__builtin_ia32_inserti32x4_256:
9432 case X86::BI__builtin_ia32_insertf64x2_256:
9433 case X86::BI__builtin_ia32_inserti64x2_256:
9434 case X86::BI__builtin_ia32_insertf64x2_512:
9435 case X86::BI__builtin_ia32_inserti64x2_512: {
9436 unsigned DstNumElts = Ops[0]->getType()->getVectorNumElements();
9437 unsigned SrcNumElts = Ops[1]->getType()->getVectorNumElements();
9438 unsigned SubVectors = DstNumElts / SrcNumElts;
9439 unsigned Index = cast<ConstantInt>(Ops[2])->getZExtValue();
9440 assert(llvm::isPowerOf2_32(SubVectors) &&
"Expected power of 2 subvectors");
9441 Index &= SubVectors - 1;
9442 Index *= SrcNumElts;
9444 uint32_t Indices[16];
9445 for (
unsigned i = 0; i != DstNumElts; ++i)
9446 Indices[i] = (i >= SrcNumElts) ? SrcNumElts + (i % SrcNumElts) : i;
9448 Value *Op1 = Builder.CreateShuffleVector(Ops[1],
9449 UndefValue::get(Ops[1]->getType()),
9450 makeArrayRef(Indices, DstNumElts),
9453 for (
unsigned i = 0; i != DstNumElts; ++i) {
9454 if (i >= Index && i < (Index + SrcNumElts))
9455 Indices[i] = (i - Index) + DstNumElts;
9460 return Builder.CreateShuffleVector(Ops[0], Op1,
9461 makeArrayRef(Indices, DstNumElts),
9464 case X86::BI__builtin_ia32_pmovqd512_mask:
9465 case X86::BI__builtin_ia32_pmovwb512_mask: {
9466 Value *Res = Builder.CreateTrunc(Ops[0], Ops[1]->getType());
9469 case X86::BI__builtin_ia32_pmovdb512_mask:
9470 case X86::BI__builtin_ia32_pmovdw512_mask:
9471 case X86::BI__builtin_ia32_pmovqw512_mask: {
9472 if (
const auto *C = dyn_cast<Constant>(Ops[2]))
9473 if (C->isAllOnesValue())
9474 return Builder.CreateTrunc(Ops[0], Ops[1]->getType());
9477 switch (BuiltinID) {
9478 default: llvm_unreachable(
"Unsupported intrinsic!");
9479 case X86::BI__builtin_ia32_pmovdb512_mask:
9480 IID = Intrinsic::x86_avx512_mask_pmov_db_512;
9482 case X86::BI__builtin_ia32_pmovdw512_mask:
9483 IID = Intrinsic::x86_avx512_mask_pmov_dw_512;
9485 case X86::BI__builtin_ia32_pmovqw512_mask:
9486 IID = Intrinsic::x86_avx512_mask_pmov_qw_512;
9490 Function *Intr = CGM.getIntrinsic(IID);
9491 return Builder.CreateCall(Intr, Ops);
9493 case X86::BI__builtin_ia32_pblendw128:
9494 case X86::BI__builtin_ia32_blendpd:
9495 case X86::BI__builtin_ia32_blendps:
9496 case X86::BI__builtin_ia32_blendpd256:
9497 case X86::BI__builtin_ia32_blendps256:
9498 case X86::BI__builtin_ia32_pblendw256:
9499 case X86::BI__builtin_ia32_pblendd128:
9500 case X86::BI__builtin_ia32_pblendd256: {
9501 unsigned NumElts = Ops[0]->getType()->getVectorNumElements();
9502 unsigned Imm = cast<llvm::ConstantInt>(Ops[2])->getZExtValue();
9504 uint32_t Indices[16];
9507 for (
unsigned i = 0; i != NumElts; ++i)
9508 Indices[i] = ((Imm >> (i % 8)) & 0x1) ? NumElts + i : i;
9510 return Builder.CreateShuffleVector(Ops[0], Ops[1],
9511 makeArrayRef(Indices, NumElts),
9514 case X86::BI__builtin_ia32_pshuflw:
9515 case X86::BI__builtin_ia32_pshuflw256:
9516 case X86::BI__builtin_ia32_pshuflw512: {
9517 uint32_t Imm = cast<llvm::ConstantInt>(Ops[1])->getZExtValue();
9519 unsigned NumElts = Ty->getVectorNumElements();
9522 Imm = (Imm & 0xff) * 0x01010101;
9524 uint32_t Indices[32];
9525 for (
unsigned l = 0; l != NumElts; l += 8) {
9526 for (
unsigned i = 0; i != 4; ++i) {
9527 Indices[l + i] = l + (Imm & 3);
9530 for (
unsigned i = 4; i != 8; ++i)
9531 Indices[l + i] = l + i;
9534 return Builder.CreateShuffleVector(Ops[0], UndefValue::get(Ty),
9535 makeArrayRef(Indices, NumElts),
9538 case X86::BI__builtin_ia32_pshufhw:
9539 case X86::BI__builtin_ia32_pshufhw256:
9540 case X86::BI__builtin_ia32_pshufhw512: {
9541 uint32_t Imm = cast<llvm::ConstantInt>(Ops[1])->getZExtValue();
9543 unsigned NumElts = Ty->getVectorNumElements();
9546 Imm = (Imm & 0xff) * 0x01010101;
9548 uint32_t Indices[32];
9549 for (
unsigned l = 0; l != NumElts; l += 8) {
9550 for (
unsigned i = 0; i != 4; ++i)
9551 Indices[l + i] = l + i;
9552 for (
unsigned i = 4; i != 8; ++i) {
9553 Indices[l + i] = l + 4 + (Imm & 3);
9558 return Builder.CreateShuffleVector(Ops[0], UndefValue::get(Ty),
9559 makeArrayRef(Indices, NumElts),
9562 case X86::BI__builtin_ia32_pshufd:
9563 case X86::BI__builtin_ia32_pshufd256:
9564 case X86::BI__builtin_ia32_pshufd512:
9565 case X86::BI__builtin_ia32_vpermilpd:
9566 case X86::BI__builtin_ia32_vpermilps:
9567 case X86::BI__builtin_ia32_vpermilpd256:
9568 case X86::BI__builtin_ia32_vpermilps256:
9569 case X86::BI__builtin_ia32_vpermilpd512:
9570 case X86::BI__builtin_ia32_vpermilps512: {
9571 uint32_t Imm = cast<llvm::ConstantInt>(Ops[1])->getZExtValue();
9573 unsigned NumElts = Ty->getVectorNumElements();
9574 unsigned NumLanes = Ty->getPrimitiveSizeInBits() / 128;
9575 unsigned NumLaneElts = NumElts / NumLanes;
9578 Imm = (Imm & 0xff) * 0x01010101;
9580 uint32_t Indices[16];
9581 for (
unsigned l = 0; l != NumElts; l += NumLaneElts) {
9582 for (
unsigned i = 0; i != NumLaneElts; ++i) {
9583 Indices[i + l] = (Imm % NumLaneElts) + l;
9588 return Builder.CreateShuffleVector(Ops[0], UndefValue::get(Ty),
9589 makeArrayRef(Indices, NumElts),
9592 case X86::BI__builtin_ia32_shufpd:
9593 case X86::BI__builtin_ia32_shufpd256:
9594 case X86::BI__builtin_ia32_shufpd512:
9595 case X86::BI__builtin_ia32_shufps:
9596 case X86::BI__builtin_ia32_shufps256:
9597 case X86::BI__builtin_ia32_shufps512: {
9598 uint32_t Imm = cast<llvm::ConstantInt>(Ops[2])->getZExtValue();
9600 unsigned NumElts = Ty->getVectorNumElements();
9601 unsigned NumLanes = Ty->getPrimitiveSizeInBits() / 128;
9602 unsigned NumLaneElts = NumElts / NumLanes;
9605 Imm = (Imm & 0xff) * 0x01010101;
9607 uint32_t Indices[16];
9608 for (
unsigned l = 0; l != NumElts; l += NumLaneElts) {
9609 for (
unsigned i = 0; i != NumLaneElts; ++i) {
9610 unsigned Index = Imm % NumLaneElts;
9612 if (i >= (NumLaneElts / 2))
9614 Indices[l + i] = l + Index;
9618 return Builder.CreateShuffleVector(Ops[0], Ops[1],
9619 makeArrayRef(Indices, NumElts),
9622 case X86::BI__builtin_ia32_permdi256:
9623 case X86::BI__builtin_ia32_permdf256:
9624 case X86::BI__builtin_ia32_permdi512:
9625 case X86::BI__builtin_ia32_permdf512: {
9626 unsigned Imm = cast<llvm::ConstantInt>(Ops[1])->getZExtValue();
9628 unsigned NumElts = Ty->getVectorNumElements();
9631 uint32_t Indices[8];
9632 for (
unsigned l = 0; l != NumElts; l += 4)
9633 for (
unsigned i = 0; i != 4; ++i)
9634 Indices[l + i] = l + ((Imm >> (2 * i)) & 0x3);
9636 return Builder.CreateShuffleVector(Ops[0], UndefValue::get(Ty),
9637 makeArrayRef(Indices, NumElts),
9640 case X86::BI__builtin_ia32_palignr128:
9641 case X86::BI__builtin_ia32_palignr256:
9642 case X86::BI__builtin_ia32_palignr512: {
9643 unsigned ShiftVal = cast<llvm::ConstantInt>(Ops[2])->getZExtValue() & 0xff;
9645 unsigned NumElts = Ops[0]->getType()->getVectorNumElements();
9646 assert(NumElts % 16 == 0);
9651 return llvm::Constant::getNullValue(ConvertType(E->
getType()));
9655 if (ShiftVal > 16) {
9658 Ops[0] = llvm::Constant::getNullValue(Ops[0]->getType());
9661 uint32_t Indices[64];
9663 for (
unsigned l = 0; l != NumElts; l += 16) {
9664 for (
unsigned i = 0; i != 16; ++i) {
9665 unsigned Idx = ShiftVal + i;
9667 Idx += NumElts - 16;
9668 Indices[l + i] = Idx + l;
9672 return Builder.CreateShuffleVector(Ops[1], Ops[0],
9673 makeArrayRef(Indices, NumElts),
9676 case X86::BI__builtin_ia32_alignd128:
9677 case X86::BI__builtin_ia32_alignd256:
9678 case X86::BI__builtin_ia32_alignd512:
9679 case X86::BI__builtin_ia32_alignq128:
9680 case X86::BI__builtin_ia32_alignq256:
9681 case X86::BI__builtin_ia32_alignq512: {
9682 unsigned NumElts = Ops[0]->getType()->getVectorNumElements();
9683 unsigned ShiftVal = cast<llvm::ConstantInt>(Ops[2])->getZExtValue() & 0xff;
9686 ShiftVal &= (2 * NumElts) - 1;
9688 uint32_t Indices[16];
9689 for (
unsigned i = 0; i != NumElts; ++i)
9690 Indices[i] = i + ShiftVal;
9692 return Builder.CreateShuffleVector(Ops[1], Ops[0],
9693 makeArrayRef(Indices, NumElts),
9696 case X86::BI__builtin_ia32_shuf_f32x4_256:
9697 case X86::BI__builtin_ia32_shuf_f64x2_256:
9698 case X86::BI__builtin_ia32_shuf_i32x4_256:
9699 case X86::BI__builtin_ia32_shuf_i64x2_256:
9700 case X86::BI__builtin_ia32_shuf_f32x4:
9701 case X86::BI__builtin_ia32_shuf_f64x2:
9702 case X86::BI__builtin_ia32_shuf_i32x4:
9703 case X86::BI__builtin_ia32_shuf_i64x2: {
9704 unsigned Imm = cast<llvm::ConstantInt>(Ops[2])->getZExtValue();
9706 unsigned NumElts = Ty->getVectorNumElements();
9707 unsigned NumLanes = Ty->getPrimitiveSizeInBits() == 512 ? 4 : 2;
9708 unsigned NumLaneElts = NumElts / NumLanes;
9710 uint32_t Indices[16];
9711 for (
unsigned l = 0; l != NumElts; l += NumLaneElts) {
9712 unsigned Index = (Imm % NumLanes) * NumLaneElts;
9714 if (l >= (NumElts / 2))
9716 for (
unsigned i = 0; i != NumLaneElts; ++i) {
9717 Indices[l + i] = Index + i;
9721 return Builder.CreateShuffleVector(Ops[0], Ops[1],
9722 makeArrayRef(Indices, NumElts),
9726 case X86::BI__builtin_ia32_vperm2f128_pd256:
9727 case X86::BI__builtin_ia32_vperm2f128_ps256:
9728 case X86::BI__builtin_ia32_vperm2f128_si256:
9729 case X86::BI__builtin_ia32_permti256: {
9730 unsigned Imm = cast<llvm::ConstantInt>(Ops[2])->getZExtValue();
9731 unsigned NumElts = Ops[0]->getType()->getVectorNumElements();
9739 uint32_t Indices[8];
9740 for (
unsigned l = 0; l != 2; ++l) {
9742 if (Imm & (1 << ((l * 4) + 3)))
9743 OutOps[l] = llvm::ConstantAggregateZero::get(Ops[0]->getType());
9744 else if (Imm & (1 << ((l * 4) + 1)))
9749 for (
unsigned i = 0; i != NumElts/2; ++i) {
9751 unsigned Idx = (l * NumElts) + i;
9754 if (Imm & (1 << (l * 4)))
9756 Indices[(l * (NumElts/2)) + i] = Idx;
9760 return Builder.CreateShuffleVector(OutOps[0], OutOps[1],
9761 makeArrayRef(Indices, NumElts),
9765 case X86::BI__builtin_ia32_pslldqi128_byteshift:
9766 case X86::BI__builtin_ia32_pslldqi256_byteshift:
9767 case X86::BI__builtin_ia32_pslldqi512_byteshift: {
9768 unsigned ShiftVal = cast<llvm::ConstantInt>(Ops[1])->getZExtValue() & 0xff;
9771 unsigned NumElts = ResultType->getVectorNumElements() * 8;
9775 return llvm::Constant::getNullValue(ResultType);
9777 uint32_t Indices[64];
9779 for (
unsigned l = 0; l != NumElts; l += 16) {
9780 for (
unsigned i = 0; i != 16; ++i) {
9781 unsigned Idx = NumElts + i - ShiftVal;
9782 if (Idx < NumElts) Idx -= NumElts - 16;
9783 Indices[l + i] = Idx + l;
9787 llvm::Type *VecTy = llvm::VectorType::get(Int8Ty, NumElts);
9788 Value *Cast = Builder.CreateBitCast(Ops[0], VecTy,
"cast");
9789 Value *Zero = llvm::Constant::getNullValue(VecTy);
9790 Value *SV = Builder.CreateShuffleVector(Zero, Cast,
9791 makeArrayRef(Indices, NumElts),
9793 return Builder.CreateBitCast(SV, Ops[0]->getType(),
"cast");
9795 case X86::BI__builtin_ia32_psrldqi128_byteshift:
9796 case X86::BI__builtin_ia32_psrldqi256_byteshift:
9797 case X86::BI__builtin_ia32_psrldqi512_byteshift: {
9798 unsigned ShiftVal = cast<llvm::ConstantInt>(Ops[1])->getZExtValue() & 0xff;
9801 unsigned NumElts = ResultType->getVectorNumElements() * 8;
9805 return llvm::Constant::getNullValue(ResultType);
9807 uint32_t Indices[64];
9809 for (
unsigned l = 0; l != NumElts; l += 16) {
9810 for (
unsigned i = 0; i != 16; ++i) {
9811 unsigned Idx = i + ShiftVal;
9812 if (Idx >= 16) Idx += NumElts - 16;
9813 Indices[l + i] = Idx + l;
9817 llvm::Type *VecTy = llvm::VectorType::get(Int8Ty, NumElts);
9818 Value *Cast = Builder.CreateBitCast(Ops[0], VecTy,
"cast");
9819 Value *Zero = llvm::Constant::getNullValue(VecTy);
9820 Value *SV = Builder.CreateShuffleVector(Cast, Zero,
9821 makeArrayRef(Indices, NumElts),
9823 return Builder.CreateBitCast(SV, ResultType,
"cast");
9825 case X86::BI__builtin_ia32_movnti:
9826 case X86::BI__builtin_ia32_movnti64:
9827 case X86::BI__builtin_ia32_movntsd:
9828 case X86::BI__builtin_ia32_movntss: {
9829 llvm::MDNode *
Node = llvm::MDNode::get(
9830 getLLVMContext(), llvm::ConstantAsMetadata::get(Builder.getInt32(1)));
9832 Value *Ptr = Ops[0];
9833 Value *Src = Ops[1];
9836 if (BuiltinID == X86::BI__builtin_ia32_movntsd ||
9837 BuiltinID == X86::BI__builtin_ia32_movntss)
9838 Src = Builder.CreateExtractElement(Src, (uint64_t)0,
"extract");
9841 Value *BC = Builder.CreateBitCast(
9842 Ptr, llvm::PointerType::getUnqual(Src->getType()),
"cast");
9845 StoreInst *SI = Builder.CreateDefaultAlignedStore(Src, BC);
9846 SI->setMetadata(CGM.getModule().getMDKindID(
"nontemporal"),
Node);
9847 SI->setAlignment(1);
9851 case X86::BI__builtin_ia32_selectb_128:
9852 case X86::BI__builtin_ia32_selectb_256:
9853 case X86::BI__builtin_ia32_selectb_512:
9854 case X86::BI__builtin_ia32_selectw_128:
9855 case X86::BI__builtin_ia32_selectw_256:
9856 case X86::BI__builtin_ia32_selectw_512:
9857 case X86::BI__builtin_ia32_selectd_128:
9858 case X86::BI__builtin_ia32_selectd_256:
9859 case X86::BI__builtin_ia32_selectd_512:
9860 case X86::BI__builtin_ia32_selectq_128:
9861 case X86::BI__builtin_ia32_selectq_256:
9862 case X86::BI__builtin_ia32_selectq_512:
9863 case X86::BI__builtin_ia32_selectps_128:
9864 case X86::BI__builtin_ia32_selectps_256:
9865 case X86::BI__builtin_ia32_selectps_512:
9866 case X86::BI__builtin_ia32_selectpd_128:
9867 case X86::BI__builtin_ia32_selectpd_256:
9868 case X86::BI__builtin_ia32_selectpd_512:
9870 case X86::BI__builtin_ia32_selectss_128:
9871 case X86::BI__builtin_ia32_selectsd_128: {
9872 Value *A = Builder.CreateExtractElement(Ops[1], (uint64_t)0);
9873 Value *B = Builder.CreateExtractElement(Ops[2], (uint64_t)0);
9875 return Builder.CreateInsertElement(Ops[1], A, (uint64_t)0);
9877 case X86::BI__builtin_ia32_cmpb128_mask:
9878 case X86::BI__builtin_ia32_cmpb256_mask:
9879 case X86::BI__builtin_ia32_cmpb512_mask:
9880 case X86::BI__builtin_ia32_cmpw128_mask:
9881 case X86::BI__builtin_ia32_cmpw256_mask:
9882 case X86::BI__builtin_ia32_cmpw512_mask:
9883 case X86::BI__builtin_ia32_cmpd128_mask:
9884 case X86::BI__builtin_ia32_cmpd256_mask:
9885 case X86::BI__builtin_ia32_cmpd512_mask:
9886 case X86::BI__builtin_ia32_cmpq128_mask:
9887 case X86::BI__builtin_ia32_cmpq256_mask:
9888 case X86::BI__builtin_ia32_cmpq512_mask: {
9889 unsigned CC = cast<llvm::ConstantInt>(Ops[2])->getZExtValue() & 0x7;
9892 case X86::BI__builtin_ia32_ucmpb128_mask:
9893 case X86::BI__builtin_ia32_ucmpb256_mask:
9894 case X86::BI__builtin_ia32_ucmpb512_mask:
9895 case X86::BI__builtin_ia32_ucmpw128_mask:
9896 case X86::BI__builtin_ia32_ucmpw256_mask:
9897 case X86::BI__builtin_ia32_ucmpw512_mask:
9898 case X86::BI__builtin_ia32_ucmpd128_mask:
9899 case X86::BI__builtin_ia32_ucmpd256_mask:
9900 case X86::BI__builtin_ia32_ucmpd512_mask:
9901 case X86::BI__builtin_ia32_ucmpq128_mask:
9902 case X86::BI__builtin_ia32_ucmpq256_mask:
9903 case X86::BI__builtin_ia32_ucmpq512_mask: {
9904 unsigned CC = cast<llvm::ConstantInt>(Ops[2])->getZExtValue() & 0x7;
9908 case X86::BI__builtin_ia32_kortestchi:
9909 case X86::BI__builtin_ia32_kortestzhi: {
9912 if (BuiltinID == X86::BI__builtin_ia32_kortestchi)
9913 C = llvm::Constant::getAllOnesValue(Builder.getInt16Ty());
9915 C = llvm::Constant::getNullValue(Builder.getInt16Ty());
9916 Value *Cmp = Builder.CreateICmpEQ(Or, C);
9917 return Builder.CreateZExt(Cmp, ConvertType(E->
getType()));
9920 case X86::BI__builtin_ia32_kandhi:
9922 case X86::BI__builtin_ia32_kandnhi:
9924 case X86::BI__builtin_ia32_korhi:
9926 case X86::BI__builtin_ia32_kxnorhi:
9928 case X86::BI__builtin_ia32_kxorhi:
9930 case X86::BI__builtin_ia32_knothi: {
9932 return Builder.CreateBitCast(Builder.CreateNot(Ops[0]),
9933 Builder.getInt16Ty());
9936 case X86::BI__builtin_ia32_kunpckdi:
9937 case X86::BI__builtin_ia32_kunpcksi:
9938 case X86::BI__builtin_ia32_kunpckhi: {
9939 unsigned NumElts = Ops[0]->getType()->getScalarSizeInBits();
9942 uint32_t Indices[64];
9943 for (
unsigned i = 0; i != NumElts; ++i)
9948 LHS = Builder.CreateShuffleVector(LHS, LHS,
9949 makeArrayRef(Indices, NumElts / 2));
9950 RHS = Builder.CreateShuffleVector(RHS, RHS,
9951 makeArrayRef(Indices, NumElts / 2));
9954 Value *Res = Builder.CreateShuffleVector(RHS, LHS,
9955 makeArrayRef(Indices, NumElts));
9956 return Builder.CreateBitCast(Res, Ops[0]->getType());
9959 case X86::BI__builtin_ia32_vplzcntd_128:
9960 case X86::BI__builtin_ia32_vplzcntd_256:
9961 case X86::BI__builtin_ia32_vplzcntd_512:
9962 case X86::BI__builtin_ia32_vplzcntq_128:
9963 case X86::BI__builtin_ia32_vplzcntq_256:
9964 case X86::BI__builtin_ia32_vplzcntq_512: {
9965 Function *F = CGM.getIntrinsic(Intrinsic::ctlz, Ops[0]->getType());
9966 return Builder.CreateCall(F, {Ops[0],Builder.getInt1(
false)});
9968 case X86::BI__builtin_ia32_sqrtss:
9969 case X86::BI__builtin_ia32_sqrtsd: {
9970 Value *A = Builder.CreateExtractElement(Ops[0], (uint64_t)0);
9972 A = Builder.CreateCall(F, {A});
9973 return Builder.CreateInsertElement(Ops[0], A, (uint64_t)0);
9975 case X86::BI__builtin_ia32_sqrtsd_round_mask:
9976 case X86::BI__builtin_ia32_sqrtss_round_mask: {
9977 unsigned CC = cast<llvm::ConstantInt>(Ops[4])->getZExtValue();
9981 Intrinsic::ID IID = BuiltinID == X86::BI__builtin_ia32_sqrtsd_round_mask ?
9982 Intrinsic::x86_avx512_mask_sqrt_sd :
9983 Intrinsic::x86_avx512_mask_sqrt_ss;
9984 return Builder.CreateCall(CGM.getIntrinsic(IID), Ops);
9986 Value *A = Builder.CreateExtractElement(Ops[1], (uint64_t)0);
9988 A = Builder.CreateCall(F, A);
9989 Value *Src = Builder.CreateExtractElement(Ops[2], (uint64_t)0);
9991 return Builder.CreateInsertElement(Ops[0], A, (uint64_t)0);
9993 case X86::BI__builtin_ia32_sqrtpd256:
9994 case X86::BI__builtin_ia32_sqrtpd:
9995 case X86::BI__builtin_ia32_sqrtps256:
9996 case X86::BI__builtin_ia32_sqrtps:
9997 case X86::BI__builtin_ia32_sqrtps512:
9998 case X86::BI__builtin_ia32_sqrtpd512: {
9999 if (Ops.size() == 2) {
10000 unsigned CC = cast<llvm::ConstantInt>(Ops[1])->getZExtValue();
10004 Intrinsic::ID IID = BuiltinID == X86::BI__builtin_ia32_sqrtps512 ?
10005 Intrinsic::x86_avx512_sqrt_ps_512 :
10006 Intrinsic::x86_avx512_sqrt_pd_512;
10007 return Builder.CreateCall(CGM.getIntrinsic(IID), Ops);
10011 return Builder.CreateCall(F, Ops[0]);
10013 case X86::BI__builtin_ia32_pabsb128:
10014 case X86::BI__builtin_ia32_pabsw128:
10015 case X86::BI__builtin_ia32_pabsd128:
10016 case X86::BI__builtin_ia32_pabsb256:
10017 case X86::BI__builtin_ia32_pabsw256:
10018 case X86::BI__builtin_ia32_pabsd256:
10019 case X86::BI__builtin_ia32_pabsq128:
10020 case X86::BI__builtin_ia32_pabsq256:
10021 case X86::BI__builtin_ia32_pabsb512:
10022 case X86::BI__builtin_ia32_pabsw512:
10023 case X86::BI__builtin_ia32_pabsd512:
10024 case X86::BI__builtin_ia32_pabsq512:
10027 case X86::BI__builtin_ia32_pmaxsb128:
10028 case X86::BI__builtin_ia32_pmaxsw128:
10029 case X86::BI__builtin_ia32_pmaxsd128:
10030 case X86::BI__builtin_ia32_pmaxsq128:
10031 case X86::BI__builtin_ia32_pmaxsb256:
10032 case X86::BI__builtin_ia32_pmaxsw256:
10033 case X86::BI__builtin_ia32_pmaxsd256:
10034 case X86::BI__builtin_ia32_pmaxsq256:
10035 case X86::BI__builtin_ia32_pmaxsb512:
10036 case X86::BI__builtin_ia32_pmaxsw512:
10037 case X86::BI__builtin_ia32_pmaxsd512:
10038 case X86::BI__builtin_ia32_pmaxsq512:
10040 case X86::BI__builtin_ia32_pmaxub128:
10041 case X86::BI__builtin_ia32_pmaxuw128:
10042 case X86::BI__builtin_ia32_pmaxud128:
10043 case X86::BI__builtin_ia32_pmaxuq128:
10044 case X86::BI__builtin_ia32_pmaxub256:
10045 case X86::BI__builtin_ia32_pmaxuw256:
10046 case X86::BI__builtin_ia32_pmaxud256:
10047 case X86::BI__builtin_ia32_pmaxuq256:
10048 case X86::BI__builtin_ia32_pmaxub512:
10049 case X86::BI__builtin_ia32_pmaxuw512:
10050 case X86::BI__builtin_ia32_pmaxud512:
10051 case X86::BI__builtin_ia32_pmaxuq512:
10053 case X86::BI__builtin_ia32_pminsb128:
10054 case X86::BI__builtin_ia32_pminsw128:
10055 case X86::BI__builtin_ia32_pminsd128:
10056 case X86::BI__builtin_ia32_pminsq128:
10057 case X86::BI__builtin_ia32_pminsb256:
10058 case X86::BI__builtin_ia32_pminsw256:
10059 case X86::BI__builtin_ia32_pminsd256:
10060 case X86::BI__builtin_ia32_pminsq256:
10061 case X86::BI__builtin_ia32_pminsb512:
10062 case X86::BI__builtin_ia32_pminsw512:
10063 case X86::BI__builtin_ia32_pminsd512:
10064 case X86::BI__builtin_ia32_pminsq512:
10066 case X86::BI__builtin_ia32_pminub128:
10067 case X86::BI__builtin_ia32_pminuw128:
10068 case X86::BI__builtin_ia32_pminud128:
10069 case X86::BI__builtin_ia32_pminuq128:
10070 case X86::BI__builtin_ia32_pminub256:
10071 case X86::BI__builtin_ia32_pminuw256:
10072 case X86::BI__builtin_ia32_pminud256:
10073 case X86::BI__builtin_ia32_pminuq256:
10074 case X86::BI__builtin_ia32_pminub512:
10075 case X86::BI__builtin_ia32_pminuw512:
10076 case X86::BI__builtin_ia32_pminud512:
10077 case X86::BI__builtin_ia32_pminuq512:
10080 case X86::BI__builtin_ia32_pmuludq128:
10081 case X86::BI__builtin_ia32_pmuludq256:
10082 case X86::BI__builtin_ia32_pmuludq512:
10085 case X86::BI__builtin_ia32_pmuldq128:
10086 case X86::BI__builtin_ia32_pmuldq256:
10087 case X86::BI__builtin_ia32_pmuldq512:
10090 case X86::BI__builtin_ia32_pternlogd512_mask:
10091 case X86::BI__builtin_ia32_pternlogq512_mask:
10092 case X86::BI__builtin_ia32_pternlogd128_mask:
10093 case X86::BI__builtin_ia32_pternlogd256_mask:
10094 case X86::BI__builtin_ia32_pternlogq128_mask:
10095 case X86::BI__builtin_ia32_pternlogq256_mask:
10098 case X86::BI__builtin_ia32_pternlogd512_maskz:
10099 case X86::BI__builtin_ia32_pternlogq512_maskz:
10100 case X86::BI__builtin_ia32_pternlogd128_maskz:
10101 case X86::BI__builtin_ia32_pternlogd256_maskz:
10102 case X86::BI__builtin_ia32_pternlogq128_maskz:
10103 case X86::BI__builtin_ia32_pternlogq256_maskz:
10107 case X86::BI__builtin_ia32_pswapdsf:
10108 case X86::BI__builtin_ia32_pswapdsi: {
10109 llvm::Type *MMXTy = llvm::Type::getX86_MMXTy(getLLVMContext());
10110 Ops[0] = Builder.CreateBitCast(Ops[0], MMXTy,
"cast");
10111 llvm::Function *F = CGM.getIntrinsic(Intrinsic::x86_3dnowa_pswapd);
10112 return Builder.CreateCall(F, Ops,
"pswapd");
10114 case X86::BI__builtin_ia32_rdrand16_step:
10115 case X86::BI__builtin_ia32_rdrand32_step:
10116 case X86::BI__builtin_ia32_rdrand64_step:
10117 case X86::BI__builtin_ia32_rdseed16_step:
10118 case X86::BI__builtin_ia32_rdseed32_step:
10119 case X86::BI__builtin_ia32_rdseed64_step: {
10121 switch (BuiltinID) {
10122 default: llvm_unreachable(
"Unsupported intrinsic!");
10123 case X86::BI__builtin_ia32_rdrand16_step:
10124 ID = Intrinsic::x86_rdrand_16;
10126 case X86::BI__builtin_ia32_rdrand32_step:
10127 ID = Intrinsic::x86_rdrand_32;
10129 case X86::BI__builtin_ia32_rdrand64_step:
10130 ID = Intrinsic::x86_rdrand_64;
10132 case X86::BI__builtin_ia32_rdseed16_step:
10133 ID = Intrinsic::x86_rdseed_16;
10135 case X86::BI__builtin_ia32_rdseed32_step:
10136 ID = Intrinsic::x86_rdseed_32;
10138 case X86::BI__builtin_ia32_rdseed64_step:
10139 ID = Intrinsic::x86_rdseed_64;
10143 Value *Call = Builder.CreateCall(CGM.getIntrinsic(ID));
10144 Builder.CreateDefaultAlignedStore(Builder.CreateExtractValue(Call, 0),
10146 return Builder.CreateExtractValue(Call, 1);
10149 case X86::BI__builtin_ia32_fpclassps128_mask:
10150 case X86::BI__builtin_ia32_fpclassps256_mask:
10151 case X86::BI__builtin_ia32_fpclassps512_mask:
10152 case X86::BI__builtin_ia32_fpclasspd128_mask:
10153 case X86::BI__builtin_ia32_fpclasspd256_mask:
10154 case X86::BI__builtin_ia32_fpclasspd512_mask: {
10155 unsigned NumElts = Ops[0]->getType()->getVectorNumElements();
10156 Value *MaskIn = Ops[2];
10157 Ops.erase(&Ops[2]);
10160 switch (BuiltinID) {
10161 default: llvm_unreachable(
"Unsupported intrinsic!");
10162 case X86::BI__builtin_ia32_fpclassps128_mask:
10163 ID = Intrinsic::x86_avx512_fpclass_ps_128;
10165 case X86::BI__builtin_ia32_fpclassps256_mask:
10166 ID = Intrinsic::x86_avx512_fpclass_ps_256;
10168 case X86::BI__builtin_ia32_fpclassps512_mask:
10169 ID = Intrinsic::x86_avx512_fpclass_ps_512;
10171 case X86::BI__builtin_ia32_fpclasspd128_mask:
10172 ID = Intrinsic::x86_avx512_fpclass_pd_128;
10174 case X86::BI__builtin_ia32_fpclasspd256_mask:
10175 ID = Intrinsic::x86_avx512_fpclass_pd_256;
10177 case X86::BI__builtin_ia32_fpclasspd512_mask:
10178 ID = Intrinsic::x86_avx512_fpclass_pd_512;
10182 Value *Fpclass = Builder.CreateCall(CGM.getIntrinsic(ID), Ops);
10187 case X86::BI__builtin_ia32_cmpeqps:
10188 case X86::BI__builtin_ia32_cmpeqpd:
10189 return getVectorFCmpIR(CmpInst::FCMP_OEQ);
10190 case X86::BI__builtin_ia32_cmpltps:
10191 case X86::BI__builtin_ia32_cmpltpd:
10192 return getVectorFCmpIR(CmpInst::FCMP_OLT);
10193 case X86::BI__builtin_ia32_cmpleps:
10194 case X86::BI__builtin_ia32_cmplepd:
10195 return getVectorFCmpIR(CmpInst::FCMP_OLE);
10196 case X86::BI__builtin_ia32_cmpunordps:
10197 case X86::BI__builtin_ia32_cmpunordpd:
10198 return getVectorFCmpIR(CmpInst::FCMP_UNO);
10199 case X86::BI__builtin_ia32_cmpneqps:
10200 case X86::BI__builtin_ia32_cmpneqpd:
10201 return getVectorFCmpIR(CmpInst::FCMP_UNE);
10202 case X86::BI__builtin_ia32_cmpnltps:
10203 case X86::BI__builtin_ia32_cmpnltpd:
10204 return getVectorFCmpIR(CmpInst::FCMP_UGE);
10205 case X86::BI__builtin_ia32_cmpnleps:
10206 case X86::BI__builtin_ia32_cmpnlepd:
10207 return getVectorFCmpIR(CmpInst::FCMP_UGT);
10208 case X86::BI__builtin_ia32_cmpordps:
10209 case X86::BI__builtin_ia32_cmpordpd:
10210 return getVectorFCmpIR(CmpInst::FCMP_ORD);
10211 case X86::BI__builtin_ia32_cmpps:
10212 case X86::BI__builtin_ia32_cmpps256:
10213 case X86::BI__builtin_ia32_cmppd:
10214 case X86::BI__builtin_ia32_cmppd256:
10215 case X86::BI__builtin_ia32_cmpps128_mask:
10216 case X86::BI__builtin_ia32_cmpps256_mask:
10217 case X86::BI__builtin_ia32_cmpps512_mask:
10218 case X86::BI__builtin_ia32_cmppd128_mask:
10219 case X86::BI__builtin_ia32_cmppd256_mask:
10220 case X86::BI__builtin_ia32_cmppd512_mask: {
10229 unsigned CC = cast<llvm::ConstantInt>(Ops[2])->getZExtValue() & 0x1f;
10234 FCmpInst::Predicate Pred;
10236 case 0x00: Pred = FCmpInst::FCMP_OEQ;
break;
10237 case 0x01: Pred = FCmpInst::FCMP_OLT;
break;
10238 case 0x02: Pred = FCmpInst::FCMP_OLE;
break;
10239 case 0x03: Pred = FCmpInst::FCMP_UNO;
break;
10240 case 0x04: Pred = FCmpInst::FCMP_UNE;
break;
10241 case 0x05: Pred = FCmpInst::FCMP_UGE;
break;
10242 case 0x06: Pred = FCmpInst::FCMP_UGT;
break;
10243 case 0x07: Pred = FCmpInst::FCMP_ORD;
break;
10244 case 0x08: Pred = FCmpInst::FCMP_UEQ;
break;
10245 case 0x09: Pred = FCmpInst::FCMP_ULT;
break;
10246 case 0x0a: Pred = FCmpInst::FCMP_ULE;
break;
10247 case 0x0b: Pred = FCmpInst::FCMP_FALSE;
break;
10248 case 0x0c: Pred = FCmpInst::FCMP_ONE;
break;
10249 case 0x0d: Pred = FCmpInst::FCMP_OGE;
break;
10250 case 0x0e: Pred = FCmpInst::FCMP_OGT;
break;
10251 case 0x0f: Pred = FCmpInst::FCMP_TRUE;
break;
10252 case 0x10: Pred = FCmpInst::FCMP_OEQ;
break;
10253 case 0x11: Pred = FCmpInst::FCMP_OLT;
break;
10254 case 0x12: Pred = FCmpInst::FCMP_OLE;
break;
10255 case 0x13: Pred = FCmpInst::FCMP_UNO;
break;
10256 case 0x14: Pred = FCmpInst::FCMP_UNE;
break;
10257 case 0x15: Pred = FCmpInst::FCMP_UGE;
break;
10258 case 0x16: Pred = FCmpInst::FCMP_UGT;
break;
10259 case 0x17: Pred = FCmpInst::FCMP_ORD;
break;
10260 case 0x18: Pred = FCmpInst::FCMP_UEQ;
break;
10261 case 0x19: Pred = FCmpInst::FCMP_ULT;
break;
10262 case 0x1a: Pred = FCmpInst::FCMP_ULE;
break;
10263 case 0x1b: Pred = FCmpInst::FCMP_FALSE;
break;
10264 case 0x1c: Pred = FCmpInst::FCMP_ONE;
break;
10265 case 0x1d: Pred = FCmpInst::FCMP_OGE;
break;
10266 case 0x1e: Pred = FCmpInst::FCMP_OGT;
break;
10267 case 0x1f: Pred = FCmpInst::FCMP_TRUE;
break;
10268 default: llvm_unreachable(
"Unhandled CC");
10273 switch (BuiltinID) {
10274 case X86::BI__builtin_ia32_cmpps512_mask:
10275 case X86::BI__builtin_ia32_cmppd512_mask:
10276 case X86::BI__builtin_ia32_cmpps128_mask:
10277 case X86::BI__builtin_ia32_cmpps256_mask:
10278 case X86::BI__builtin_ia32_cmppd128_mask:
10279 case X86::BI__builtin_ia32_cmppd256_mask: {
10280 unsigned NumElts = Ops[0]->getType()->getVectorNumElements();
10281 Value *Cmp = Builder.CreateFCmp(Pred, Ops[0], Ops[1]);
10285 return getVectorFCmpIR(Pred);
10290 case X86::BI__builtin_ia32_cmpeqss:
10291 return getCmpIntrinsicCall(Intrinsic::x86_sse_cmp_ss, 0);
10292 case X86::BI__builtin_ia32_cmpltss:
10293 return getCmpIntrinsicCall(Intrinsic::x86_sse_cmp_ss, 1);
10294 case X86::BI__builtin_ia32_cmpless:
10295 return getCmpIntrinsicCall(Intrinsic::x86_sse_cmp_ss, 2);
10296 case X86::BI__builtin_ia32_cmpunordss:
10297 return getCmpIntrinsicCall(Intrinsic::x86_sse_cmp_ss, 3);
10298 case X86::BI__builtin_ia32_cmpneqss:
10299 return getCmpIntrinsicCall(Intrinsic::x86_sse_cmp_ss, 4);
10300 case X86::BI__builtin_ia32_cmpnltss:
10301 return getCmpIntrinsicCall(Intrinsic::x86_sse_cmp_ss, 5);
10302 case X86::BI__builtin_ia32_cmpnless:
10303 return getCmpIntrinsicCall(Intrinsic::x86_sse_cmp_ss, 6);
10304 case X86::BI__builtin_ia32_cmpordss:
10305 return getCmpIntrinsicCall(Intrinsic::x86_sse_cmp_ss, 7);
10306 case X86::BI__builtin_ia32_cmpeqsd:
10307 return getCmpIntrinsicCall(Intrinsic::x86_sse2_cmp_sd, 0);
10308 case X86::BI__builtin_ia32_cmpltsd:
10309 return getCmpIntrinsicCall(Intrinsic::x86_sse2_cmp_sd, 1);
10310 case X86::BI__builtin_ia32_cmplesd:
10311 return getCmpIntrinsicCall(Intrinsic::x86_sse2_cmp_sd, 2);
10312 case X86::BI__builtin_ia32_cmpunordsd:
10313 return getCmpIntrinsicCall(Intrinsic::x86_sse2_cmp_sd, 3);
10314 case X86::BI__builtin_ia32_cmpneqsd:
10315 return getCmpIntrinsicCall(Intrinsic::x86_sse2_cmp_sd, 4);
10316 case X86::BI__builtin_ia32_cmpnltsd:
10317 return getCmpIntrinsicCall(Intrinsic::x86_sse2_cmp_sd, 5);
10318 case X86::BI__builtin_ia32_cmpnlesd:
10319 return getCmpIntrinsicCall(Intrinsic::x86_sse2_cmp_sd, 6);
10320 case X86::BI__builtin_ia32_cmpordsd:
10321 return getCmpIntrinsicCall(Intrinsic::x86_sse2_cmp_sd, 7);
10323 case X86::BI__emul:
10324 case X86::BI__emulu: {
10325 llvm::Type *Int64Ty = llvm::IntegerType::get(getLLVMContext(), 64);
10326 bool isSigned = (BuiltinID == X86::BI__emul);
10327 Value *LHS = Builder.CreateIntCast(Ops[0], Int64Ty, isSigned);
10328 Value *RHS = Builder.CreateIntCast(Ops[1], Int64Ty, isSigned);
10329 return Builder.CreateMul(LHS, RHS,
"", !isSigned, isSigned);
10331 case X86::BI__mulh:
10332 case X86::BI__umulh:
10333 case X86::BI_mul128:
10334 case X86::BI_umul128: {
10336 llvm::Type *Int128Ty = llvm::IntegerType::get(getLLVMContext(), 128);
10338 bool IsSigned = (BuiltinID == X86::BI__mulh || BuiltinID == X86::BI_mul128);
10339 Value *LHS = Builder.CreateIntCast(Ops[0], Int128Ty, IsSigned);
10340 Value *RHS = Builder.CreateIntCast(Ops[1], Int128Ty, IsSigned);
10342 Value *MulResult, *HigherBits;
10344 MulResult = Builder.CreateNSWMul(LHS, RHS);
10345 HigherBits = Builder.CreateAShr(MulResult, 64);
10347 MulResult = Builder.CreateNUWMul(LHS, RHS);
10348 HigherBits = Builder.CreateLShr(MulResult, 64);
10350 HigherBits = Builder.CreateIntCast(HigherBits, ResType, IsSigned);
10352 if (BuiltinID == X86::BI__mulh || BuiltinID == X86::BI__umulh)
10355 Address HighBitsAddress = EmitPointerWithAlignment(E->
getArg(2));
10356 Builder.CreateStore(HigherBits, HighBitsAddress);
10357 return Builder.CreateIntCast(MulResult, ResType, IsSigned);
10360 case X86::BI__faststorefence: {
10361 return Builder.CreateFence(llvm::AtomicOrdering::SequentiallyConsistent,
10364 case X86::BI__shiftleft128:
10365 case X86::BI__shiftright128: {
10372 llvm::Type *Int128Ty = Builder.getInt128Ty();
10373 Value *Val = Builder.CreateOr(
10374 Builder.CreateShl(Builder.CreateZExt(Ops[1], Int128Ty), 64),
10375 Builder.CreateZExt(Ops[0], Int128Ty));
10376 Value *Amt = Builder.CreateAnd(Builder.CreateZExt(Ops[2], Int128Ty),
10377 llvm::ConstantInt::get(Int128Ty, 0x3f));
10379 if (BuiltinID == X86::BI__shiftleft128)
10380 Res = Builder.CreateLShr(Builder.CreateShl(Val, Amt), 64);
10382 Res = Builder.CreateLShr(Val, Amt);
10383 return Builder.CreateTrunc(Res, Int64Ty);
10385 case X86::BI_ReadWriteBarrier:
10386 case X86::BI_ReadBarrier:
10387 case X86::BI_WriteBarrier: {
10388 return Builder.CreateFence(llvm::AtomicOrdering::SequentiallyConsistent,
10389 llvm::SyncScope::SingleThread);
10391 case X86::BI_BitScanForward:
10392 case X86::BI_BitScanForward64:
10393 return EmitMSVCBuiltinExpr(MSVCIntrin::_BitScanForward, E);
10394 case X86::BI_BitScanReverse:
10395 case X86::BI_BitScanReverse64:
10396 return EmitMSVCBuiltinExpr(MSVCIntrin::_BitScanReverse, E);
10398 case X86::BI_InterlockedAnd64:
10399 return EmitMSVCBuiltinExpr(MSVCIntrin::_InterlockedAnd, E);
10400 case X86::BI_InterlockedExchange64:
10401 return EmitMSVCBuiltinExpr(MSVCIntrin::_InterlockedExchange, E);
10402 case X86::BI_InterlockedExchangeAdd64:
10403 return EmitMSVCBuiltinExpr(MSVCIntrin::_InterlockedExchangeAdd, E);
10404 case X86::BI_InterlockedExchangeSub64:
10405 return EmitMSVCBuiltinExpr(MSVCIntrin::_InterlockedExchangeSub, E);
10406 case X86::BI_InterlockedOr64:
10407 return EmitMSVCBuiltinExpr(MSVCIntrin::_InterlockedOr, E);
10408 case X86::BI_InterlockedXor64:
10409 return EmitMSVCBuiltinExpr(MSVCIntrin::_InterlockedXor, E);
10410 case X86::BI_InterlockedDecrement64:
10411 return EmitMSVCBuiltinExpr(MSVCIntrin::_InterlockedDecrement, E);
10412 case X86::BI_InterlockedIncrement64:
10413 return EmitMSVCBuiltinExpr(MSVCIntrin::_InterlockedIncrement, E);
10414 case X86::BI_InterlockedCompareExchange128: {
10421 llvm::Type *Int128Ty = Builder.getInt128Ty();
10422 llvm::Type *Int128PtrTy = Int128Ty->getPointerTo();
10424 Value *Destination =
10425 Builder.CreateBitCast(EmitScalarExpr(E->
getArg(0)), Int128PtrTy);
10426 Value *ExchangeHigh128 =
10427 Builder.CreateZExt(EmitScalarExpr(E->
getArg(1)), Int128Ty);
10428 Value *ExchangeLow128 =
10429 Builder.CreateZExt(EmitScalarExpr(E->
getArg(2)), Int128Ty);
10431 Builder.CreateBitCast(EmitScalarExpr(E->
getArg(3)), Int128PtrTy),
10432 getContext().toCharUnitsFromBits(128));
10434 Value *Exchange = Builder.CreateOr(
10435 Builder.CreateShl(ExchangeHigh128, 64,
"",
false,
false),
10438 Value *Comparand = Builder.CreateLoad(ComparandResult);
10440 AtomicCmpXchgInst *CXI =
10441 Builder.CreateAtomicCmpXchg(Destination, Comparand, Exchange,
10442 AtomicOrdering::SequentiallyConsistent,
10443 AtomicOrdering::SequentiallyConsistent);
10444 CXI->setVolatile(
true);
10447 Builder.CreateStore(Builder.CreateExtractValue(CXI, 0), ComparandResult);
10450 Value *Success = Builder.CreateExtractValue(CXI, 1);
10451 return Builder.CreateZExt(Success, ConvertType(E->
getType()));
10454 case X86::BI_AddressOfReturnAddress: {
10455 Value *F = CGM.getIntrinsic(Intrinsic::addressofreturnaddress);
10456 return Builder.CreateCall(F);
10458 case X86::BI__stosb: {
10461 return Builder.CreateMemSet(Ops[0], Ops[1], Ops[2], 1,
true);
10465 return EmitTrapCall(Intrinsic::trap);
10466 case X86::BI__int2c: {
10468 llvm::FunctionType *FTy = llvm::FunctionType::get(VoidTy,
false);
10469 llvm::InlineAsm *IA =
10470 llvm::InlineAsm::get(FTy,
"int $$0x2c",
"",
true);
10471 llvm::AttributeList NoReturnAttr = llvm::AttributeList::get(
10472 getLLVMContext(), llvm::AttributeList::FunctionIndex,
10473 llvm::Attribute::NoReturn);
10474 CallSite CS = Builder.CreateCall(IA);
10475 CS.setAttributes(NoReturnAttr);
10476 return CS.getInstruction();
10478 case X86::BI__readfsbyte:
10479 case X86::BI__readfsword:
10480 case X86::BI__readfsdword:
10481 case X86::BI__readfsqword: {
10483 Value *Ptr = Builder.CreateIntToPtr(EmitScalarExpr(E->
getArg(0)),
10484 llvm::PointerType::get(IntTy, 257));
10485 LoadInst *Load = Builder.CreateAlignedLoad(
10486 IntTy, Ptr, getContext().getTypeAlignInChars(E->
getType()));
10487 Load->setVolatile(
true);
10490 case X86::BI__readgsbyte:
10491 case X86::BI__readgsword:
10492 case X86::BI__readgsdword:
10493 case X86::BI__readgsqword: {
10495 Value *Ptr = Builder.CreateIntToPtr(EmitScalarExpr(E->
getArg(0)),
10496 llvm::PointerType::get(IntTy, 256));
10497 LoadInst *Load = Builder.CreateAlignedLoad(
10498 IntTy, Ptr, getContext().getTypeAlignInChars(E->
getType()));
10499 Load->setVolatile(
true);
10510 for (
unsigned i = 0, e = E->
getNumArgs(); i != e; i++)
10511 Ops.push_back(EmitScalarExpr(E->
getArg(i)));
10515 switch (BuiltinID) {
10516 default:
return nullptr;
10520 case PPC::BI__builtin_ppc_get_timebase:
10521 return Builder.CreateCall(CGM.getIntrinsic(Intrinsic::readcyclecounter));
10524 case PPC::BI__builtin_altivec_lvx:
10525 case PPC::BI__builtin_altivec_lvxl:
10526 case PPC::BI__builtin_altivec_lvebx:
10527 case PPC::BI__builtin_altivec_lvehx:
10528 case PPC::BI__builtin_altivec_lvewx:
10529 case PPC::BI__builtin_altivec_lvsl:
10530 case PPC::BI__builtin_altivec_lvsr:
10531 case PPC::BI__builtin_vsx_lxvd2x:
10532 case PPC::BI__builtin_vsx_lxvw4x:
10533 case PPC::BI__builtin_vsx_lxvd2x_be:
10534 case PPC::BI__builtin_vsx_lxvw4x_be:
10535 case PPC::BI__builtin_vsx_lxvl:
10536 case PPC::BI__builtin_vsx_lxvll:
10538 if(BuiltinID == PPC::BI__builtin_vsx_lxvl ||
10539 BuiltinID == PPC::BI__builtin_vsx_lxvll){
10540 Ops[0] = Builder.CreateBitCast(Ops[0], Int8PtrTy);
10542 Ops[1] = Builder.CreateBitCast(Ops[1], Int8PtrTy);
10543 Ops[0] = Builder.CreateGEP(Ops[1], Ops[0]);
10547 switch (BuiltinID) {
10548 default: llvm_unreachable(
"Unsupported ld/lvsl/lvsr intrinsic!");
10549 case PPC::BI__builtin_altivec_lvx:
10550 ID = Intrinsic::ppc_altivec_lvx;
10552 case PPC::BI__builtin_altivec_lvxl:
10553 ID = Intrinsic::ppc_altivec_lvxl;
10555 case PPC::BI__builtin_altivec_lvebx:
10556 ID = Intrinsic::ppc_altivec_lvebx;
10558 case PPC::BI__builtin_altivec_lvehx:
10559 ID = Intrinsic::ppc_altivec_lvehx;
10561 case PPC::BI__builtin_altivec_lvewx:
10562 ID = Intrinsic::ppc_altivec_lvewx;
10564 case PPC::BI__builtin_altivec_lvsl:
10565 ID = Intrinsic::ppc_altivec_lvsl;
10567 case PPC::BI__builtin_altivec_lvsr:
10568 ID = Intrinsic::ppc_altivec_lvsr;
10570 case PPC::BI__builtin_vsx_lxvd2x:
10571 ID = Intrinsic::ppc_vsx_lxvd2x;
10573 case PPC::BI__builtin_vsx_lxvw4x:
10574 ID = Intrinsic::ppc_vsx_lxvw4x;
10576 case PPC::BI__builtin_vsx_lxvd2x_be:
10577 ID = Intrinsic::ppc_vsx_lxvd2x_be;
10579 case PPC::BI__builtin_vsx_lxvw4x_be:
10580 ID = Intrinsic::ppc_vsx_lxvw4x_be;
10582 case PPC::BI__builtin_vsx_lxvl:
10583 ID = Intrinsic::ppc_vsx_lxvl;
10585 case PPC::BI__builtin_vsx_lxvll:
10586 ID = Intrinsic::ppc_vsx_lxvll;
10589 llvm::Function *F = CGM.getIntrinsic(ID);
10590 return Builder.CreateCall(F, Ops,
"");
10594 case PPC::BI__builtin_altivec_stvx:
10595 case PPC::BI__builtin_altivec_stvxl:
10596 case PPC::BI__builtin_altivec_stvebx:
10597 case PPC::BI__builtin_altivec_stvehx:
10598 case PPC::BI__builtin_altivec_stvewx:
10599 case PPC::BI__builtin_vsx_stxvd2x:
10600 case PPC::BI__builtin_vsx_stxvw4x:
10601 case PPC::BI__builtin_vsx_stxvd2x_be:
10602 case PPC::BI__builtin_vsx_stxvw4x_be:
10603 case PPC::BI__builtin_vsx_stxvl:
10604 case PPC::BI__builtin_vsx_stxvll:
10606 if(BuiltinID == PPC::BI__builtin_vsx_stxvl ||
10607 BuiltinID == PPC::BI__builtin_vsx_stxvll ){
10608 Ops[1] = Builder.CreateBitCast(Ops[1], Int8PtrTy);
10610 Ops[2] = Builder.CreateBitCast(Ops[2], Int8PtrTy);
10611 Ops[1] = Builder.CreateGEP(Ops[2], Ops[1]);
10615 switch (BuiltinID) {
10616 default: llvm_unreachable(
"Unsupported st intrinsic!");
10617 case PPC::BI__builtin_altivec_stvx:
10618 ID = Intrinsic::ppc_altivec_stvx;
10620 case PPC::BI__builtin_altivec_stvxl:
10621 ID = Intrinsic::ppc_altivec_stvxl;
10623 case PPC::BI__builtin_altivec_stvebx:
10624 ID = Intrinsic::ppc_altivec_stvebx;
10626 case PPC::BI__builtin_altivec_stvehx:
10627 ID = Intrinsic::ppc_altivec_stvehx;
10629 case PPC::BI__builtin_altivec_stvewx:
10630 ID = Intrinsic::ppc_altivec_stvewx;
10632 case PPC::BI__builtin_vsx_stxvd2x:
10633 ID = Intrinsic::ppc_vsx_stxvd2x;
10635 case PPC::BI__builtin_vsx_stxvw4x:
10636 ID = Intrinsic::ppc_vsx_stxvw4x;
10638 case PPC::BI__builtin_vsx_stxvd2x_be:
10639 ID = Intrinsic::ppc_vsx_stxvd2x_be;
10641 case PPC::BI__builtin_vsx_stxvw4x_be:
10642 ID = Intrinsic::ppc_vsx_stxvw4x_be;
10644 case PPC::BI__builtin_vsx_stxvl:
10645 ID = Intrinsic::ppc_vsx_stxvl;
10647 case PPC::BI__builtin_vsx_stxvll:
10648 ID = Intrinsic::ppc_vsx_stxvll;
10651 llvm::Function *F = CGM.getIntrinsic(ID);
10652 return Builder.CreateCall(F, Ops,
"");
10655 case PPC::BI__builtin_vsx_xvsqrtsp:
10656 case PPC::BI__builtin_vsx_xvsqrtdp: {
10660 llvm::Function *F = CGM.getIntrinsic(ID, ResultType);
10661 return Builder.CreateCall(F, X);
10664 case PPC::BI__builtin_altivec_vclzb:
10665 case PPC::BI__builtin_altivec_vclzh:
10666 case PPC::BI__builtin_altivec_vclzw:
10667 case PPC::BI__builtin_altivec_vclzd: {
10670 Value *Undef = ConstantInt::get(Builder.getInt1Ty(),
false);
10671 Function *F = CGM.getIntrinsic(Intrinsic::ctlz, ResultType);
10672 return Builder.CreateCall(F, {
X, Undef});
10674 case PPC::BI__builtin_altivec_vctzb:
10675 case PPC::BI__builtin_altivec_vctzh:
10676 case PPC::BI__builtin_altivec_vctzw:
10677 case PPC::BI__builtin_altivec_vctzd: {
10680 Value *Undef = ConstantInt::get(Builder.getInt1Ty(),
false);
10681 Function *F = CGM.getIntrinsic(Intrinsic::cttz, ResultType);
10682 return Builder.CreateCall(F, {
X, Undef});
10684 case PPC::BI__builtin_altivec_vpopcntb:
10685 case PPC::BI__builtin_altivec_vpopcnth:
10686 case PPC::BI__builtin_altivec_vpopcntw:
10687 case PPC::BI__builtin_altivec_vpopcntd: {
10690 llvm::Function *F = CGM.getIntrinsic(Intrinsic::ctpop, ResultType);
10691 return Builder.CreateCall(F, X);
10694 case PPC::BI__builtin_vsx_xvcpsgnsp:
10695 case PPC::BI__builtin_vsx_xvcpsgndp: {
10700 llvm::Function *F = CGM.getIntrinsic(ID, ResultType);
10701 return Builder.CreateCall(F, {
X, Y});
10704 case PPC::BI__builtin_vsx_xvrspip:
10705 case PPC::BI__builtin_vsx_xvrdpip:
10706 case PPC::BI__builtin_vsx_xvrdpim:
10707 case PPC::BI__builtin_vsx_xvrspim:
10708 case PPC::BI__builtin_vsx_xvrdpi:
10709 case PPC::BI__builtin_vsx_xvrspi:
10710 case PPC::BI__builtin_vsx_xvrdpic:
10711 case PPC::BI__builtin_vsx_xvrspic:
10712 case PPC::BI__builtin_vsx_xvrdpiz:
10713 case PPC::BI__builtin_vsx_xvrspiz: {
10716 if (BuiltinID == PPC::BI__builtin_vsx_xvrdpim ||
10717 BuiltinID == PPC::BI__builtin_vsx_xvrspim)
10719 else if (BuiltinID == PPC::BI__builtin_vsx_xvrdpi ||
10720 BuiltinID == PPC::BI__builtin_vsx_xvrspi)
10722 else if (BuiltinID == PPC::BI__builtin_vsx_xvrdpic ||
10723 BuiltinID == PPC::BI__builtin_vsx_xvrspic)
10725 else if (BuiltinID == PPC::BI__builtin_vsx_xvrdpip ||
10726 BuiltinID == PPC::BI__builtin_vsx_xvrspip)
10728 else if (BuiltinID == PPC::BI__builtin_vsx_xvrdpiz ||
10729 BuiltinID == PPC::BI__builtin_vsx_xvrspiz)
10731 llvm::Function *F = CGM.getIntrinsic(ID, ResultType);
10732 return Builder.CreateCall(F, X);
10736 case PPC::BI__builtin_vsx_xvabsdp:
10737 case PPC::BI__builtin_vsx_xvabssp: {
10741 return Builder.CreateCall(F, X);
10745 case PPC::BI__builtin_vsx_xvmaddadp:
10746 case PPC::BI__builtin_vsx_xvmaddasp:
10747 case PPC::BI__builtin_vsx_xvnmaddadp:
10748 case PPC::BI__builtin_vsx_xvnmaddasp:
10749 case PPC::BI__builtin_vsx_xvmsubadp:
10750 case PPC::BI__builtin_vsx_xvmsubasp:
10751 case PPC::BI__builtin_vsx_xvnmsubadp:
10752 case PPC::BI__builtin_vsx_xvnmsubasp: {
10757 Value *Zero = llvm::ConstantFP::getZeroValueForNegation(ResultType);
10758 llvm::Function *F = CGM.getIntrinsic(
Intrinsic::fma, ResultType);
10759 switch (BuiltinID) {
10760 case PPC::BI__builtin_vsx_xvmaddadp:
10761 case PPC::BI__builtin_vsx_xvmaddasp:
10762 return Builder.CreateCall(F, {
X, Y, Z});
10763 case PPC::BI__builtin_vsx_xvnmaddadp:
10764 case PPC::BI__builtin_vsx_xvnmaddasp:
10765 return Builder.CreateFSub(Zero,
10766 Builder.CreateCall(F, {X, Y, Z}),
"sub");
10767 case PPC::BI__builtin_vsx_xvmsubadp:
10768 case PPC::BI__builtin_vsx_xvmsubasp:
10769 return Builder.CreateCall(F,
10770 {
X, Y, Builder.CreateFSub(Zero, Z,
"sub")});
10771 case PPC::BI__builtin_vsx_xvnmsubadp:
10772 case PPC::BI__builtin_vsx_xvnmsubasp:
10774 Builder.CreateCall(F, {
X, Y, Builder.CreateFSub(Zero, Z,
"sub")});
10775 return Builder.CreateFSub(Zero, FsubRes,
"sub");
10777 llvm_unreachable(
"Unknown FMA operation");
10781 case PPC::BI__builtin_vsx_insertword: {
10782 llvm::Function *F = CGM.getIntrinsic(Intrinsic::ppc_vsx_xxinsertw);
10786 ConstantInt *ArgCI = dyn_cast<ConstantInt>(Ops[2]);
10788 "Third arg to xxinsertw intrinsic must be constant integer");
10789 const int64_t MaxIndex = 12;
10790 int64_t Index =
clamp(ArgCI->getSExtValue(), 0, MaxIndex);
10797 std::swap(Ops[0], Ops[1]);
10801 Ops[1] = Builder.CreateBitCast(Ops[1], llvm::VectorType::get(Int64Ty, 2));
10803 if (getTarget().isLittleEndian()) {
10805 Constant *ShuffleElts[2] = { ConstantInt::get(Int32Ty, 1),
10806 ConstantInt::get(Int32Ty, 0)
10808 Constant *ShuffleMask = llvm::ConstantVector::get(ShuffleElts);
10811 Ops[0] = Builder.CreateBitCast(Ops[0], llvm::VectorType::get(Int64Ty, 2));
10812 Ops[0] = Builder.CreateShuffleVector(Ops[0], Ops[0], ShuffleMask);
10815 Index = MaxIndex - Index;
10819 Ops[0] = Builder.CreateBitCast(Ops[0], llvm::VectorType::get(Int32Ty, 4));
10820 Ops[2] = ConstantInt::getSigned(Int32Ty, Index);
10821 return Builder.CreateCall(F, Ops);
10824 case PPC::BI__builtin_vsx_extractuword: {
10825 llvm::Function *F = CGM.getIntrinsic(Intrinsic::ppc_vsx_xxextractuw);
10828 Ops[0] = Builder.CreateBitCast(Ops[0], llvm::VectorType::get(Int64Ty, 2));
10832 ConstantInt *ArgCI = dyn_cast<ConstantInt>(Ops[1]);
10834 "Second Arg to xxextractuw intrinsic must be a constant integer!");
10835 const int64_t MaxIndex = 12;
10836 int64_t Index =
clamp(ArgCI->getSExtValue(), 0, MaxIndex);
10838 if (getTarget().isLittleEndian()) {
10840 Index = MaxIndex - Index;
10841 Ops[1] = ConstantInt::getSigned(Int32Ty, Index);
10844 Value *Call = Builder.CreateCall(F, Ops);
10847 Constant *ShuffleElts[2] = { ConstantInt::get(Int32Ty, 1),
10848 ConstantInt::get(Int32Ty, 0)
10850 Constant *ShuffleMask = llvm::ConstantVector::get(ShuffleElts);
10852 Value *ShuffleCall = Builder.CreateShuffleVector(Call, Call, ShuffleMask);
10853 return ShuffleCall;
10855 Ops[1] = ConstantInt::getSigned(Int32Ty, Index);
10856 return Builder.CreateCall(F, Ops);
10860 case PPC::BI__builtin_vsx_xxpermdi: {
10861 ConstantInt *ArgCI = dyn_cast<ConstantInt>(Ops[2]);
10862 assert(ArgCI &&
"Third arg must be constant integer!");
10864 unsigned Index = ArgCI->getZExtValue();
10865 Ops[0] = Builder.CreateBitCast(Ops[0], llvm::VectorType::get(Int64Ty, 2));
10866 Ops[1] = Builder.CreateBitCast(Ops[1], llvm::VectorType::get(Int64Ty, 2));
10871 unsigned ElemIdx0 = (Index & 2) >> 1;
10872 unsigned ElemIdx1 = 2 + (Index & 1);
10874 Constant *ShuffleElts[2] = {ConstantInt::get(Int32Ty, ElemIdx0),
10875 ConstantInt::get(Int32Ty, ElemIdx1)};
10876 Constant *ShuffleMask = llvm::ConstantVector::get(ShuffleElts);
10878 Value *ShuffleCall =
10879 Builder.CreateShuffleVector(Ops[0], Ops[1], ShuffleMask);
10881 auto RetTy = ConvertType(BIRetType);
10882 return Builder.CreateBitCast(ShuffleCall, RetTy);
10885 case PPC::BI__builtin_vsx_xxsldwi: {
10886 ConstantInt *ArgCI = dyn_cast<ConstantInt>(Ops[2]);
10887 assert(ArgCI &&
"Third argument must be a compile time constant");
10888 unsigned Index = ArgCI->getZExtValue() & 0x3;
10889 Ops[0] = Builder.CreateBitCast(Ops[0], llvm::VectorType::get(Int32Ty, 4));
10890 Ops[1] = Builder.CreateBitCast(Ops[1], llvm::VectorType::get(Int32Ty, 4));
10897 if (getTarget().isLittleEndian()) {
10901 ElemIdx0 = (8 - Index) % 8;
10902 ElemIdx1 = (9 - Index) % 8;
10903 ElemIdx2 = (10 - Index) % 8;
10904 ElemIdx3 = (11 - Index) % 8;
10908 ElemIdx1 = Index + 1;
10909 ElemIdx2 = Index + 2;
10910 ElemIdx3 = Index + 3;
10913 Constant *ShuffleElts[4] = {ConstantInt::get(Int32Ty, ElemIdx0),
10914 ConstantInt::get(Int32Ty, ElemIdx1),
10915 ConstantInt::get(Int32Ty, ElemIdx2),
10916 ConstantInt::get(Int32Ty, ElemIdx3)};
10918 Constant *ShuffleMask = llvm::ConstantVector::get(ShuffleElts);
10919 Value *ShuffleCall =
10920 Builder.CreateShuffleVector(Ops[0], Ops[1], ShuffleMask);
10922 auto RetTy = ConvertType(BIRetType);
10923 return Builder.CreateBitCast(ShuffleCall, RetTy);
10930 switch (BuiltinID) {
10931 case AMDGPU::BI__builtin_amdgcn_div_scale:
10932 case AMDGPU::BI__builtin_amdgcn_div_scalef: {
10936 Address FlagOutPtr = EmitPointerWithAlignment(E->
getArg(3));
10942 llvm::Value *Callee = CGM.getIntrinsic(Intrinsic::amdgcn_div_scale,
10945 llvm::Value *Tmp = Builder.CreateCall(Callee, {
X, Y, Z});
10947 llvm::Value *Result = Builder.CreateExtractValue(Tmp, 0);
10948 llvm::Value *Flag = Builder.CreateExtractValue(Tmp, 1);
10951 = FlagOutPtr.
getPointer()->getType()->getPointerElementType();
10953 llvm::Value *FlagExt = Builder.CreateZExt(Flag, RealFlagType);
10954 Builder.CreateStore(FlagExt, FlagOutPtr);
10957 case AMDGPU::BI__builtin_amdgcn_div_fmas:
10958 case AMDGPU::BI__builtin_amdgcn_div_fmasf: {
10964 llvm::Value *F = CGM.getIntrinsic(Intrinsic::amdgcn_div_fmas,
10966 llvm::Value *Src3ToBool = Builder.CreateIsNotNull(Src3);
10967 return Builder.CreateCall(F, {Src0, Src1, Src2, Src3ToBool});
10970 case AMDGPU::BI__builtin_amdgcn_ds_swizzle:
10972 case AMDGPU::BI__builtin_amdgcn_mov_dpp: {
10974 for (
unsigned I = 0; I != 5; ++I)
10975 Args.push_back(EmitScalarExpr(E->
getArg(I)));
10976 Value *F = CGM.getIntrinsic(Intrinsic::amdgcn_mov_dpp,
10977 Args[0]->getType());
10978 return Builder.CreateCall(F, Args);
10980 case AMDGPU::BI__builtin_amdgcn_div_fixup:
10981 case AMDGPU::BI__builtin_amdgcn_div_fixupf:
10982 case AMDGPU::BI__builtin_amdgcn_div_fixuph:
10984 case AMDGPU::BI__builtin_amdgcn_trig_preop:
10985 case AMDGPU::BI__builtin_amdgcn_trig_preopf:
10987 case AMDGPU::BI__builtin_amdgcn_rcp:
10988 case AMDGPU::BI__builtin_amdgcn_rcpf:
10989 case AMDGPU::BI__builtin_amdgcn_rcph:
10991 case AMDGPU::BI__builtin_amdgcn_rsq:
10992 case AMDGPU::BI__builtin_amdgcn_rsqf:
10993 case AMDGPU::BI__builtin_amdgcn_rsqh:
10995 case AMDGPU::BI__builtin_amdgcn_rsq_clamp:
10996 case AMDGPU::BI__builtin_amdgcn_rsq_clampf:
10998 case AMDGPU::BI__builtin_amdgcn_sinf:
10999 case AMDGPU::BI__builtin_amdgcn_sinh:
11001 case AMDGPU::BI__builtin_amdgcn_cosf:
11002 case AMDGPU::BI__builtin_amdgcn_cosh:
11004 case AMDGPU::BI__builtin_amdgcn_log_clampf:
11006 case AMDGPU::BI__builtin_amdgcn_ldexp:
11007 case AMDGPU::BI__builtin_amdgcn_ldexpf:
11008 case AMDGPU::BI__builtin_amdgcn_ldexph:
11010 case AMDGPU::BI__builtin_amdgcn_frexp_mant:
11011 case AMDGPU::BI__builtin_amdgcn_frexp_mantf:
11012 case AMDGPU::BI__builtin_amdgcn_frexp_manth:
11014 case AMDGPU::BI__builtin_amdgcn_frexp_exp:
11015 case AMDGPU::BI__builtin_amdgcn_frexp_expf: {
11017 Value *F = CGM.getIntrinsic(Intrinsic::amdgcn_frexp_exp,
11018 { Builder.getInt32Ty(), Src0->getType() });
11019 return Builder.CreateCall(F, Src0);
11021 case AMDGPU::BI__builtin_amdgcn_frexp_exph: {
11023 Value *F = CGM.getIntrinsic(Intrinsic::amdgcn_frexp_exp,
11024 { Builder.getInt16Ty(), Src0->getType() });
11025 return Builder.CreateCall(F, Src0);
11027 case AMDGPU::BI__builtin_amdgcn_fract:
11028 case AMDGPU::BI__builtin_amdgcn_fractf:
11029 case AMDGPU::BI__builtin_amdgcn_fracth:
11031 case AMDGPU::BI__builtin_amdgcn_lerp:
11033 case AMDGPU::BI__builtin_amdgcn_uicmp:
11034 case AMDGPU::BI__builtin_amdgcn_uicmpl:
11035 case AMDGPU::BI__builtin_amdgcn_sicmp:
11036 case AMDGPU::BI__builtin_amdgcn_sicmpl:
11038 case AMDGPU::BI__builtin_amdgcn_fcmp:
11039 case AMDGPU::BI__builtin_amdgcn_fcmpf:
11041 case AMDGPU::BI__builtin_amdgcn_class:
11042 case AMDGPU::BI__builtin_amdgcn_classf:
11043 case AMDGPU::BI__builtin_amdgcn_classh:
11045 case AMDGPU::BI__builtin_amdgcn_fmed3f:
11046 case AMDGPU::BI__builtin_amdgcn_fmed3h:
11048 case AMDGPU::BI__builtin_amdgcn_read_exec: {
11049 CallInst *CI = cast<CallInst>(
11051 CI->setConvergent();
11054 case AMDGPU::BI__builtin_amdgcn_read_exec_lo:
11055 case AMDGPU::BI__builtin_amdgcn_read_exec_hi: {
11056 StringRef RegName = BuiltinID == AMDGPU::BI__builtin_amdgcn_read_exec_lo ?
11057 "exec_lo" :
"exec_hi";
11058 CallInst *CI = cast<CallInst>(
11060 CI->setConvergent();
11063 case AMDGPU::BI__builtin_amdgcn_ds_faddf:
11064 case AMDGPU::BI__builtin_amdgcn_ds_fminf:
11065 case AMDGPU::BI__builtin_amdgcn_ds_fmaxf: {
11067 for (
unsigned I = 0; I != 5; ++I)
11068 Args.push_back(EmitScalarExpr(E->
getArg(I)));
11069 const llvm::Type *PtrTy = Args[0]->getType();
11071 if (!PtrTy->isPointerTy() ||
11077 !PtrTy->getPointerElementType()->isFloatTy()) {
11079 "parameter should have type \"local float*\"");
11083 if (!Args[1]->getType()->isFloatTy()) {
11085 "parameter should have type \"float\"");
11090 switch (BuiltinID) {
11091 case AMDGPU::BI__builtin_amdgcn_ds_faddf:
11092 ID = Intrinsic::amdgcn_ds_fadd;
11094 case AMDGPU::BI__builtin_amdgcn_ds_fminf:
11095 ID = Intrinsic::amdgcn_ds_fmin;
11097 case AMDGPU::BI__builtin_amdgcn_ds_fmaxf:
11098 ID = Intrinsic::amdgcn_ds_fmax;
11101 llvm_unreachable(
"Unknown BuiltinID");
11103 Value *F = CGM.getIntrinsic(ID);
11104 return Builder.CreateCall(F, Args);
11108 case AMDGPU::BI__builtin_amdgcn_workitem_id_x:
11110 case AMDGPU::BI__builtin_amdgcn_workitem_id_y:
11112 case AMDGPU::BI__builtin_amdgcn_workitem_id_z:
11116 case AMDGPU::BI__builtin_r600_recipsqrt_ieee:
11117 case AMDGPU::BI__builtin_r600_recipsqrt_ieeef:
11119 case AMDGPU::BI__builtin_r600_read_tidig_x:
11121 case AMDGPU::BI__builtin_r600_read_tidig_y:
11123 case AMDGPU::BI__builtin_r600_read_tidig_z:
11134 unsigned IntrinsicID,
11138 for (
unsigned I = 0; I < NumArgs; ++I)
11145 return CGF.
Builder.CreateExtractValue(Call, 0);
11150 switch (BuiltinID) {
11151 case SystemZ::BI__builtin_tbegin: {
11153 Value *Control = llvm::ConstantInt::get(Int32Ty, 0xff0c);
11154 Value *F = CGM.getIntrinsic(Intrinsic::s390_tbegin);
11155 return Builder.CreateCall(F, {TDB, Control});
11157 case SystemZ::BI__builtin_tbegin_nofloat: {
11159 Value *Control = llvm::ConstantInt::get(Int32Ty, 0xff0c);
11160 Value *F = CGM.getIntrinsic(Intrinsic::s390_tbegin_nofloat);
11161 return Builder.CreateCall(F, {TDB, Control});
11163 case SystemZ::BI__builtin_tbeginc: {
11164 Value *TDB = llvm::ConstantPointerNull::get(Int8PtrTy);
11165 Value *Control = llvm::ConstantInt::get(Int32Ty, 0xff08);
11166 Value *F = CGM.getIntrinsic(Intrinsic::s390_tbeginc);
11167 return Builder.CreateCall(F, {TDB, Control});
11169 case SystemZ::BI__builtin_tabort: {
11171 Value *F = CGM.getIntrinsic(Intrinsic::s390_tabort);
11172 return Builder.CreateCall(F, Builder.CreateSExt(Data, Int64Ty,
"tabort"));
11174 case SystemZ::BI__builtin_non_tx_store: {
11177 Value *F = CGM.getIntrinsic(Intrinsic::s390_ntstg);
11178 return Builder.CreateCall(F, {Data, Address});
11186 case SystemZ::BI__builtin_s390_vpopctb:
11187 case SystemZ::BI__builtin_s390_vpopcth:
11188 case SystemZ::BI__builtin_s390_vpopctf:
11189 case SystemZ::BI__builtin_s390_vpopctg: {
11192 Function *F = CGM.getIntrinsic(Intrinsic::ctpop, ResultType);
11193 return Builder.CreateCall(F, X);
11196 case SystemZ::BI__builtin_s390_vclzb:
11197 case SystemZ::BI__builtin_s390_vclzh:
11198 case SystemZ::BI__builtin_s390_vclzf:
11199 case SystemZ::BI__builtin_s390_vclzg: {
11202 Value *Undef = ConstantInt::get(Builder.getInt1Ty(),
false);
11203 Function *F = CGM.getIntrinsic(Intrinsic::ctlz, ResultType);
11204 return Builder.CreateCall(F, {
X, Undef});
11207 case SystemZ::BI__builtin_s390_vctzb:
11208 case SystemZ::BI__builtin_s390_vctzh:
11209 case SystemZ::BI__builtin_s390_vctzf:
11210 case SystemZ::BI__builtin_s390_vctzg: {
11213 Value *Undef = ConstantInt::get(Builder.getInt1Ty(),
false);
11214 Function *F = CGM.getIntrinsic(Intrinsic::cttz, ResultType);
11215 return Builder.CreateCall(F, {
X, Undef});
11218 case SystemZ::BI__builtin_s390_vfsqsb:
11219 case SystemZ::BI__builtin_s390_vfsqdb: {
11223 return Builder.CreateCall(F, X);
11225 case SystemZ::BI__builtin_s390_vfmasb:
11226 case SystemZ::BI__builtin_s390_vfmadb: {
11232 return Builder.CreateCall(F, {
X, Y, Z});
11234 case SystemZ::BI__builtin_s390_vfmssb:
11235 case SystemZ::BI__builtin_s390_vfmsdb: {
11240 Value *Zero = llvm::ConstantFP::getZeroValueForNegation(ResultType);
11242 return Builder.CreateCall(F, {
X, Y, Builder.CreateFSub(Zero, Z,
"sub")});
11244 case SystemZ::BI__builtin_s390_vfnmasb:
11245 case SystemZ::BI__builtin_s390_vfnmadb: {
11250 Value *Zero = llvm::ConstantFP::getZeroValueForNegation(ResultType);
11252 return Builder.CreateFSub(Zero, Builder.CreateCall(F, {X, Y, Z}),
"sub");
11254 case SystemZ::BI__builtin_s390_vfnmssb:
11255 case SystemZ::BI__builtin_s390_vfnmsdb: {
11260 Value *Zero = llvm::ConstantFP::getZeroValueForNegation(ResultType);
11262 Value *NegZ = Builder.CreateFSub(Zero, Z,
"sub");
11263 return Builder.CreateFSub(Zero, Builder.CreateCall(F, {X, Y, NegZ}));
11265 case SystemZ::BI__builtin_s390_vflpsb:
11266 case SystemZ::BI__builtin_s390_vflpdb: {
11270 return Builder.CreateCall(F, X);
11272 case SystemZ::BI__builtin_s390_vflnsb:
11273 case SystemZ::BI__builtin_s390_vflndb: {
11276 Value *Zero = llvm::ConstantFP::getZeroValueForNegation(ResultType);
11278 return Builder.CreateFSub(Zero, Builder.CreateCall(F, X),
"sub");
11280 case SystemZ::BI__builtin_s390_vfisb:
11281 case SystemZ::BI__builtin_s390_vfidb: {
11285 llvm::APSInt M4, M5;
11288 assert(IsConstM4 && IsConstM5 &&
"Constant arg isn't actually constant?");
11289 (void)IsConstM4; (void)IsConstM5;
11293 switch (M4.getZExtValue()) {
11296 switch (M5.getZExtValue()) {
11302 switch (M5.getZExtValue()) {
11312 if (ID != Intrinsic::not_intrinsic) {
11313 Function *F = CGM.getIntrinsic(ID, ResultType);
11314 return Builder.CreateCall(F, X);
11316 switch (BuiltinID) {
11317 case SystemZ::BI__builtin_s390_vfisb: ID = Intrinsic::s390_vfisb;
break;
11318 case SystemZ::BI__builtin_s390_vfidb: ID = Intrinsic::s390_vfidb;
break;
11319 default: llvm_unreachable(
"Unknown BuiltinID");
11321 Function *F = CGM.getIntrinsic(ID);
11322 Value *M4Value = llvm::ConstantInt::get(getLLVMContext(), M4);
11323 Value *M5Value = llvm::ConstantInt::get(getLLVMContext(), M5);
11324 return Builder.CreateCall(F, {
X, M4Value, M5Value});
11326 case SystemZ::BI__builtin_s390_vfmaxsb:
11327 case SystemZ::BI__builtin_s390_vfmaxdb: {
11334 assert(IsConstM4 &&
"Constant arg isn't actually constant?");
11339 switch (M4.getZExtValue()) {
11341 case 4: ID = Intrinsic::maxnum;
break;
11343 if (ID != Intrinsic::not_intrinsic) {
11344 Function *F = CGM.getIntrinsic(ID, ResultType);
11345 return Builder.CreateCall(F, {
X, Y});
11347 switch (BuiltinID) {
11348 case SystemZ::BI__builtin_s390_vfmaxsb: ID = Intrinsic::s390_vfmaxsb;
break;
11349 case SystemZ::BI__builtin_s390_vfmaxdb: ID = Intrinsic::s390_vfmaxdb;
break;
11350 default: llvm_unreachable(
"Unknown BuiltinID");
11352 Function *F = CGM.getIntrinsic(ID);
11353 Value *M4Value = llvm::ConstantInt::get(getLLVMContext(), M4);
11354 return Builder.CreateCall(F, {
X, Y, M4Value});
11356 case SystemZ::BI__builtin_s390_vfminsb:
11357 case SystemZ::BI__builtin_s390_vfmindb: {
11364 assert(IsConstM4 &&
"Constant arg isn't actually constant?");
11369 switch (M4.getZExtValue()) {
11371 case 4: ID = Intrinsic::minnum;
break;
11373 if (ID != Intrinsic::not_intrinsic) {
11374 Function *F = CGM.getIntrinsic(ID, ResultType);
11375 return Builder.CreateCall(F, {
X, Y});
11377 switch (BuiltinID) {
11378 case SystemZ::BI__builtin_s390_vfminsb: ID = Intrinsic::s390_vfminsb;
break;
11379 case SystemZ::BI__builtin_s390_vfmindb: ID = Intrinsic::s390_vfmindb;
break;
11380 default: llvm_unreachable(
"Unknown BuiltinID");
11382 Function *F = CGM.getIntrinsic(ID);
11383 Value *M4Value = llvm::ConstantInt::get(getLLVMContext(), M4);
11384 return Builder.CreateCall(F, {
X, Y, M4Value});
11389 #define INTRINSIC_WITH_CC(NAME) \ 11390 case SystemZ::BI__builtin_##NAME: \ 11391 return EmitSystemZIntrinsicWithCC(*this, Intrinsic::NAME, E) 11462 #undef INTRINSIC_WITH_CC 11471 auto MakeLdg = [&](
unsigned IntrinsicID) {
11475 return Builder.CreateCall(
11476 CGM.getIntrinsic(IntrinsicID, {Ptr->getType()->getPointerElementType(),
11478 {Ptr, ConstantInt::get(Builder.getInt32Ty(), Align.getQuantity())});
11480 auto MakeScopedAtomic = [&](
unsigned IntrinsicID) {
11482 return Builder.CreateCall(
11483 CGM.getIntrinsic(IntrinsicID, {Ptr->getType()->getPointerElementType(),
11485 {Ptr, EmitScalarExpr(E->
getArg(1))});
11487 switch (BuiltinID) {
11488 case NVPTX::BI__nvvm_atom_add_gen_i:
11489 case NVPTX::BI__nvvm_atom_add_gen_l:
11490 case NVPTX::BI__nvvm_atom_add_gen_ll:
11493 case NVPTX::BI__nvvm_atom_sub_gen_i:
11494 case NVPTX::BI__nvvm_atom_sub_gen_l:
11495 case NVPTX::BI__nvvm_atom_sub_gen_ll:
11498 case NVPTX::BI__nvvm_atom_and_gen_i:
11499 case NVPTX::BI__nvvm_atom_and_gen_l:
11500 case NVPTX::BI__nvvm_atom_and_gen_ll:
11503 case NVPTX::BI__nvvm_atom_or_gen_i:
11504 case NVPTX::BI__nvvm_atom_or_gen_l:
11505 case NVPTX::BI__nvvm_atom_or_gen_ll:
11508 case NVPTX::BI__nvvm_atom_xor_gen_i:
11509 case NVPTX::BI__nvvm_atom_xor_gen_l:
11510 case NVPTX::BI__nvvm_atom_xor_gen_ll:
11513 case NVPTX::BI__nvvm_atom_xchg_gen_i:
11514 case NVPTX::BI__nvvm_atom_xchg_gen_l:
11515 case NVPTX::BI__nvvm_atom_xchg_gen_ll:
11518 case NVPTX::BI__nvvm_atom_max_gen_i:
11519 case NVPTX::BI__nvvm_atom_max_gen_l:
11520 case NVPTX::BI__nvvm_atom_max_gen_ll:
11523 case NVPTX::BI__nvvm_atom_max_gen_ui:
11524 case NVPTX::BI__nvvm_atom_max_gen_ul:
11525 case NVPTX::BI__nvvm_atom_max_gen_ull:
11528 case NVPTX::BI__nvvm_atom_min_gen_i:
11529 case NVPTX::BI__nvvm_atom_min_gen_l:
11530 case NVPTX::BI__nvvm_atom_min_gen_ll:
11533 case NVPTX::BI__nvvm_atom_min_gen_ui:
11534 case NVPTX::BI__nvvm_atom_min_gen_ul:
11535 case NVPTX::BI__nvvm_atom_min_gen_ull:
11538 case NVPTX::BI__nvvm_atom_cas_gen_i:
11539 case NVPTX::BI__nvvm_atom_cas_gen_l:
11540 case NVPTX::BI__nvvm_atom_cas_gen_ll:
11545 case NVPTX::BI__nvvm_atom_add_gen_f: {
11551 CGM.getIntrinsic(Intrinsic::nvvm_atomic_load_add_f32, Ptr->getType());
11552 return Builder.CreateCall(FnALAF32, {Ptr, Val});
11555 case NVPTX::BI__nvvm_atom_add_gen_d: {
11561 CGM.getIntrinsic(Intrinsic::nvvm_atomic_load_add_f64, Ptr->getType());
11562 return Builder.CreateCall(FnALAF64, {Ptr, Val});
11565 case NVPTX::BI__nvvm_atom_inc_gen_ui: {
11569 CGM.getIntrinsic(Intrinsic::nvvm_atomic_load_inc_32, Ptr->getType());
11570 return Builder.CreateCall(FnALI32, {Ptr, Val});
11573 case NVPTX::BI__nvvm_atom_dec_gen_ui: {
11577 CGM.getIntrinsic(Intrinsic::nvvm_atomic_load_dec_32, Ptr->getType());
11578 return Builder.CreateCall(FnALD32, {Ptr, Val});
11581 case NVPTX::BI__nvvm_ldg_c:
11582 case NVPTX::BI__nvvm_ldg_c2:
11583 case NVPTX::BI__nvvm_ldg_c4:
11584 case NVPTX::BI__nvvm_ldg_s:
11585 case NVPTX::BI__nvvm_ldg_s2:
11586 case NVPTX::BI__nvvm_ldg_s4:
11587 case NVPTX::BI__nvvm_ldg_i:
11588 case NVPTX::BI__nvvm_ldg_i2:
11589 case NVPTX::BI__nvvm_ldg_i4:
11590 case NVPTX::BI__nvvm_ldg_l:
11591 case NVPTX::BI__nvvm_ldg_ll:
11592 case NVPTX::BI__nvvm_ldg_ll2:
11593 case NVPTX::BI__nvvm_ldg_uc:
11594 case NVPTX::BI__nvvm_ldg_uc2:
11595 case NVPTX::BI__nvvm_ldg_uc4:
11596 case NVPTX::BI__nvvm_ldg_us:
11597 case NVPTX::BI__nvvm_ldg_us2:
11598 case NVPTX::BI__nvvm_ldg_us4:
11599 case NVPTX::BI__nvvm_ldg_ui:
11600 case NVPTX::BI__nvvm_ldg_ui2:
11601 case NVPTX::BI__nvvm_ldg_ui4:
11602 case NVPTX::BI__nvvm_ldg_ul:
11603 case NVPTX::BI__nvvm_ldg_ull:
11604 case NVPTX::BI__nvvm_ldg_ull2:
11608 return MakeLdg(Intrinsic::nvvm_ldg_global_i);
11609 case NVPTX::BI__nvvm_ldg_f:
11610 case NVPTX::BI__nvvm_ldg_f2:
11611 case NVPTX::BI__nvvm_ldg_f4:
11612 case NVPTX::BI__nvvm_ldg_d:
11613 case NVPTX::BI__nvvm_ldg_d2:
11614 return MakeLdg(Intrinsic::nvvm_ldg_global_f);
11616 case NVPTX::BI__nvvm_atom_cta_add_gen_i:
11617 case NVPTX::BI__nvvm_atom_cta_add_gen_l:
11618 case NVPTX::BI__nvvm_atom_cta_add_gen_ll:
11619 return MakeScopedAtomic(Intrinsic::nvvm_atomic_add_gen_i_cta);
11620 case NVPTX::BI__nvvm_atom_sys_add_gen_i:
11621 case NVPTX::BI__nvvm_atom_sys_add_gen_l:
11622 case NVPTX::BI__nvvm_atom_sys_add_gen_ll:
11623 return MakeScopedAtomic(Intrinsic::nvvm_atomic_add_gen_i_sys);
11624 case NVPTX::BI__nvvm_atom_cta_add_gen_f:
11625 case NVPTX::BI__nvvm_atom_cta_add_gen_d:
11626 return MakeScopedAtomic(Intrinsic::nvvm_atomic_add_gen_f_cta);
11627 case NVPTX::BI__nvvm_atom_sys_add_gen_f:
11628 case NVPTX::BI__nvvm_atom_sys_add_gen_d:
11629 return MakeScopedAtomic(Intrinsic::nvvm_atomic_add_gen_f_sys);
11630 case NVPTX::BI__nvvm_atom_cta_xchg_gen_i:
11631 case NVPTX::BI__nvvm_atom_cta_xchg_gen_l:
11632 case NVPTX::BI__nvvm_atom_cta_xchg_gen_ll:
11633 return MakeScopedAtomic(Intrinsic::nvvm_atomic_exch_gen_i_cta);
11634 case NVPTX::BI__nvvm_atom_sys_xchg_gen_i:
11635 case NVPTX::BI__nvvm_atom_sys_xchg_gen_l:
11636 case NVPTX::BI__nvvm_atom_sys_xchg_gen_ll:
11637 return MakeScopedAtomic(Intrinsic::nvvm_atomic_exch_gen_i_sys);
11638 case NVPTX::BI__nvvm_atom_cta_max_gen_i:
11639 case NVPTX::BI__nvvm_atom_cta_max_gen_ui:
11640 case NVPTX::BI__nvvm_atom_cta_max_gen_l:
11641 case NVPTX::BI__nvvm_atom_cta_max_gen_ul:
11642 case NVPTX::BI__nvvm_atom_cta_max_gen_ll:
11643 case NVPTX::BI__nvvm_atom_cta_max_gen_ull:
11644 return MakeScopedAtomic(Intrinsic::nvvm_atomic_max_gen_i_cta);
11645 case NVPTX::BI__nvvm_atom_sys_max_gen_i:
11646 case NVPTX::BI__nvvm_atom_sys_max_gen_ui:
11647 case NVPTX::BI__nvvm_atom_sys_max_gen_l:
11648 case NVPTX::BI__nvvm_atom_sys_max_gen_ul:
11649 case NVPTX::BI__nvvm_atom_sys_max_gen_ll:
11650 case NVPTX::BI__nvvm_atom_sys_max_gen_ull:
11651 return MakeScopedAtomic(Intrinsic::nvvm_atomic_max_gen_i_sys);
11652 case NVPTX::BI__nvvm_atom_cta_min_gen_i:
11653 case NVPTX::BI__nvvm_atom_cta_min_gen_ui:
11654 case NVPTX::BI__nvvm_atom_cta_min_gen_l:
11655 case NVPTX::BI__nvvm_atom_cta_min_gen_ul:
11656 case NVPTX::BI__nvvm_atom_cta_min_gen_ll:
11657 case NVPTX::BI__nvvm_atom_cta_min_gen_ull:
11658 return MakeScopedAtomic(Intrinsic::nvvm_atomic_min_gen_i_cta);
11659 case NVPTX::BI__nvvm_atom_sys_min_gen_i:
11660 case NVPTX::BI__nvvm_atom_sys_min_gen_ui:
11661 case NVPTX::BI__nvvm_atom_sys_min_gen_l:
11662 case NVPTX::BI__nvvm_atom_sys_min_gen_ul:
11663 case NVPTX::BI__nvvm_atom_sys_min_gen_ll:
11664 case NVPTX::BI__nvvm_atom_sys_min_gen_ull:
11665 return MakeScopedAtomic(Intrinsic::nvvm_atomic_min_gen_i_sys);
11666 case NVPTX::BI__nvvm_atom_cta_inc_gen_ui:
11667 return MakeScopedAtomic(Intrinsic::nvvm_atomic_inc_gen_i_cta);
11668 case NVPTX::BI__nvvm_atom_cta_dec_gen_ui:
11669 return MakeScopedAtomic(Intrinsic::nvvm_atomic_dec_gen_i_cta);
11670 case NVPTX::BI__nvvm_atom_sys_inc_gen_ui:
11671 return MakeScopedAtomic(Intrinsic::nvvm_atomic_inc_gen_i_sys);
11672 case NVPTX::BI__nvvm_atom_sys_dec_gen_ui:
11673 return MakeScopedAtomic(Intrinsic::nvvm_atomic_dec_gen_i_sys);
11674 case NVPTX::BI__nvvm_atom_cta_and_gen_i:
11675 case NVPTX::BI__nvvm_atom_cta_and_gen_l:
11676 case NVPTX::BI__nvvm_atom_cta_and_gen_ll:
11677 return MakeScopedAtomic(Intrinsic::nvvm_atomic_and_gen_i_cta);
11678 case NVPTX::BI__nvvm_atom_sys_and_gen_i:
11679 case NVPTX::BI__nvvm_atom_sys_and_gen_l:
11680 case NVPTX::BI__nvvm_atom_sys_and_gen_ll:
11681 return MakeScopedAtomic(Intrinsic::nvvm_atomic_and_gen_i_sys);
11682 case NVPTX::BI__nvvm_atom_cta_or_gen_i:
11683 case NVPTX::BI__nvvm_atom_cta_or_gen_l:
11684 case NVPTX::BI__nvvm_atom_cta_or_gen_ll:
11685 return MakeScopedAtomic(Intrinsic::nvvm_atomic_or_gen_i_cta);
11686 case NVPTX::BI__nvvm_atom_sys_or_gen_i:
11687 case NVPTX::BI__nvvm_atom_sys_or_gen_l:
11688 case NVPTX::BI__nvvm_atom_sys_or_gen_ll:
11689 return MakeScopedAtomic(Intrinsic::nvvm_atomic_or_gen_i_sys);
11690 case NVPTX::BI__nvvm_atom_cta_xor_gen_i:
11691 case NVPTX::BI__nvvm_atom_cta_xor_gen_l:
11692 case NVPTX::BI__nvvm_atom_cta_xor_gen_ll:
11693 return MakeScopedAtomic(Intrinsic::nvvm_atomic_xor_gen_i_cta);
11694 case NVPTX::BI__nvvm_atom_sys_xor_gen_i:
11695 case NVPTX::BI__nvvm_atom_sys_xor_gen_l:
11696 case NVPTX::BI__nvvm_atom_sys_xor_gen_ll:
11697 return MakeScopedAtomic(Intrinsic::nvvm_atomic_xor_gen_i_sys);
11698 case NVPTX::BI__nvvm_atom_cta_cas_gen_i:
11699 case NVPTX::BI__nvvm_atom_cta_cas_gen_l:
11700 case NVPTX::BI__nvvm_atom_cta_cas_gen_ll: {
11702 return Builder.CreateCall(
11704 Intrinsic::nvvm_atomic_cas_gen_i_cta,
11705 {Ptr->getType()->getPointerElementType(), Ptr->getType()}),
11706 {Ptr, EmitScalarExpr(E->
getArg(1)), EmitScalarExpr(E->
getArg(2))});
11708 case NVPTX::BI__nvvm_atom_sys_cas_gen_i:
11709 case NVPTX::BI__nvvm_atom_sys_cas_gen_l:
11710 case NVPTX::BI__nvvm_atom_sys_cas_gen_ll: {
11712 return Builder.CreateCall(
11714 Intrinsic::nvvm_atomic_cas_gen_i_sys,
11715 {Ptr->getType()->getPointerElementType(), Ptr->getType()}),
11716 {Ptr, EmitScalarExpr(E->
getArg(1)), EmitScalarExpr(E->
getArg(2))});
11718 case NVPTX::BI__nvvm_match_all_sync_i32p:
11719 case NVPTX::BI__nvvm_match_all_sync_i64p: {
11722 Address PredOutPtr = EmitPointerWithAlignment(E->
getArg(2));
11723 Value *ResultPair = Builder.CreateCall(
11724 CGM.getIntrinsic(BuiltinID == NVPTX::BI__nvvm_match_all_sync_i32p
11725 ? Intrinsic::nvvm_match_all_sync_i32p
11726 : Intrinsic::nvvm_match_all_sync_i64p),
11728 Value *Pred = Builder.CreateZExt(Builder.CreateExtractValue(ResultPair, 1),
11729 PredOutPtr.getElementType());
11730 Builder.CreateStore(Pred, PredOutPtr);
11731 return Builder.CreateExtractValue(ResultPair, 0);
11733 case NVPTX::BI__hmma_m16n16k16_ld_a:
11734 case NVPTX::BI__hmma_m16n16k16_ld_b:
11735 case NVPTX::BI__hmma_m16n16k16_ld_c_f16:
11736 case NVPTX::BI__hmma_m16n16k16_ld_c_f32:
11737 case NVPTX::BI__hmma_m32n8k16_ld_a:
11738 case NVPTX::BI__hmma_m32n8k16_ld_b:
11739 case NVPTX::BI__hmma_m32n8k16_ld_c_f16:
11740 case NVPTX::BI__hmma_m32n8k16_ld_c_f32:
11741 case NVPTX::BI__hmma_m8n32k16_ld_a:
11742 case NVPTX::BI__hmma_m8n32k16_ld_b:
11743 case NVPTX::BI__hmma_m8n32k16_ld_c_f16:
11744 case NVPTX::BI__hmma_m8n32k16_ld_c_f32: {
11748 llvm::APSInt isColMajorArg;
11751 bool isColMajor = isColMajorArg.getSExtValue();
11753 unsigned NumResults;
11754 switch (BuiltinID) {
11755 case NVPTX::BI__hmma_m16n16k16_ld_a:
11756 IID = isColMajor ? Intrinsic::nvvm_wmma_m16n16k16_load_a_f16_col_stride
11757 : Intrinsic::nvvm_wmma_m16n16k16_load_a_f16_row_stride;
11760 case NVPTX::BI__hmma_m16n16k16_ld_b:
11761 IID = isColMajor ? Intrinsic::nvvm_wmma_m16n16k16_load_b_f16_col_stride
11762 : Intrinsic::nvvm_wmma_m16n16k16_load_b_f16_row_stride;
11765 case NVPTX::BI__hmma_m16n16k16_ld_c_f16:
11766 IID = isColMajor ? Intrinsic::nvvm_wmma_m16n16k16_load_c_f16_col_stride
11767 : Intrinsic::nvvm_wmma_m16n16k16_load_c_f16_row_stride;
11770 case NVPTX::BI__hmma_m16n16k16_ld_c_f32:
11771 IID = isColMajor ? Intrinsic::nvvm_wmma_m16n16k16_load_c_f32_col_stride
11772 : Intrinsic::nvvm_wmma_m16n16k16_load_c_f32_row_stride;
11775 case NVPTX::BI__hmma_m32n8k16_ld_a:
11776 IID = isColMajor ? Intrinsic::nvvm_wmma_m32n8k16_load_a_f16_col_stride
11777 : Intrinsic::nvvm_wmma_m32n8k16_load_a_f16_row_stride;
11780 case NVPTX::BI__hmma_m32n8k16_ld_b:
11781 IID = isColMajor ? Intrinsic::nvvm_wmma_m32n8k16_load_b_f16_col_stride
11782 : Intrinsic::nvvm_wmma_m32n8k16_load_b_f16_row_stride;
11785 case NVPTX::BI__hmma_m32n8k16_ld_c_f16:
11786 IID = isColMajor ? Intrinsic::nvvm_wmma_m32n8k16_load_c_f16_col_stride
11787 : Intrinsic::nvvm_wmma_m32n8k16_load_c_f16_row_stride;
11790 case NVPTX::BI__hmma_m32n8k16_ld_c_f32:
11791 IID = isColMajor ? Intrinsic::nvvm_wmma_m32n8k16_load_c_f32_col_stride
11792 : Intrinsic::nvvm_wmma_m32n8k16_load_c_f32_row_stride;
11795 case NVPTX::BI__hmma_m8n32k16_ld_a:
11796 IID = isColMajor ? Intrinsic::nvvm_wmma_m8n32k16_load_a_f16_col_stride
11797 : Intrinsic::nvvm_wmma_m8n32k16_load_a_f16_row_stride;
11800 case NVPTX::BI__hmma_m8n32k16_ld_b:
11801 IID = isColMajor ? Intrinsic::nvvm_wmma_m8n32k16_load_b_f16_col_stride
11802 : Intrinsic::nvvm_wmma_m8n32k16_load_b_f16_row_stride;
11805 case NVPTX::BI__hmma_m8n32k16_ld_c_f16:
11806 IID = isColMajor ? Intrinsic::nvvm_wmma_m8n32k16_load_c_f16_col_stride
11807 : Intrinsic::nvvm_wmma_m8n32k16_load_c_f16_row_stride;
11810 case NVPTX::BI__hmma_m8n32k16_ld_c_f32:
11811 IID = isColMajor ? Intrinsic::nvvm_wmma_m8n32k16_load_c_f32_col_stride
11812 : Intrinsic::nvvm_wmma_m8n32k16_load_c_f32_row_stride;
11816 llvm_unreachable(
"Unexpected builtin ID.");
11819 Builder.CreateCall(CGM.getIntrinsic(IID, Src->getType()), {Src, Ldm});
11822 for (
unsigned i = 0; i < NumResults; ++i) {
11823 Builder.CreateAlignedStore(
11824 Builder.CreateBitCast(Builder.CreateExtractValue(Result, i),
11826 Builder.CreateGEP(Dst.
getPointer(), llvm::ConstantInt::get(IntTy, i)),
11832 case NVPTX::BI__hmma_m16n16k16_st_c_f16:
11833 case NVPTX::BI__hmma_m16n16k16_st_c_f32:
11834 case NVPTX::BI__hmma_m32n8k16_st_c_f16:
11835 case NVPTX::BI__hmma_m32n8k16_st_c_f32:
11836 case NVPTX::BI__hmma_m8n32k16_st_c_f16:
11837 case NVPTX::BI__hmma_m8n32k16_st_c_f32: {
11841 llvm::APSInt isColMajorArg;
11844 bool isColMajor = isColMajorArg.getSExtValue();
11846 unsigned NumResults = 8;
11849 switch (BuiltinID) {
11850 case NVPTX::BI__hmma_m16n16k16_st_c_f16:
11851 IID = isColMajor ? Intrinsic::nvvm_wmma_m16n16k16_store_d_f16_col_stride
11852 : Intrinsic::nvvm_wmma_m16n16k16_store_d_f16_row_stride;
11855 case NVPTX::BI__hmma_m16n16k16_st_c_f32:
11856 IID = isColMajor ? Intrinsic::nvvm_wmma_m16n16k16_store_d_f32_col_stride
11857 : Intrinsic::nvvm_wmma_m16n16k16_store_d_f32_row_stride;
11859 case NVPTX::BI__hmma_m32n8k16_st_c_f16:
11860 IID = isColMajor ? Intrinsic::nvvm_wmma_m32n8k16_store_d_f16_col_stride
11861 : Intrinsic::nvvm_wmma_m32n8k16_store_d_f16_row_stride;
11864 case NVPTX::BI__hmma_m32n8k16_st_c_f32:
11865 IID = isColMajor ? Intrinsic::nvvm_wmma_m32n8k16_store_d_f32_col_stride
11866 : Intrinsic::nvvm_wmma_m32n8k16_store_d_f32_row_stride;
11868 case NVPTX::BI__hmma_m8n32k16_st_c_f16:
11869 IID = isColMajor ? Intrinsic::nvvm_wmma_m8n32k16_store_d_f16_col_stride
11870 : Intrinsic::nvvm_wmma_m8n32k16_store_d_f16_row_stride;
11873 case NVPTX::BI__hmma_m8n32k16_st_c_f32:
11874 IID = isColMajor ? Intrinsic::nvvm_wmma_m8n32k16_store_d_f32_col_stride
11875 : Intrinsic::nvvm_wmma_m8n32k16_store_d_f32_row_stride;
11878 llvm_unreachable(
"Unexpected builtin ID.");
11880 Function *Intrinsic = CGM.getIntrinsic(IID, Dst->getType());
11881 llvm::Type *ParamType = Intrinsic->getFunctionType()->getParamType(1);
11883 for (
unsigned i = 0; i < NumResults; ++i) {
11884 Value *V = Builder.CreateAlignedLoad(
11885 Builder.CreateGEP(Src.getPointer(), llvm::ConstantInt::get(IntTy, i)),
11887 Values.push_back(Builder.CreateBitCast(V, ParamType));
11889 Values.push_back(Ldm);
11890 Value *Result = Builder.CreateCall(Intrinsic, Values);
11896 case NVPTX::BI__hmma_m16n16k16_mma_f16f16:
11897 case NVPTX::BI__hmma_m16n16k16_mma_f32f16:
11898 case NVPTX::BI__hmma_m16n16k16_mma_f32f32:
11899 case NVPTX::BI__hmma_m16n16k16_mma_f16f32:
11900 case NVPTX::BI__hmma_m32n8k16_mma_f16f16:
11901 case NVPTX::BI__hmma_m32n8k16_mma_f32f16:
11902 case NVPTX::BI__hmma_m32n8k16_mma_f32f32:
11903 case NVPTX::BI__hmma_m32n8k16_mma_f16f32:
11904 case NVPTX::BI__hmma_m8n32k16_mma_f16f16:
11905 case NVPTX::BI__hmma_m8n32k16_mma_f32f16:
11906 case NVPTX::BI__hmma_m8n32k16_mma_f32f32:
11907 case NVPTX::BI__hmma_m8n32k16_mma_f16f32: {
11912 llvm::APSInt LayoutArg;
11915 int Layout = LayoutArg.getSExtValue();
11916 if (Layout < 0 || Layout > 3)
11918 llvm::APSInt SatfArg;
11921 bool Satf = SatfArg.getSExtValue();
11924 #define MMA_VARIANTS(geom, type) {{ \ 11925 Intrinsic::nvvm_wmma_##geom##_mma_row_row_##type, \ 11926 Intrinsic::nvvm_wmma_##geom##_mma_row_row_##type##_satfinite, \ 11927 Intrinsic::nvvm_wmma_##geom##_mma_row_col_##type, \ 11928 Intrinsic::nvvm_wmma_##geom##_mma_row_col_##type##_satfinite, \ 11929 Intrinsic::nvvm_wmma_##geom##_mma_col_row_##type, \ 11930 Intrinsic::nvvm_wmma_##geom##_mma_col_row_##type##_satfinite, \ 11931 Intrinsic::nvvm_wmma_##geom##_mma_col_col_##type, \ 11932 Intrinsic::nvvm_wmma_##geom##_mma_col_col_##type##_satfinite \ 11936 auto getMMAIntrinsic = [Layout, Satf](std::array<unsigned, 8> Variants) {
11937 unsigned Index = Layout * 2 + Satf;
11939 return Variants[Index];
11944 switch (BuiltinID) {
11945 case NVPTX::BI__hmma_m16n16k16_mma_f16f16:
11946 IID = getMMAIntrinsic(
MMA_VARIANTS(m16n16k16, f16_f16));
11950 case NVPTX::BI__hmma_m16n16k16_mma_f32f16:
11951 IID = getMMAIntrinsic(
MMA_VARIANTS(m16n16k16, f32_f16));
11955 case NVPTX::BI__hmma_m16n16k16_mma_f16f32:
11956 IID = getMMAIntrinsic(
MMA_VARIANTS(m16n16k16, f16_f32));
11960 case NVPTX::BI__hmma_m16n16k16_mma_f32f32:
11961 IID = getMMAIntrinsic(
MMA_VARIANTS(m16n16k16, f32_f32));
11965 case NVPTX::BI__hmma_m32n8k16_mma_f16f16:
11966 IID = getMMAIntrinsic(
MMA_VARIANTS(m32n8k16, f16_f16));
11970 case NVPTX::BI__hmma_m32n8k16_mma_f32f16:
11971 IID = getMMAIntrinsic(
MMA_VARIANTS(m32n8k16, f32_f16));
11975 case NVPTX::BI__hmma_m32n8k16_mma_f16f32:
11976 IID = getMMAIntrinsic(
MMA_VARIANTS(m32n8k16, f16_f32));
11980 case NVPTX::BI__hmma_m32n8k16_mma_f32f32:
11981 IID = getMMAIntrinsic(
MMA_VARIANTS(m32n8k16, f32_f32));
11985 case NVPTX::BI__hmma_m8n32k16_mma_f16f16:
11986 IID = getMMAIntrinsic(
MMA_VARIANTS(m8n32k16, f16_f16));
11990 case NVPTX::BI__hmma_m8n32k16_mma_f32f16:
11991 IID = getMMAIntrinsic(
MMA_VARIANTS(m8n32k16, f32_f16));
11995 case NVPTX::BI__hmma_m8n32k16_mma_f16f32:
11996 IID = getMMAIntrinsic(
MMA_VARIANTS(m8n32k16, f16_f32));
12000 case NVPTX::BI__hmma_m8n32k16_mma_f32f32:
12001 IID = getMMAIntrinsic(
MMA_VARIANTS(m8n32k16, f32_f32));
12006 llvm_unreachable(
"Unexpected builtin ID.");
12008 #undef MMA_VARIANTS 12011 Function *Intrinsic = CGM.getIntrinsic(IID);
12012 llvm::Type *ABType = Intrinsic->getFunctionType()->getParamType(0);
12014 for (
unsigned i = 0; i < 8; ++i) {
12015 Value *V = Builder.CreateAlignedLoad(
12016 Builder.CreateGEP(SrcA.getPointer(),
12017 llvm::ConstantInt::get(IntTy, i)),
12019 Values.push_back(Builder.CreateBitCast(V, ABType));
12022 for (
unsigned i = 0; i < 8; ++i) {
12023 Value *V = Builder.CreateAlignedLoad(
12024 Builder.CreateGEP(SrcB.getPointer(),
12025 llvm::ConstantInt::get(IntTy, i)),
12027 Values.push_back(Builder.CreateBitCast(V, ABType));
12030 llvm::Type *CType = Intrinsic->getFunctionType()->getParamType(16);
12031 for (
unsigned i = 0; i < NumEltsC; ++i) {
12032 Value *V = Builder.CreateAlignedLoad(
12033 Builder.CreateGEP(SrcC.getPointer(),
12034 llvm::ConstantInt::get(IntTy, i)),
12036 Values.push_back(Builder.CreateBitCast(V, CType));
12038 Value *Result = Builder.CreateCall(Intrinsic, Values);
12040 for (
unsigned i = 0; i < NumEltsD; ++i)
12041 Builder.CreateAlignedStore(
12042 Builder.CreateBitCast(Builder.CreateExtractValue(Result, i), DType),
12043 Builder.CreateGEP(Dst.
getPointer(), llvm::ConstantInt::get(IntTy, i)),
12054 switch (BuiltinID) {
12055 case WebAssembly::BI__builtin_wasm_memory_size: {
12058 Value *Callee = CGM.getIntrinsic(Intrinsic::wasm_memory_size, ResultType);
12059 return Builder.CreateCall(Callee, I);
12061 case WebAssembly::BI__builtin_wasm_memory_grow: {
12064 EmitScalarExpr(E->
getArg(0)),
12065 EmitScalarExpr(E->
getArg(1))
12067 Value *Callee = CGM.getIntrinsic(Intrinsic::wasm_memory_grow, ResultType);
12068 return Builder.CreateCall(Callee, Args);
12070 case WebAssembly::BI__builtin_wasm_mem_size: {
12073 Value *Callee = CGM.getIntrinsic(Intrinsic::wasm_mem_size, ResultType);
12074 return Builder.CreateCall(Callee, I);
12076 case WebAssembly::BI__builtin_wasm_mem_grow: {
12079 EmitScalarExpr(E->
getArg(0)),
12080 EmitScalarExpr(E->
getArg(1))
12082 Value *Callee = CGM.getIntrinsic(Intrinsic::wasm_mem_grow, ResultType);
12083 return Builder.CreateCall(Callee, Args);
12085 case WebAssembly::BI__builtin_wasm_current_memory: {
12087 Value *Callee = CGM.getIntrinsic(Intrinsic::wasm_current_memory, ResultType);
12088 return Builder.CreateCall(Callee);
12090 case WebAssembly::BI__builtin_wasm_grow_memory: {
12092 Value *Callee = CGM.getIntrinsic(Intrinsic::wasm_grow_memory, X->getType());
12093 return Builder.CreateCall(Callee, X);
12095 case WebAssembly::BI__builtin_wasm_throw: {
12098 Value *Callee = CGM.getIntrinsic(Intrinsic::wasm_throw);
12099 return Builder.CreateCall(Callee, {Tag, Obj});
12101 case WebAssembly::BI__builtin_wasm_rethrow: {
12102 Value *Callee = CGM.getIntrinsic(Intrinsic::wasm_rethrow);
12103 return Builder.CreateCall(Callee);
12116 auto MakeCircLd = [&](
unsigned IntID,
bool HasImm) {
12119 BP =
Address(Builder.CreateBitCast(BP.getPointer(), Int8PtrPtrTy),
12120 BP.getAlignment());
12125 EmitScalarExpr(E->
getArg(3)) };
12128 EmitScalarExpr(E->
getArg(2)) };
12130 llvm::Value *Result = Builder.CreateCall(CGM.getIntrinsic(IntID), Ops);
12131 llvm::Value *NewBase = Builder.CreateExtractValue(Result, 1);
12133 NewBase->getType()->getPointerTo());
12137 Builder.CreateAlignedStore(NewBase, LV, Dest.
getAlignment());
12138 return Builder.CreateExtractValue(Result, 0);
12141 auto MakeCircSt = [&](
unsigned IntID,
bool HasImm) {
12144 BP =
Address(Builder.CreateBitCast(BP.getPointer(), Int8PtrPtrTy),
12145 BP.getAlignment());
12150 EmitScalarExpr(E->
getArg(3)), EmitScalarExpr(E->
getArg(4)) };
12153 EmitScalarExpr(E->
getArg(2)), EmitScalarExpr(E->
getArg(3)) };
12155 llvm::Value *NewBase = Builder.CreateCall(CGM.getIntrinsic(IntID), Ops);
12157 NewBase->getType()->getPointerTo());
12161 return Builder.CreateAlignedStore(NewBase, LV, Dest.
getAlignment());
12167 auto MakeBrevLd = [&](
unsigned IntID,
llvm::Type *DestTy) {
12172 Builder.CreateBitCast(EmitScalarExpr(E->
getArg(0)), Int8PtrTy);
12178 DestAddr =
Address(Builder.CreateBitCast(DestAddr.getPointer(), Int8PtrTy),
12179 DestAddr.getAlignment());
12180 llvm::Value *DestAddress = DestAddr.getPointer();
12185 Ops = {BaseAddress, EmitScalarExpr(E->
getArg(2))};
12187 llvm::Value *Result = Builder.CreateCall(CGM.getIntrinsic(IntID), Ops);
12189 llvm::Value *DestVal = Builder.CreateExtractValue(Result, 0);
12194 DestVal = Builder.CreateTrunc(DestVal, DestTy);
12197 Builder.CreateBitCast(DestAddress, DestVal->getType()->getPointerTo());
12198 Builder.CreateAlignedStore(DestVal, DestForStore, DestAddr.getAlignment());
12200 return Builder.CreateExtractValue(Result, 1);
12203 switch (BuiltinID) {
12204 case Hexagon::BI__builtin_HEXAGON_V6_vaddcarry:
12205 case Hexagon::BI__builtin_HEXAGON_V6_vaddcarry_128B: {
12208 if (BuiltinID == Hexagon::BI__builtin_HEXAGON_V6_vaddcarry) {
12210 ID = Intrinsic::hexagon_V6_vaddcarry;
12213 ID = Intrinsic::hexagon_V6_vaddcarry_128B;
12215 Dest = Builder.CreateBitCast(Dest,
12216 llvm::VectorType::get(Builder.getInt1Ty(), Size)->getPointerTo(0));
12217 LoadInst *QLd = Builder.CreateLoad(Dest);
12218 Ops = { EmitScalarExpr(E->
getArg(0)), EmitScalarExpr(E->
getArg(1)), QLd };
12219 llvm::Value *Result = Builder.CreateCall(CGM.getIntrinsic(ID), Ops);
12220 llvm::Value *Vprd = Builder.CreateExtractValue(Result, 1);
12222 Vprd->getType()->getPointerTo(0));
12223 Builder.CreateAlignedStore(Vprd, Base, Dest.
getAlignment());
12224 return Builder.CreateExtractValue(Result, 0);
12226 case Hexagon::BI__builtin_HEXAGON_V6_vsubcarry:
12227 case Hexagon::BI__builtin_HEXAGON_V6_vsubcarry_128B: {
12230 if (BuiltinID == Hexagon::BI__builtin_HEXAGON_V6_vsubcarry) {
12232 ID = Intrinsic::hexagon_V6_vsubcarry;
12235 ID = Intrinsic::hexagon_V6_vsubcarry_128B;
12237 Dest = Builder.CreateBitCast(Dest,
12238 llvm::VectorType::get(Builder.getInt1Ty(), Size)->getPointerTo(0));
12239 LoadInst *QLd = Builder.CreateLoad(Dest);
12240 Ops = { EmitScalarExpr(E->
getArg(0)), EmitScalarExpr(E->
getArg(1)), QLd };
12241 llvm::Value *Result = Builder.CreateCall(CGM.getIntrinsic(ID), Ops);
12242 llvm::Value *Vprd = Builder.CreateExtractValue(Result, 1);
12244 Vprd->getType()->getPointerTo(0));
12245 Builder.CreateAlignedStore(Vprd, Base, Dest.
getAlignment());
12246 return Builder.CreateExtractValue(Result, 0);
12248 case Hexagon::BI__builtin_HEXAGON_L2_loadrub_pci:
12249 return MakeCircLd(Intrinsic::hexagon_L2_loadrub_pci,
true);
12250 case Hexagon::BI__builtin_HEXAGON_L2_loadrb_pci:
12251 return MakeCircLd(Intrinsic::hexagon_L2_loadrb_pci,
true);
12252 case Hexagon::BI__builtin_HEXAGON_L2_loadruh_pci:
12253 return MakeCircLd(Intrinsic::hexagon_L2_loadruh_pci,
true);
12254 case Hexagon::BI__builtin_HEXAGON_L2_loadrh_pci:
12255 return MakeCircLd(Intrinsic::hexagon_L2_loadrh_pci,
true);
12256 case Hexagon::BI__builtin_HEXAGON_L2_loadri_pci:
12257 return MakeCircLd(Intrinsic::hexagon_L2_loadri_pci,
true);
12258 case Hexagon::BI__builtin_HEXAGON_L2_loadrd_pci:
12259 return MakeCircLd(Intrinsic::hexagon_L2_loadrd_pci,
true);
12260 case Hexagon::BI__builtin_HEXAGON_L2_loadrub_pcr:
12261 return MakeCircLd(Intrinsic::hexagon_L2_loadrub_pcr,
false);
12262 case Hexagon::BI__builtin_HEXAGON_L2_loadrb_pcr:
12263 return MakeCircLd(Intrinsic::hexagon_L2_loadrb_pcr,
false);
12264 case Hexagon::BI__builtin_HEXAGON_L2_loadruh_pcr:
12265 return MakeCircLd(Intrinsic::hexagon_L2_loadruh_pcr,
false);
12266 case Hexagon::BI__builtin_HEXAGON_L2_loadrh_pcr:
12267 return MakeCircLd(Intrinsic::hexagon_L2_loadrh_pcr,
false);
12268 case Hexagon::BI__builtin_HEXAGON_L2_loadri_pcr:
12269 return MakeCircLd(Intrinsic::hexagon_L2_loadri_pcr,
false);
12270 case Hexagon::BI__builtin_HEXAGON_L2_loadrd_pcr:
12271 return MakeCircLd(Intrinsic::hexagon_L2_loadrd_pcr,
false);
12272 case Hexagon::BI__builtin_HEXAGON_S2_storerb_pci:
12273 return MakeCircSt(Intrinsic::hexagon_S2_storerb_pci,
true);
12274 case Hexagon::BI__builtin_HEXAGON_S2_storerh_pci:
12275 return MakeCircSt(Intrinsic::hexagon_S2_storerh_pci,
true);
12276 case Hexagon::BI__builtin_HEXAGON_S2_storerf_pci:
12277 return MakeCircSt(Intrinsic::hexagon_S2_storerf_pci,
true);
12278 case Hexagon::BI__builtin_HEXAGON_S2_storeri_pci:
12279 return MakeCircSt(Intrinsic::hexagon_S2_storeri_pci,
true);
12280 case Hexagon::BI__builtin_HEXAGON_S2_storerd_pci:
12281 return MakeCircSt(Intrinsic::hexagon_S2_storerd_pci,
true);
12282 case Hexagon::BI__builtin_HEXAGON_S2_storerb_pcr:
12283 return MakeCircSt(Intrinsic::hexagon_S2_storerb_pcr,
false);
12284 case Hexagon::BI__builtin_HEXAGON_S2_storerh_pcr:
12285 return MakeCircSt(Intrinsic::hexagon_S2_storerh_pcr,
false);
12286 case Hexagon::BI__builtin_HEXAGON_S2_storerf_pcr:
12287 return MakeCircSt(Intrinsic::hexagon_S2_storerf_pcr,
false);
12288 case Hexagon::BI__builtin_HEXAGON_S2_storeri_pcr:
12289 return MakeCircSt(Intrinsic::hexagon_S2_storeri_pcr,
false);
12290 case Hexagon::BI__builtin_HEXAGON_S2_storerd_pcr:
12291 return MakeCircSt(Intrinsic::hexagon_S2_storerd_pcr,
false);
12292 case Hexagon::BI__builtin_brev_ldub:
12293 return MakeBrevLd(Intrinsic::hexagon_L2_loadrub_pbr, Int8Ty);
12294 case Hexagon::BI__builtin_brev_ldb:
12295 return MakeBrevLd(Intrinsic::hexagon_L2_loadrb_pbr, Int8Ty);
12296 case Hexagon::BI__builtin_brev_lduh:
12297 return MakeBrevLd(Intrinsic::hexagon_L2_loadruh_pbr, Int16Ty);
12298 case Hexagon::BI__builtin_brev_ldh:
12299 return MakeBrevLd(Intrinsic::hexagon_L2_loadrh_pbr, Int16Ty);
12300 case Hexagon::BI__builtin_brev_ldw:
12301 return MakeBrevLd(Intrinsic::hexagon_L2_loadri_pbr, Int32Ty);
12302 case Hexagon::BI__builtin_brev_ldd:
12303 return MakeBrevLd(Intrinsic::hexagon_L2_loadrd_pbr, Int64Ty);
ReturnValueSlot - Contains the address where the return value of a function can be stored...
Defines the clang::ASTContext interface.
static Value * emitFPIntBuiltin(CodeGenFunction &CGF, const CallExpr *E, unsigned IntrinsicID)
Represents a function declaration or definition.
llvm::IntegerType * IntTy
int
Other implicit parameter.
llvm::Value * EmitAArch64CompareBuiltinExpr(llvm::Value *Op, llvm::Type *Ty, const llvm::CmpInst::Predicate Fp, const llvm::CmpInst::Predicate Ip, const llvm::Twine &Name="")
static Value * EmitX86FMAExpr(CodeGenFunction &CGF, ArrayRef< Value *> Ops, unsigned BuiltinID, bool IsAddSub)
bool EvaluateAsRValue(EvalResult &Result, const ASTContext &Ctx) const
EvaluateAsRValue - Return true if this is a constant which we can fold to an rvalue using any crazy t...
PointerType - C99 6.7.5.1 - Pointer Declarators.
static Value * EmitX86ExpandLoad(CodeGenFunction &CGF, ArrayRef< Value *> Ops)
A (possibly-)qualified type.
bool isBlockPointerType() const
#define fma(__x, __y, __z)
static WidthAndSignedness getIntegerWidthAndSignedness(const clang::ASTContext &context, const clang::QualType Type)
unsigned char getSummaryByte() const
Expr * getArg(unsigned Arg)
getArg - Return the specified argument.
static Value * emitBinaryBuiltin(CodeGenFunction &CGF, const CallExpr *E, unsigned IntrinsicID)
static struct WidthAndSignedness EncompassingIntegerType(ArrayRef< struct WidthAndSignedness > Types)
llvm::Value * EmitNVPTXBuiltinExpr(unsigned BuiltinID, const CallExpr *E)
llvm::LLVMContext & getLLVMContext()
DominatorTree GraphTraits specialization so the DominatorTree can be iterable by generic graph iterat...
const ASTRecordLayout & getASTRecordLayout(const RecordDecl *D) const
Get or compute information about the layout of the specified record (struct/union/class) D...
unsigned getNumArgs() const
getNumArgs - Return the number of actual arguments to this call.
QualType getPointeeType() const
If this is a pointer, ObjC object pointer, or block pointer, this returns the respective pointee...
const llvm::Triple & getTriple() const
Returns the target triple of the primary target.
static const Builtin::Info BuiltinInfo[]
static Value * EmitX86MinMax(CodeGenFunction &CGF, ICmpInst::Predicate Pred, ArrayRef< Value *> Ops)
Address EmitPointerWithAlignment(const Expr *Addr, LValueBaseInfo *BaseInfo=nullptr, TBAAAccessInfo *TBAAInfo=nullptr)
EmitPointerWithAlignment - Given an expression with a pointer type, emit the value and compute our be...
constexpr XRayInstrMask Typed
bool isRecordType() const
static Value * getMaskVecValue(CodeGenFunction &CGF, Value *Mask, unsigned NumElts)
static Value * EmitX86Ternlog(CodeGenFunction &CGF, bool ZeroMask, ArrayRef< Value *> Ops)
llvm::Value * EmitMSVCBuiltinExpr(MSVCIntrin BuiltinID, const CallExpr *E)
static Value * EmitToInt(CodeGenFunction &CGF, llvm::Value *V, QualType T, llvm::IntegerType *IntType)
Emit the conversions required to turn the given value into an integer of the given size...
static Value * EmitSpecialRegisterBuiltin(CodeGenFunction &CGF, const CallExpr *E, llvm::Type *RegisterType, llvm::Type *ValueType, bool IsRead, StringRef SysReg="")
The base class of the type hierarchy.
static Value * EmitX86Muldq(CodeGenFunction &CGF, bool IsSigned, ArrayRef< Value *> Ops)
#define NEONMAP1(NameBase, LLVMIntrinsic, TypeModifier)
llvm::IntegerType * Int8Ty
i8, i16, i32, and i64
static CanQualType getOSLogArgType(ASTContext &C, int Size)
Get the argument type for arguments to os_log_helper.
virtual llvm::Value * getPipeElemAlign(const Expr *PipeArg)
bool EvaluateAsInt(llvm::APSInt &Result, const ASTContext &Ctx, SideEffectsKind AllowSideEffects=SE_NoSideEffects) const
EvaluateAsInt - Return true if this is a constant which we can fold and convert to an integer...
RValue EmitCall(const CGFunctionInfo &CallInfo, const CGCallee &Callee, ReturnValueSlot ReturnValue, const CallArgList &Args, llvm::Instruction **callOrInvoke, SourceLocation Loc)
EmitCall - Generate a call of the given function, expecting the given result type, and using the given argument list which specifies both the LLVM arguments and the types they were derived from.
llvm::Value * EmitCheckedArgForBuiltin(const Expr *E, BuiltinCheckKind Kind)
Emits an argument for a call to a builtin.
Objects with "hidden" visibility are not seen by the dynamic linker.
static bool isSpecialMixedSignMultiply(unsigned BuiltinID, WidthAndSignedness Op1Info, WidthAndSignedness Op2Info, WidthAndSignedness ResultInfo)
Determine if a binop is a checked mixed-sign multiply we can specialize.
RAII object to set/unset CodeGenFunction::IsSanitizerScope.
static Value * EmitX86MaskedLoad(CodeGenFunction &CGF, ArrayRef< Value *> Ops, unsigned Align)
const T * getAs() const
Member-template getAs<specific type>'.
const void * Store
Store - This opaque type encapsulates an immutable mapping from locations to values.
static Value * EmitSystemZIntrinsicWithCC(CodeGenFunction &CGF, unsigned IntrinsicID, const CallExpr *E)
Handle a SystemZ function in which the final argument is a pointer to an int that receives the post-i...
static int64_t clamp(int64_t Value, int64_t Low, int64_t High)
llvm::Value * getPointer() const
Represents a parameter to a function.
static bool HasExtraNeonArgument(unsigned BuiltinID)
Return true if BuiltinID is an overloaded Neon intrinsic with an extra argument that specifies the ve...
The collection of all-type qualifiers we support.
void add(RValue rvalue, QualType type)
Expr * IgnoreImpCasts() LLVM_READONLY
IgnoreImpCasts - Skip past any implicit casts which might surround this expression.
Represents a struct/union/class.
void __ovld prefetch(const __global char *p, size_t num_elements)
Prefetch num_elements * sizeof(gentype) bytes into the global cache.
const TargetInfo & getTarget() const
One of these records is kept for each identifier that is lexed.
Address getAddress() const
CodeGenFunction - This class organizes the per-function state that is used while generating LLVM code...
Holds long-lived AST nodes (such as types and decls) that can be referred to throughout the semantic ...
llvm::IntegerType * Int64Ty
RecordDecl * getDefinition() const
Returns the RecordDecl that actually defines this struct/union/class.
field_range fields() const
llvm::Value * BuildVector(ArrayRef< llvm::Value *> Ops)
llvm::IntegerType * SizeTy
static Value * EmitX86CompressStore(CodeGenFunction &CGF, ArrayRef< Value *> Ops)
llvm::Value * EmitAArch64BuiltinExpr(unsigned BuiltinID, const CallExpr *E, llvm::Triple::ArchType Arch)
static Value * EmitScalarFMAExpr(CodeGenFunction &CGF, MutableArrayRef< Value *> Ops, Value *Upper, bool ZeroMask=false, unsigned PTIdx=0, bool NegAcc=false)
static bool NEONSIMDIntrinsicsProvenSorted
llvm::Value * EmitHexagonBuiltinExpr(unsigned BuiltinID, const CallExpr *E)
static Value * EmitAArch64TblBuiltinExpr(CodeGenFunction &CGF, unsigned BuiltinID, const CallExpr *E, SmallVectorImpl< Value *> &Ops, llvm::Triple::ArchType Arch)
static RValue EmitBinaryAtomicPost(CodeGenFunction &CGF, llvm::AtomicRMWInst::BinOp Kind, const CallExpr *E, Instruction::BinaryOps Op, bool Invert=false)
Utility to insert an atomic instruction based Instrinsic::ID and the expression node, where the return value is the result of the operation.
static llvm::Value * EmitX86BitTestIntrinsic(CodeGenFunction &CGF, BitTest BT, const CallExpr *E, Value *BitBase, Value *BitPos)
static const NeonIntrinsicInfo AArch64SIMDIntrinsicMap[]
void EmitStoreOfScalar(llvm::Value *Value, Address Addr, bool Volatile, QualType Ty, AlignmentSource Source=AlignmentSource::Type, bool isInit=false, bool isNontemporal=false)
EmitStoreOfScalar - Store a scalar value to an address, taking care to appropriately convert from the...
static ApplyDebugLocation CreateArtificial(CodeGenFunction &CGF)
Apply TemporaryLocation if it is valid.
static Value * MakeBinaryAtomicValue(CodeGenFunction &CGF, llvm::AtomicRMWInst::BinOp Kind, const CallExpr *E)
Utility to insert an atomic instruction based on Instrinsic::ID and the expression node...
llvm::Value * EmitCommonNeonBuiltinExpr(unsigned BuiltinID, unsigned LLVMIntrinsic, unsigned AltLLVMIntrinsic, const char *NameHint, unsigned Modifier, const CallExpr *E, SmallVectorImpl< llvm::Value *> &Ops, Address PtrOp0, Address PtrOp1, llvm::Triple::ArchType Arch)
static Value * EmitX86MaskedStore(CodeGenFunction &CGF, ArrayRef< Value *> Ops, unsigned Align)
unsigned char getNumArgsByte() const
bool isLibFunction(unsigned ID) const
Return true if this is a builtin for a libc/libm function, with a "__builtin_" prefix (e...
CharUnits - This is an opaque type for sizes expressed in character units.
static const NeonIntrinsicInfo AArch64SISDIntrinsicMap[]
APValue Val
Val - This is the value the expression can be folded to.
#define INTRINSIC_WITH_CC(NAME)
static Value * EmitX86MaskLogic(CodeGenFunction &CGF, Instruction::BinaryOps Opc, unsigned NumElts, ArrayRef< Value *> Ops, bool InvertLHS=false)
CharUnits getAlignment() const
Return the alignment of this pointer.
bool isVolatileQualified() const
Determine whether this type is volatile-qualified.
static bool AArch64SISDIntrinsicsProvenSorted
LangAS getAddressSpace() const
__DEVICE__ double powi(double __a, int __b)
static Value * EmitX86ScalarSelect(CodeGenFunction &CGF, Value *Mask, Value *Op0, Value *Op1)
Expr * IgnoreParenCasts() LLVM_READONLY
IgnoreParenCasts - Ignore parentheses and casts.
RValue EmitBuiltinExpr(const FunctionDecl *FD, unsigned BuiltinID, const CallExpr *E, ReturnValueSlot ReturnValue)
bool computeOSLogBufferLayout(clang::ASTContext &Ctx, const clang::CallExpr *E, OSLogBufferLayout &layout)
static FunctionDecl * Create(ASTContext &C, DeclContext *DC, SourceLocation StartLoc, SourceLocation NLoc, DeclarationName N, QualType T, TypeSourceInfo *TInfo, StorageClass SC, bool isInlineSpecified=false, bool hasWrittenPrototype=true, bool isConstexprSpecified=false)
static llvm::Value * dumpRecord(CodeGenFunction &CGF, QualType RType, Value *&RecordPtr, CharUnits Align, Value *Func, int Lvl)
llvm::Constant * CreateRuntimeFunction(llvm::FunctionType *Ty, StringRef Name, llvm::AttributeList ExtraAttrs=llvm::AttributeList(), bool Local=false)
Create a new runtime function with the specified type and name.
bool tryEvaluateObjectSize(uint64_t &Result, ASTContext &Ctx, unsigned Type) const
If the current Expr is a pointer, this will try to statically determine the number of bytes available...
llvm::Value * EmitLoadOfScalar(Address Addr, bool Volatile, QualType Ty, SourceLocation Loc, AlignmentSource Source=AlignmentSource::Type, bool isNontemporal=false)
EmitLoadOfScalar - Load a scalar value from an address, taking care to appropriately convert from the...
llvm::Type * HalfTy
float, double
static CharUnits One()
One - Construct a CharUnits quantity of one.
std::pair< llvm::Value *, llvm::Value * > ComplexPairTy
ASTContext & getContext() const
Represents a prototype with parameter type info, e.g.
llvm::Value * EmitISOVolatileStore(const CallExpr *E)
#define NEONMAP0(NameBase)
RValue - This trivial value class is used to represent the result of an expression that is evaluated...
const char * getName(unsigned ID) const
Return the identifier name for the specified builtin, e.g.
QuantityType getQuantity() const
getQuantity - Get the raw integer representation of this quantity.
llvm::APSInt EvaluateKnownConstInt(const ASTContext &Ctx, SmallVectorImpl< PartialDiagnosticAt > *Diag=nullptr) const
EvaluateKnownConstInt - Call EvaluateAsRValue and return the folded integer.
static Value * EmitX86MaskedCompare(CodeGenFunction &CGF, unsigned CC, bool Signed, ArrayRef< Value *> Ops)
ASTRecordLayout - This class contains layout information for one RecordDecl, which is a struct/union/...
bool HasSideEffects(const ASTContext &Ctx, bool IncludePossibleEffects=true) const
HasSideEffects - This routine returns true for all those expressions which have any effect other than...
llvm::Value * EmitVAStartEnd(llvm::Value *ArgValue, bool IsStart)
Emits a call to an LLVM variable-argument intrinsic, either llvm.va_start or llvm.va_end.
Exposes information about the current target.
#define copysign(__x, __y)
Expr - This represents one expression.
static RValue emitLibraryCall(CodeGenFunction &CGF, const FunctionDecl *FD, const CallExpr *E, llvm::Constant *calleeValue)
const T * castAs() const
Member-template castAs<specific type>.
static CGCallee forDirect(llvm::Constant *functionPtr, const CGCalleeInfo &abstractInfo=CGCalleeInfo())
const Expr * getCallee() const
#define INTRINSIC_X86_XSAVE_ID(NAME)
static Value * EmitX86Select(CodeGenFunction &CGF, Value *Mask, Value *Op0, Value *Op1)
llvm::Value * EmitToMemory(llvm::Value *Value, QualType Ty)
EmitToMemory - Change a scalar value from its value representation to its in-memory representation...
static Value * emitUnaryBuiltin(CodeGenFunction &CGF, const CallExpr *E, unsigned IntrinsicID)
static CharUnits fromQuantity(QuantityType Quantity)
fromQuantity - Construct a CharUnits quantity from a raw integer type.
static Value * EmitFromInt(CodeGenFunction &CGF, llvm::Value *V, QualType T, llvm::Type *ResultType)
static SVal getValue(SVal val, SValBuilder &svalBuilder)
llvm::LLVMContext & getLLVMContext()
bool isSignedIntegerType() const
Return true if this is an integer type that is signed, according to C99 6.2.5p4 [char, signed char, short, int, long..], or an enum decl which has a signed representation.
llvm::IntegerType * Int32Ty
static Value * emitRangedBuiltin(CodeGenFunction &CGF, unsigned IntrinsicID, int low, int high)
static const NeonIntrinsicInfo ARMSIMDIntrinsicMap[]
LValue MakeNaturalAlignAddrLValue(llvm::Value *V, QualType T)
static bool areBOSTypesCompatible(int From, int To)
Checks if using the result of __builtin_object_size(p, From) in place of __builtin_object_size(p, To) is correct.
RValue emitBuiltinOSLogFormat(const CallExpr &E)
Emit IR for __builtin_os_log_format.
static const NeonIntrinsicInfo * findNeonIntrinsicInMap(ArrayRef< NeonIntrinsicInfo > IntrinsicMap, unsigned BuiltinID, bool &MapProvenSorted)
static Value * EmitNontemporalLoad(CodeGenFunction &CGF, const CallExpr *E)
virtual llvm::Value * getPipeElemSize(const Expr *PipeArg)
llvm::Value * EmitX86BuiltinExpr(unsigned BuiltinID, const CallExpr *E)
GlobalDecl - represents a global declaration.
llvm::Value * EmitAMDGPUBuiltinExpr(unsigned BuiltinID, const CallExpr *E)
The l-value was considered opaque, so the alignment was determined from a type.
llvm::Value * EmitPPCBuiltinExpr(unsigned BuiltinID, const CallExpr *E)
static llvm::Value * EmitOverflowIntrinsic(CodeGenFunction &CGF, const llvm::Intrinsic::ID IntrinsicID, llvm::Value *X, llvm::Value *Y, llvm::Value *&Carry)
Emit a call to llvm.
uint64_t getFieldOffset(unsigned FieldNo) const
getFieldOffset - Get the offset of the given field index, in bits.
EltType getEltType() const
Enumerates target-specific builtins in their own namespaces within namespace clang.
Address CreateBitCast(Address Addr, llvm::Type *Ty, const llvm::Twine &Name="")
Encodes a location in the source.
static RValue getIgnored()
IdentifierInfo & get(StringRef Name)
Return the identifier token info for the specified named identifier.
LangAS getAddressSpace() const
Return the address space of this type.
llvm::Value * EmitFromMemory(llvm::Value *Value, QualType Ty)
EmitFromMemory - Change a scalar value from its memory representation to its value representation...
static Value * EmitX86MaskedCompareResult(CodeGenFunction &CGF, Value *Cmp, unsigned NumElts, Value *MaskIn)
static llvm::VectorType * GetFloatNeonType(CodeGenFunction *CGF, NeonTypeFlags IntTypeFlags)
llvm::CallSite EmitRuntimeCallOrInvoke(llvm::Value *callee, ArrayRef< llvm::Value *> args, const Twine &name="")
Emits a call or invoke instruction to the given runtime function.
ASTContext & getASTContext() const LLVM_READONLY
SourceLocation getLocStart() const LLVM_READONLY
static Value * EmitNontemporalStore(CodeGenFunction &CGF, const CallExpr *E)
llvm::IntegerType * Int16Ty
static Value * MakeAtomicCmpXchgValue(CodeGenFunction &CGF, const CallExpr *E, bool ReturnBool)
Utility to insert an atomic cmpxchg instruction.
static llvm::AtomicOrdering getBitTestAtomicOrdering(BitTest::InterlockingKind I)
OpenMPLinearClauseKind Modifier
Modifier of 'linear' clause.
#define MMA_VARIANTS(geom, type)
const ParmVarDecl * getParamDecl(unsigned i) const
constexpr XRayInstrMask Custom
Represents a canonical, potentially-qualified type.
llvm::Constant * emitAbstract(const Expr *E, QualType T)
Emit the result of the given expression as an abstract constant, asserting that it succeeded...
llvm::Function * generateBuiltinOSLogHelperFunction(const analyze_os_log::OSLogBufferLayout &Layout, CharUnits BufferAlignment)
static RValue EmitMSVCRTSetJmp(CodeGenFunction &CGF, MSVCSetJmpKind SJKind, const CallExpr *E)
MSVC handles setjmp a bit differently on different platforms.
All available information about a concrete callee.
bool operator<(DeclarationName LHS, DeclarationName RHS)
Ordering on two declaration names.
static Value * EmitX86SExtMask(CodeGenFunction &CGF, Value *Op, llvm::Type *DstTy)
char __ovld __cnfn rotate(char v, char i)
For each element in v, the bits are shifted left by the number of bits given by the corresponding ele...
bool hasSameUnqualifiedType(QualType T1, QualType T2) const
Determine whether the given types are equivalent after cvr-qualifiers have been removed.
SourceLocation getExprLoc() const LLVM_READONLY
getExprLoc - Return the preferred location for the arrow when diagnosing a problem with a generic exp...
static Value * EmitFAbs(CodeGenFunction &CGF, Value *V)
EmitFAbs - Emit a call to .fabs().
Like Angled, but marks system directories.
llvm::Value * EmitScalarExpr(const Expr *E, bool IgnoreResultAssign=false)
EmitScalarExpr - Emit the computation of the specified expression of LLVM scalar type, returning the result.
FunctionArgList - Type for representing both the decl and type of parameters to a function...
ast_type_traits::DynTypedNode Node
CGFunctionInfo - Class to encapsulate the information about a function definition.
llvm::Value * EmitNeonRShiftImm(llvm::Value *Vec, llvm::Value *Amt, llvm::Type *Ty, bool usgn, const char *name)
This class organizes the cross-function state that is used while generating LLVM code.
Dataflow Directional Tag Classes.
#define NEONMAP2(NameBase, LLVMIntrinsic, AltLLVMIntrinsic, TypeModifier)
bool hasSideEffects() const
static RValue getComplex(llvm::Value *V1, llvm::Value *V2)
EvalResult is a struct with detailed info about an evaluated expression.
FunctionDecl * getDirectCallee()
If the callee is a FunctionDecl, return it. Otherwise return 0.
Address CreateStructGEP(Address Addr, unsigned Index, CharUnits Offset, const llvm::Twine &Name="")
llvm::Value * EmitWebAssemblyBuiltinExpr(unsigned BuiltinID, const CallExpr *E)
llvm::Value * EmitNeonCall(llvm::Function *F, SmallVectorImpl< llvm::Value *> &O, const char *name, unsigned shift=0, bool rightshift=false)
void EmitARCIntrinsicUse(ArrayRef< llvm::Value *> values)
Given a number of pointers, inform the optimizer that they're being intrinsically used up until this ...
llvm::LoadInst * CreateLoad(Address Addr, const llvm::Twine &Name="")
static RValue EmitCheckedMixedSignMultiply(CodeGenFunction &CGF, const clang::Expr *Op1, WidthAndSignedness Op1Info, const clang::Expr *Op2, WidthAndSignedness Op2Info, const clang::Expr *ResultArg, QualType ResultQTy, WidthAndSignedness ResultInfo)
Emit a checked mixed-sign multiply.
static std::string getAsString(SplitQualType split, const PrintingPolicy &Policy)
bool isBooleanType() const
static Value * EmitTargetArchBuiltinExpr(CodeGenFunction *CGF, unsigned BuiltinID, const CallExpr *E, llvm::Triple::ArchType Arch)
llvm::Function * getIntrinsic(unsigned IID, ArrayRef< llvm::Type *> Tys=None)
bool isIntegerConstantExpr(llvm::APSInt &Result, const ASTContext &Ctx, SourceLocation *Loc=nullptr, bool isEvaluated=true) const
isIntegerConstantExpr - Return true if this expression is a valid integer constant expression...
llvm::StoreInst * CreateStore(llvm::Value *Val, Address Addr, bool IsVolatile=false)
Flags to identify the types for overloaded Neon builtins.
QualType getCallReturnType(const ASTContext &Ctx) const
getCallReturnType - Get the return type of the call expr.
Expr * IgnoreParenImpCasts() LLVM_READONLY
IgnoreParenImpCasts - Ignore parentheses and implicit casts.
SmallVector< OSLogBufferItem, 4 > Items
llvm::Value * EmitNeonSplat(llvm::Value *V, llvm::Constant *Idx)
#define X86_VENDOR(ENUM, STRING)
A helper class that allows the use of isa/cast/dyncast to detect TagType objects of structs/unions/cl...
llvm::Value * vectorWrapScalar16(llvm::Value *Op)
static llvm::Value * getDefaultBuiltinObjectSizeResult(unsigned Type, llvm::IntegerType *ResType)
bool isIntegerType() const
isIntegerType() does not include complex integers (a GCC extension).
llvm::Type * getElementType() const
Return the type of the values stored in this address.
llvm::Value * EmitNeonShiftVector(llvm::Value *V, llvm::Type *Ty, bool negateForRightShift)
llvm::PointerType * Int8PtrTy
static Value * packTBLDVectorList(CodeGenFunction &CGF, ArrayRef< Value *> Ops, Value *ExtOp, Value *IndexOp, llvm::Type *ResTy, unsigned IntID, const char *Name)
static llvm::Value * EmitBitTestIntrinsic(CodeGenFunction &CGF, unsigned BuiltinID, const CallExpr *E)
Emit a _bittest* intrinsic.
void setNontemporal(bool Value)
llvm::Value * EmitARMBuiltinExpr(unsigned BuiltinID, const CallExpr *E, llvm::Triple::ArchType Arch)
uint64_t getTypeSize(QualType T) const
Return the size of the specified (complete) type T, in bits.
CanQualType getCanonicalType(QualType T) const
Return the canonical (structural) type corresponding to the specified potentially non-canonical type ...
X
Add a minimal nested name specifier fixit hint to allow lookup of a tag name from an outer enclosing ...
CharUnits toCharUnitsFromBits(int64_t BitSize) const
Convert a size in bits to a size in characters.
static Value * EmitX86Abs(CodeGenFunction &CGF, ArrayRef< Value *> Ops)
TranslationUnitDecl * getTranslationUnitDecl() const
TypeInfo getTypeInfo(const Type *T) const
Get the size and alignment of the specified complete type in bits.
llvm::Function * LookupNeonLLVMIntrinsic(unsigned IntrinsicID, unsigned Modifier, llvm::Type *ArgTy, const CallExpr *E)
llvm::Type * ConvertType(QualType T)
Qualifiers getQualifiers() const
Retrieve the set of qualifiers applied to this type.
BuiltinCheckKind
Specifies which type of sanitizer check to apply when handling a particular builtin.
static Value * EmitCommonNeonSISDBuiltinExpr(CodeGenFunction &CGF, const NeonIntrinsicInfo &SISDInfo, SmallVectorImpl< Value *> &Ops, const CallExpr *E)
Builtin::Context & BuiltinInfo
static char bitActionToX86BTCode(BitTest::ActionKind A)
llvm::Constant * getBuiltinLibFunction(const FunctionDecl *FD, unsigned BuiltinID)
Given a builtin id for a function like "__builtin_fabsf", return a Function* for "fabsf".
Defines the clang::TargetInfo interface.
CallExpr - Represents a function call (C99 6.5.2.2, C++ [expr.call]).
static Value * emitTernaryBuiltin(CodeGenFunction &CGF, const CallExpr *E, unsigned IntrinsicID)
StringRef getName() const
Get the name of identifier for this declaration as a StringRef.
__DEVICE__ int max(int __a, int __b)
static Value * EmitSignBit(CodeGenFunction &CGF, Value *V)
Emit the computation of the sign bit for a floating point value.
static RValue get(llvm::Value *V)
static uint32_t GetX86CpuSupportsMask(ArrayRef< StringRef > FeatureStrs)
bool isPointerType() const
__DEVICE__ int min(int __a, int __b)
static ApplyDebugLocation CreateEmpty(CodeGenFunction &CGF)
Set the IRBuilder to not attach debug locations.
LValue - This represents an lvalue references.
Information for lazily generating a cleanup.
static Value * EmitX86ConvertToMask(CodeGenFunction &CGF, Value *In)
CallArgList - Type for representing both the value and type of arguments in a call.
static RValue EmitBinaryAtomic(CodeGenFunction &CGF, llvm::AtomicRMWInst::BinOp Kind, const CallExpr *E)
#define X86_FEATURE_COMPAT(VAL, ENUM, STR)
llvm::Value * EmitTargetBuiltinExpr(unsigned BuiltinID, const CallExpr *E)
EmitTargetBuiltinExpr - Emit the given builtin call.
static llvm::VectorType * GetNeonType(CodeGenFunction *CGF, NeonTypeFlags TypeFlags, bool HasLegalHalfType=true, bool V1Ty=false)
static bool AArch64SIMDIntrinsicsProvenSorted
QualType getIntTypeForBitwidth(unsigned DestWidth, unsigned Signed) const
getIntTypeForBitwidth - sets integer QualTy according to specified details: bitwidth, signed/unsigned.
llvm::Value * EmitISOVolatileLoad(const CallExpr *E)
llvm::Value * EmitSystemZBuiltinExpr(unsigned BuiltinID, const CallExpr *E)
static OMPLinearClause * Create(const ASTContext &C, SourceLocation StartLoc, SourceLocation LParenLoc, OpenMPLinearClauseKind Modifier, SourceLocation ModifierLoc, SourceLocation ColonLoc, SourceLocation EndLoc, ArrayRef< Expr *> VL, ArrayRef< Expr *> PL, ArrayRef< Expr *> IL, Expr *Step, Expr *CalcStep, Stmt *PreInit, Expr *PostUpdate)
Creates clause with a list of variables VL and a linear step Step.
CanQualType getSizeType() const
Return the unique type for "size_t" (C99 7.17), defined in <stddef.h>.