26 #include "llvm/ADT/StringExtras.h"
27 #include "llvm/IR/CallSite.h"
28 #include "llvm/IR/DataLayout.h"
29 #include "llvm/IR/InlineAsm.h"
30 #include "llvm/IR/Intrinsics.h"
31 #include "llvm/IR/MDBuilder.h"
34 using namespace clang;
35 using namespace CodeGen;
39 int64_t
clamp(int64_t
Value, int64_t Low, int64_t High) {
56 if (FD->
hasAttr<AsmLabelAttr>())
57 Name = getMangledName(D);
61 llvm::FunctionType *Ty =
62 cast<llvm::FunctionType>(getTypes().ConvertType(FD->
getType()));
64 return GetOrCreateLLVMFunction(Name, Ty, D,
false);
70 QualType T, llvm::IntegerType *IntType) {
73 if (V->getType()->isPointerTy())
74 return CGF.
Builder.CreatePtrToInt(V, IntType);
76 assert(V->getType() == IntType);
84 if (ResultType->isPointerTy())
85 return CGF.
Builder.CreateIntToPtr(V, ResultType);
87 assert(V->getType() == ResultType);
94 llvm::AtomicRMWInst::BinOp
Kind,
103 unsigned AddrSpace = DestPtr->getType()->getPointerAddressSpace();
105 llvm::IntegerType *IntType =
108 llvm::Type *IntPtrType = IntType->getPointerTo(AddrSpace);
114 Args[1] =
EmitToInt(CGF, Args[1], T, IntType);
117 Kind, Args[0], Args[1], llvm::AtomicOrdering::SequentiallyConsistent);
128 Address, llvm::PointerType::getUnqual(Val->getType()),
"cast");
144 llvm::AtomicRMWInst::BinOp
Kind,
153 llvm::AtomicRMWInst::BinOp
Kind,
155 Instruction::BinaryOps Op,
156 bool Invert =
false) {
164 unsigned AddrSpace = DestPtr->getType()->getPointerAddressSpace();
166 llvm::IntegerType *IntType =
169 llvm::Type *IntPtrType = IntType->getPointerTo(AddrSpace);
174 Args[1] =
EmitToInt(CGF, Args[1], T, IntType);
178 Kind, Args[0], Args[1], llvm::AtomicOrdering::SequentiallyConsistent);
179 Result = CGF.
Builder.CreateBinOp(Op, Result, Args[1]);
181 Result = CGF.
Builder.CreateBinOp(llvm::Instruction::Xor, Result,
182 llvm::ConstantInt::get(IntType, -1));
202 unsigned AddrSpace = DestPtr->getType()->getPointerAddressSpace();
204 llvm::IntegerType *IntType = llvm::IntegerType::get(
206 llvm::Type *IntPtrType = IntType->getPointerTo(AddrSpace);
212 Args[1] =
EmitToInt(CGF, Args[1], T, IntType);
216 Args[0], Args[1], Args[2], llvm::AtomicOrdering::SequentiallyConsistent,
217 llvm::AtomicOrdering::SequentiallyConsistent);
220 return CGF.
Builder.CreateZExt(CGF.
Builder.CreateExtractValue(Pair, 1),
232 unsigned IntrinsicID) {
236 return CGF.
Builder.CreateCall(F, Src0);
242 unsigned IntrinsicID) {
247 return CGF.
Builder.CreateCall(F, { Src0, Src1 });
253 unsigned IntrinsicID) {
259 return CGF.
Builder.CreateCall(F, { Src0, Src1, Src2 });
265 unsigned IntrinsicID) {
270 return CGF.
Builder.CreateCall(F, {Src0, Src1});
276 llvm::CallInst *Call = CGF.
Builder.CreateCall(F, V);
277 Call->setDoesNotAccessMemory();
287 int Width = Ty->getPrimitiveSizeInBits();
288 llvm::Type *IntTy = llvm::IntegerType::get(C, Width);
290 if (Ty->isPPC_FP128Ty()) {
300 Value *ShiftCst = llvm::ConstantInt::get(IntTy, Width);
301 V = CGF.
Builder.CreateLShr(V, ShiftCst);
305 IntTy = llvm::IntegerType::get(C, Width);
306 V = CGF.
Builder.CreateTrunc(V, IntTy);
308 Value *Zero = llvm::Constant::getNullValue(IntTy);
309 return CGF.
Builder.CreateICmpSLT(V, Zero);
313 const CallExpr *
E, llvm::Constant *calleeValue) {
332 assert(X->getType() == Y->getType() &&
333 "Arguments must be the same type. (Did you forget to make sure both "
334 "arguments have the same integer width?)");
338 Carry = CGF.
Builder.CreateExtractValue(Tmp, 1);
339 return CGF.
Builder.CreateExtractValue(Tmp, 0);
343 unsigned IntrinsicID,
346 llvm::MDNode *RNode = MDHelper.createRange(APInt(32, low), APInt(32, high));
348 llvm::Instruction *Call = CGF.
Builder.CreateCall(F);
349 Call->setMetadata(llvm::LLVMContext::MD_range, RNode);
354 struct WidthAndSignedness {
360 static WidthAndSignedness
363 assert(Type->
isIntegerType() &&
"Given type is not an integer.");
366 return {Width, Signed};
372 static struct WidthAndSignedness
374 assert(Types.size() > 0 &&
"Empty list of types.");
378 for (
const auto &
Type : Types) {
379 Signed |=
Type.Signed;
387 for (
const auto &
Type : Types) {
388 unsigned MinWidth =
Type.Width + (Signed && !
Type.Signed);
389 if (Width < MinWidth) {
394 return {Width, Signed};
399 if (ArgValue->getType() != DestType)
401 Builder.CreateBitCast(ArgValue, DestType, ArgValue->getName().data());
403 Intrinsic::ID inst = IsStart ? Intrinsic::vastart : Intrinsic::vaend;
404 return Builder.CreateCall(CGM.getIntrinsic(inst), ArgValue);
413 return From == To || (From == 0 && To == 1) || (From == 3 && To == 2);
418 return ConstantInt::get(ResType, (Type & 2) ? 0 : -1,
true);
422 CodeGenFunction::evaluateOrEmitBuiltinObjectSize(
const Expr *
E,
unsigned Type,
423 llvm::IntegerType *ResType,
427 return emitBuiltinObjectSize(E, Type, ResType, EmittedE);
428 return ConstantInt::get(ResType, ObjectSize,
true);
441 CodeGenFunction::emitBuiltinObjectSize(
const Expr *E,
unsigned Type,
442 llvm::IntegerType *ResType,
448 auto *PS = D->getDecl()->
getAttr<PassObjectSizeAttr>();
449 if (Param !=
nullptr && PS !=
nullptr &&
451 auto Iter = SizeArguments.find(Param);
452 assert(Iter != SizeArguments.end());
455 auto DIter = LocalDeclMap.find(D);
456 assert(DIter != LocalDeclMap.end());
458 return EmitLoadOfScalar(DIter->second,
false,
469 Value *Ptr = EmittedE ? EmittedE : EmitScalarExpr(E);
470 assert(Ptr->getType()->isPointerTy() &&
471 "Non-pointer passed to __builtin_object_size?");
473 Value *F = CGM.getIntrinsic(Intrinsic::objectsize, {ResType, Ptr->getType()});
479 return Builder.CreateCall(F, {Ptr, Min, NullIsUnknown});
488 _InterlockedDecrement,
489 _InterlockedExchange,
490 _InterlockedExchangeAdd,
491 _InterlockedExchangeSub,
492 _InterlockedIncrement,
495 _interlockedbittestandset,
502 case MSVCIntrin::_BitScanForward:
503 case MSVCIntrin::_BitScanReverse: {
508 EmitScalarExpr(E->
getArg(0))->getType()->getPointerElementType();
511 Value *ArgZero = llvm::Constant::getNullValue(ArgType);
512 Value *ResZero = llvm::Constant::getNullValue(ResultType);
513 Value *ResOne = llvm::ConstantInt::get(ResultType, 1);
516 BasicBlock *
End = createBasicBlock(
"bitscan_end", this->CurFn);
518 PHINode *
Result =
Builder.CreatePHI(ResultType, 2,
"bitscan_result");
521 Value *IsZero =
Builder.CreateICmpEQ(ArgValue, ArgZero);
522 BasicBlock *NotZero = createBasicBlock(
"bitscan_not_zero", this->CurFn);
523 Builder.CreateCondBr(IsZero, End, NotZero);
524 Result->addIncoming(ResZero, Begin);
526 Builder.SetInsertPoint(NotZero);
527 Address IndexAddress = EmitPointerWithAlignment(E->
getArg(0));
529 if (BuiltinID == MSVCIntrin::_BitScanForward) {
530 Value *F = CGM.getIntrinsic(Intrinsic::cttz, ArgType);
532 ZeroCount =
Builder.CreateIntCast(ZeroCount, IndexType,
false);
533 Builder.CreateStore(ZeroCount, IndexAddress,
false);
535 unsigned ArgWidth = cast<llvm::IntegerType>(ArgType)->getBitWidth();
536 Value *ArgTypeLastIndex = llvm::ConstantInt::get(IndexType, ArgWidth - 1);
538 Value *F = CGM.getIntrinsic(Intrinsic::ctlz, ArgType);
540 ZeroCount =
Builder.CreateIntCast(ZeroCount, IndexType,
false);
541 Value *Index =
Builder.CreateNSWSub(ArgTypeLastIndex, ZeroCount);
542 Builder.CreateStore(Index, IndexAddress,
false);
545 Result->addIncoming(ResOne, NotZero);
550 case MSVCIntrin::_InterlockedAnd:
552 case MSVCIntrin::_InterlockedExchange:
554 case MSVCIntrin::_InterlockedExchangeAdd:
556 case MSVCIntrin::_InterlockedExchangeSub:
558 case MSVCIntrin::_InterlockedOr:
560 case MSVCIntrin::_InterlockedXor:
563 case MSVCIntrin::_interlockedbittestandset: {
566 AtomicRMWInst *RMWI =
Builder.CreateAtomicRMW(
567 AtomicRMWInst::Or, Addr,
568 Builder.CreateShl(ConstantInt::get(Bit->getType(), 1), Bit),
569 llvm::AtomicOrdering::SequentiallyConsistent);
575 return Builder.CreateAnd(Truncated,
576 ConstantInt::get(Truncated->getType(), 1));
579 case MSVCIntrin::_InterlockedDecrement: {
581 AtomicRMWInst *RMWI =
Builder.CreateAtomicRMW(
583 EmitScalarExpr(E->
getArg(0)),
584 ConstantInt::get(IntTy, 1),
585 llvm::AtomicOrdering::SequentiallyConsistent);
586 return Builder.CreateSub(RMWI, ConstantInt::get(IntTy, 1));
588 case MSVCIntrin::_InterlockedIncrement: {
590 AtomicRMWInst *RMWI =
Builder.CreateAtomicRMW(
592 EmitScalarExpr(E->
getArg(0)),
593 ConstantInt::get(IntTy, 1),
594 llvm::AtomicOrdering::SequentiallyConsistent);
595 return Builder.CreateAdd(RMWI, ConstantInt::get(IntTy, 1));
598 case MSVCIntrin::__fastfail: {
602 llvm::Triple::ArchType ISA = getTarget().getTriple().getArch();
603 StringRef Asm, Constraints;
606 ErrorUnsupported(E,
"__fastfail call for this architecture");
608 case llvm::Triple::x86:
609 case llvm::Triple::x86_64:
611 Constraints =
"{cx}";
613 case llvm::Triple::thumb:
615 Constraints =
"{r0}";
618 llvm::FunctionType *FTy = llvm::FunctionType::get(VoidTy, {Int32Ty},
false);
619 llvm::InlineAsm *IA =
620 llvm::InlineAsm::get(FTy, Asm, Constraints,
true);
621 llvm::AttributeList NoReturnAttr = llvm::AttributeList::get(
622 getLLVMContext(), llvm::AttributeList::FunctionIndex,
623 llvm::Attribute::NoReturn);
624 CallSite CS =
Builder.CreateCall(IA, EmitScalarExpr(E->
getArg(0)));
625 CS.setAttributes(NoReturnAttr);
626 return CS.getInstruction();
629 llvm_unreachable(
"Incorrect MSVC intrinsic!");
635 CallObjCArcUse(
llvm::Value *
object) : object(object) {}
645 unsigned BuiltinID,
const CallExpr *E,
652 return RValue::get(llvm::ConstantInt::get(getLLVMContext(),
655 return RValue::get(llvm::ConstantFP::get(getLLVMContext(),
661 case Builtin::BI__builtin___CFStringMakeConstantString:
662 case Builtin::BI__builtin___NSStringMakeConstantString:
664 case Builtin::BI__builtin_stdarg_start:
665 case Builtin::BI__builtin_va_start:
666 case Builtin::BI__va_start:
667 case Builtin::BI__builtin_va_end:
669 EmitVAStartEnd(BuiltinID == Builtin::BI__va_start
670 ? EmitScalarExpr(E->
getArg(0))
671 : EmitVAListRef(E->
getArg(0)).getPointer(),
672 BuiltinID != Builtin::BI__builtin_va_end));
673 case Builtin::BI__builtin_va_copy: {
674 Value *DstPtr = EmitVAListRef(E->
getArg(0)).getPointer();
675 Value *SrcPtr = EmitVAListRef(E->
getArg(1)).getPointer();
679 DstPtr =
Builder.CreateBitCast(DstPtr, Type);
680 SrcPtr =
Builder.CreateBitCast(SrcPtr, Type);
684 case Builtin::BI__builtin_abs:
685 case Builtin::BI__builtin_labs:
686 case Builtin::BI__builtin_llabs: {
691 Builder.CreateICmpSGE(ArgValue,
692 llvm::Constant::getNullValue(ArgValue->getType()),
695 Builder.CreateSelect(CmpResult, ArgValue, NegOp,
"abs");
699 case Builtin::BI__builtin_fabs:
700 case Builtin::BI__builtin_fabsf:
701 case Builtin::BI__builtin_fabsl: {
704 case Builtin::BI__builtin_fmod:
705 case Builtin::BI__builtin_fmodf:
706 case Builtin::BI__builtin_fmodl: {
712 case Builtin::BI__builtin_copysign:
713 case Builtin::BI__builtin_copysignf:
714 case Builtin::BI__builtin_copysignl: {
717 case Builtin::BI__builtin_ceil:
718 case Builtin::BI__builtin_ceilf:
719 case Builtin::BI__builtin_ceill: {
722 case Builtin::BI__builtin_floor:
723 case Builtin::BI__builtin_floorf:
724 case Builtin::BI__builtin_floorl: {
727 case Builtin::BI__builtin_trunc:
728 case Builtin::BI__builtin_truncf:
729 case Builtin::BI__builtin_truncl: {
732 case Builtin::BI__builtin_rint:
733 case Builtin::BI__builtin_rintf:
734 case Builtin::BI__builtin_rintl: {
737 case Builtin::BI__builtin_nearbyint:
738 case Builtin::BI__builtin_nearbyintf:
739 case Builtin::BI__builtin_nearbyintl: {
742 case Builtin::BI__builtin_round:
743 case Builtin::BI__builtin_roundf:
744 case Builtin::BI__builtin_roundl: {
747 case Builtin::BI__builtin_fmin:
748 case Builtin::BI__builtin_fminf:
749 case Builtin::BI__builtin_fminl: {
752 case Builtin::BI__builtin_fmax:
753 case Builtin::BI__builtin_fmaxf:
754 case Builtin::BI__builtin_fmaxl: {
757 case Builtin::BI__builtin_conj:
758 case Builtin::BI__builtin_conjf:
759 case Builtin::BI__builtin_conjl: {
761 Value *Real = ComplexVal.first;
762 Value *Imag = ComplexVal.second;
764 Imag->getType()->isFPOrFPVectorTy()
765 ? llvm::ConstantFP::getZeroValueForNegation(Imag->getType())
766 : llvm::Constant::getNullValue(Imag->getType());
768 Imag =
Builder.CreateFSub(Zero, Imag,
"sub");
771 case Builtin::BI__builtin_creal:
772 case Builtin::BI__builtin_crealf:
773 case Builtin::BI__builtin_creall:
774 case Builtin::BIcreal:
775 case Builtin::BIcrealf:
776 case Builtin::BIcreall: {
781 case Builtin::BI__builtin_cimag:
782 case Builtin::BI__builtin_cimagf:
783 case Builtin::BI__builtin_cimagl:
784 case Builtin::BIcimag:
785 case Builtin::BIcimagf:
786 case Builtin::BIcimagl: {
791 case Builtin::BI__builtin_ctzs:
792 case Builtin::BI__builtin_ctz:
793 case Builtin::BI__builtin_ctzl:
794 case Builtin::BI__builtin_ctzll: {
798 Value *F = CGM.getIntrinsic(Intrinsic::cttz, ArgType);
801 Value *ZeroUndef =
Builder.getInt1(getTarget().isCLZForZeroUndef());
802 Value *Result =
Builder.CreateCall(F, {ArgValue, ZeroUndef});
803 if (Result->getType() != ResultType)
804 Result =
Builder.CreateIntCast(Result, ResultType,
true,
808 case Builtin::BI__builtin_clzs:
809 case Builtin::BI__builtin_clz:
810 case Builtin::BI__builtin_clzl:
811 case Builtin::BI__builtin_clzll: {
815 Value *F = CGM.getIntrinsic(Intrinsic::ctlz, ArgType);
818 Value *ZeroUndef =
Builder.getInt1(getTarget().isCLZForZeroUndef());
819 Value *Result =
Builder.CreateCall(F, {ArgValue, ZeroUndef});
820 if (Result->getType() != ResultType)
821 Result =
Builder.CreateIntCast(Result, ResultType,
true,
825 case Builtin::BI__builtin_ffs:
826 case Builtin::BI__builtin_ffsl:
827 case Builtin::BI__builtin_ffsll: {
832 Value *F = CGM.getIntrinsic(Intrinsic::cttz, ArgType);
837 llvm::ConstantInt::get(ArgType, 1));
838 Value *Zero = llvm::Constant::getNullValue(ArgType);
839 Value *IsZero =
Builder.CreateICmpEQ(ArgValue, Zero,
"iszero");
840 Value *Result =
Builder.CreateSelect(IsZero, Zero, Tmp,
"ffs");
841 if (Result->getType() != ResultType)
842 Result =
Builder.CreateIntCast(Result, ResultType,
true,
846 case Builtin::BI__builtin_parity:
847 case Builtin::BI__builtin_parityl:
848 case Builtin::BI__builtin_parityll: {
853 Value *F = CGM.getIntrinsic(Intrinsic::ctpop, ArgType);
857 Value *Result =
Builder.CreateAnd(Tmp, llvm::ConstantInt::get(ArgType, 1));
858 if (Result->getType() != ResultType)
859 Result =
Builder.CreateIntCast(Result, ResultType,
true,
863 case Builtin::BI__popcnt16:
864 case Builtin::BI__popcnt:
865 case Builtin::BI__popcnt64:
866 case Builtin::BI__builtin_popcount:
867 case Builtin::BI__builtin_popcountl:
868 case Builtin::BI__builtin_popcountll: {
872 Value *F = CGM.getIntrinsic(Intrinsic::ctpop, ArgType);
876 if (Result->getType() != ResultType)
877 Result =
Builder.CreateIntCast(Result, ResultType,
true,
881 case Builtin::BI_rotr8:
882 case Builtin::BI_rotr16:
883 case Builtin::BI_rotr:
884 case Builtin::BI_lrotr:
885 case Builtin::BI_rotr64: {
890 Shift =
Builder.CreateIntCast(Shift, ArgType,
false);
891 unsigned ArgWidth = cast<llvm::IntegerType>(ArgType)->getBitWidth();
892 Value *ArgTypeSize = llvm::ConstantInt::get(ArgType, ArgWidth);
893 Value *ArgZero = llvm::Constant::getNullValue(ArgType);
895 Value *Mask = llvm::ConstantInt::get(ArgType, ArgWidth - 1);
896 Shift =
Builder.CreateAnd(Shift, Mask);
897 Value *LeftShift =
Builder.CreateSub(ArgTypeSize, Shift);
901 Value *Rotated =
Builder.CreateOr(LeftShifted, RightShifted);
903 Value *ShiftIsZero =
Builder.CreateICmpEQ(Shift, ArgZero);
904 Value *Result =
Builder.CreateSelect(ShiftIsZero, Val, Rotated);
907 case Builtin::BI_rotl8:
908 case Builtin::BI_rotl16:
909 case Builtin::BI_rotl:
910 case Builtin::BI_lrotl:
911 case Builtin::BI_rotl64: {
916 Shift =
Builder.CreateIntCast(Shift, ArgType,
false);
917 unsigned ArgWidth = cast<llvm::IntegerType>(ArgType)->getBitWidth();
918 Value *ArgTypeSize = llvm::ConstantInt::get(ArgType, ArgWidth);
919 Value *ArgZero = llvm::Constant::getNullValue(ArgType);
921 Value *Mask = llvm::ConstantInt::get(ArgType, ArgWidth - 1);
922 Shift =
Builder.CreateAnd(Shift, Mask);
923 Value *RightShift =
Builder.CreateSub(ArgTypeSize, Shift);
926 Value *RightShifted =
Builder.CreateLShr(Val, RightShift);
927 Value *Rotated =
Builder.CreateOr(LeftShifted, RightShifted);
929 Value *ShiftIsZero =
Builder.CreateICmpEQ(Shift, ArgZero);
930 Value *Result =
Builder.CreateSelect(ShiftIsZero, Val, Rotated);
933 case Builtin::BI__builtin_unpredictable: {
939 case Builtin::BI__builtin_expect: {
943 Value *ExpectedValue = EmitScalarExpr(E->
getArg(1));
947 if (CGM.getCodeGenOpts().OptimizationLevel == 0)
950 Value *FnExpect = CGM.getIntrinsic(Intrinsic::expect, ArgType);
952 Builder.CreateCall(FnExpect, {ArgValue, ExpectedValue},
"expval");
955 case Builtin::BI__builtin_assume_aligned: {
960 Value *AlignmentValue = EmitScalarExpr(E->
getArg(1));
961 ConstantInt *AlignmentCI = cast<ConstantInt>(AlignmentValue);
962 unsigned Alignment = (
unsigned) AlignmentCI->getZExtValue();
964 EmitAlignmentAssumption(PtrValue, Alignment, OffsetValue);
967 case Builtin::BI__assume:
968 case Builtin::BI__builtin_assume: {
973 Value *FnAssume = CGM.getIntrinsic(Intrinsic::assume);
976 case Builtin::BI__builtin_bswap16:
977 case Builtin::BI__builtin_bswap32:
978 case Builtin::BI__builtin_bswap64: {
981 case Builtin::BI__builtin_bitreverse8:
982 case Builtin::BI__builtin_bitreverse16:
983 case Builtin::BI__builtin_bitreverse32:
984 case Builtin::BI__builtin_bitreverse64: {
987 case Builtin::BI__builtin_object_size: {
990 auto *ResType = cast<llvm::IntegerType>(ConvertType(E->
getType()));
997 case Builtin::BI__builtin_prefetch: {
1001 llvm::ConstantInt::get(Int32Ty, 0);
1003 llvm::ConstantInt::get(Int32Ty, 3);
1004 Value *Data = llvm::ConstantInt::get(Int32Ty, 1);
1008 case Builtin::BI__builtin_readcyclecounter: {
1009 Value *F = CGM.getIntrinsic(Intrinsic::readcyclecounter);
1012 case Builtin::BI__builtin___clear_cache: {
1014 Value *
End = EmitScalarExpr(E->getArg(1));
1015 Value *F = CGM.getIntrinsic(Intrinsic::clear_cache);
1018 case Builtin::BI__builtin_trap:
1019 return RValue::get(EmitTrapCall(Intrinsic::trap));
1020 case Builtin::BI__debugbreak:
1021 return RValue::get(EmitTrapCall(Intrinsic::debugtrap));
1022 case Builtin::BI__builtin_unreachable: {
1023 if (SanOpts.has(SanitizerKind::Unreachable)) {
1024 SanitizerScope SanScope(
this);
1025 EmitCheck(std::make_pair(static_cast<llvm::Value *>(
Builder.getFalse()),
1026 SanitizerKind::Unreachable),
1027 SanitizerHandler::BuiltinUnreachable,
1028 EmitCheckSourceLocation(E->
getExprLoc()), None);
1033 EmitBlock(createBasicBlock(
"unreachable.cont"));
1038 case Builtin::BI__builtin_powi:
1039 case Builtin::BI__builtin_powif:
1040 case Builtin::BI__builtin_powil: {
1041 Value *
Base = EmitScalarExpr(E->getArg(0));
1042 Value *Exponent = EmitScalarExpr(E->getArg(1));
1044 Value *F = CGM.getIntrinsic(Intrinsic::powi, ArgType);
1048 case Builtin::BI__builtin_isgreater:
1049 case Builtin::BI__builtin_isgreaterequal:
1050 case Builtin::BI__builtin_isless:
1051 case Builtin::BI__builtin_islessequal:
1052 case Builtin::BI__builtin_islessgreater:
1053 case Builtin::BI__builtin_isunordered: {
1056 Value *LHS = EmitScalarExpr(E->getArg(0));
1057 Value *RHS = EmitScalarExpr(E->getArg(1));
1059 switch (BuiltinID) {
1060 default: llvm_unreachable(
"Unknown ordered comparison");
1061 case Builtin::BI__builtin_isgreater:
1062 LHS =
Builder.CreateFCmpOGT(LHS, RHS,
"cmp");
1064 case Builtin::BI__builtin_isgreaterequal:
1065 LHS =
Builder.CreateFCmpOGE(LHS, RHS,
"cmp");
1067 case Builtin::BI__builtin_isless:
1068 LHS =
Builder.CreateFCmpOLT(LHS, RHS,
"cmp");
1070 case Builtin::BI__builtin_islessequal:
1071 LHS =
Builder.CreateFCmpOLE(LHS, RHS,
"cmp");
1073 case Builtin::BI__builtin_islessgreater:
1074 LHS =
Builder.CreateFCmpONE(LHS, RHS,
"cmp");
1076 case Builtin::BI__builtin_isunordered:
1077 LHS =
Builder.CreateFCmpUNO(LHS, RHS,
"cmp");
1081 return RValue::get(
Builder.CreateZExt(LHS, ConvertType(E->
getType())));
1083 case Builtin::BI__builtin_isnan: {
1084 Value *V = EmitScalarExpr(E->getArg(0));
1085 V =
Builder.CreateFCmpUNO(V, V,
"cmp");
1086 return RValue::get(
Builder.CreateZExt(V, ConvertType(E->
getType())));
1089 case Builtin::BIfinite:
1090 case Builtin::BI__finite:
1091 case Builtin::BIfinitef:
1092 case Builtin::BI__finitef:
1093 case Builtin::BIfinitel:
1094 case Builtin::BI__finitel:
1095 case Builtin::BI__builtin_isinf:
1096 case Builtin::BI__builtin_isfinite: {
1100 Value *V = EmitScalarExpr(E->getArg(0));
1102 Constant *Infinity = ConstantFP::getInfinity(V->getType());
1103 CmpInst::Predicate Pred = (BuiltinID == Builtin::BI__builtin_isinf)
1105 : CmpInst::FCMP_ONE;
1106 Value *FCmp =
Builder.CreateFCmp(Pred, Fabs, Infinity,
"cmpinf");
1107 return RValue::get(
Builder.CreateZExt(FCmp, ConvertType(E->
getType())));
1110 case Builtin::BI__builtin_isinf_sign: {
1112 Value *Arg = EmitScalarExpr(E->getArg(0));
1115 AbsArg, ConstantFP::getInfinity(Arg->getType()),
"isinf");
1118 llvm::Type *IntTy = ConvertType(E->
getType());
1119 Value *Zero = Constant::getNullValue(IntTy);
1120 Value *One = ConstantInt::get(IntTy, 1);
1121 Value *NegativeOne = ConstantInt::get(IntTy, -1);
1122 Value *SignResult =
Builder.CreateSelect(IsNeg, NegativeOne, One);
1124 return RValue::get(Result);
1127 case Builtin::BI__builtin_isnormal: {
1129 Value *V = EmitScalarExpr(E->getArg(0));
1133 Value *IsLessThanInf =
1134 Builder.CreateFCmpULT(Abs, ConstantFP::getInfinity(V->getType()),
"isinf");
1135 APFloat Smallest = APFloat::getSmallestNormalized(
1136 getContext().getFloatTypeSemantics(E->getArg(0)->
getType()));
1138 Builder.CreateFCmpUGE(Abs, ConstantFP::get(V->getContext(), Smallest),
1140 V =
Builder.CreateAnd(Eq, IsLessThanInf,
"and");
1141 V =
Builder.CreateAnd(V, IsNormal,
"and");
1142 return RValue::get(
Builder.CreateZExt(V, ConvertType(E->
getType())));
1145 case Builtin::BI__builtin_fpclassify: {
1146 Value *V = EmitScalarExpr(E->getArg(5));
1147 llvm::Type *Ty = ConvertType(E->getArg(5)->
getType());
1150 BasicBlock *Begin =
Builder.GetInsertBlock();
1151 BasicBlock *End = createBasicBlock(
"fpclassify_end", this->CurFn);
1155 "fpclassify_result");
1158 Builder.SetInsertPoint(Begin);
1159 Value *IsZero =
Builder.CreateFCmpOEQ(V, Constant::getNullValue(Ty),
1161 Value *ZeroLiteral = EmitScalarExpr(E->getArg(4));
1162 BasicBlock *NotZero = createBasicBlock(
"fpclassify_not_zero", this->CurFn);
1163 Builder.CreateCondBr(IsZero, End, NotZero);
1164 Result->addIncoming(ZeroLiteral, Begin);
1167 Builder.SetInsertPoint(NotZero);
1169 Value *NanLiteral = EmitScalarExpr(E->getArg(0));
1170 BasicBlock *NotNan = createBasicBlock(
"fpclassify_not_nan", this->CurFn);
1171 Builder.CreateCondBr(IsNan, End, NotNan);
1172 Result->addIncoming(NanLiteral, NotZero);
1175 Builder.SetInsertPoint(NotNan);
1178 Builder.CreateFCmpOEQ(VAbs, ConstantFP::getInfinity(V->getType()),
1180 Value *InfLiteral = EmitScalarExpr(E->getArg(1));
1181 BasicBlock *NotInf = createBasicBlock(
"fpclassify_not_inf", this->CurFn);
1182 Builder.CreateCondBr(IsInf, End, NotInf);
1183 Result->addIncoming(InfLiteral, NotNan);
1186 Builder.SetInsertPoint(NotInf);
1187 APFloat Smallest = APFloat::getSmallestNormalized(
1188 getContext().getFloatTypeSemantics(E->getArg(5)->
getType()));
1190 Builder.CreateFCmpUGE(VAbs, ConstantFP::get(V->getContext(), Smallest),
1192 Value *NormalResult =
1193 Builder.CreateSelect(IsNormal, EmitScalarExpr(E->getArg(2)),
1194 EmitScalarExpr(E->getArg(3)));
1196 Result->addIncoming(NormalResult, NotInf);
1200 return RValue::get(Result);
1203 case Builtin::BIalloca:
1204 case Builtin::BI_alloca:
1205 case Builtin::BI__builtin_alloca: {
1206 Value *Size = EmitScalarExpr(E->getArg(0));
1207 const TargetInfo &TI = getContext().getTargetInfo();
1209 unsigned SuitableAlignmentInBytes =
1211 .toCharUnitsFromBits(TI.getSuitableAlign())
1213 AllocaInst *AI =
Builder.CreateAlloca(
Builder.getInt8Ty(), Size);
1214 AI->setAlignment(SuitableAlignmentInBytes);
1215 return RValue::get(AI);
1218 case Builtin::BI__builtin_alloca_with_align: {
1219 Value *Size = EmitScalarExpr(E->getArg(0));
1220 Value *AlignmentInBitsValue = EmitScalarExpr(E->getArg(1));
1221 auto *AlignmentInBitsCI = cast<ConstantInt>(AlignmentInBitsValue);
1222 unsigned AlignmentInBits = AlignmentInBitsCI->getZExtValue();
1223 unsigned AlignmentInBytes =
1224 CGM.getContext().toCharUnitsFromBits(AlignmentInBits).getQuantity();
1225 AllocaInst *AI =
Builder.CreateAlloca(
Builder.getInt8Ty(), Size);
1226 AI->setAlignment(AlignmentInBytes);
1227 return RValue::get(AI);
1230 case Builtin::BIbzero:
1231 case Builtin::BI__builtin_bzero: {
1232 Address Dest = EmitPointerWithAlignment(E->getArg(0));
1233 Value *SizeVal = EmitScalarExpr(E->getArg(1));
1239 case Builtin::BImemcpy:
1240 case Builtin::BI__builtin_memcpy: {
1241 Address Dest = EmitPointerWithAlignment(E->getArg(0));
1242 Address Src = EmitPointerWithAlignment(E->getArg(1));
1243 Value *SizeVal = EmitScalarExpr(E->getArg(2));
1246 EmitNonNullArgCheck(RValue::get(Src.getPointer()), E->getArg(1)->
getType(),
1248 Builder.CreateMemCpy(Dest, Src, SizeVal,
false);
1252 case Builtin::BI__builtin_char_memchr:
1253 BuiltinID = Builtin::BI__builtin_memchr;
1256 case Builtin::BI__builtin___memcpy_chk: {
1258 llvm::APSInt Size, DstSize;
1262 if (Size.ugt(DstSize))
1264 Address Dest = EmitPointerWithAlignment(E->getArg(0));
1265 Address Src = EmitPointerWithAlignment(E->getArg(1));
1266 Value *SizeVal = llvm::ConstantInt::get(
Builder.getContext(), Size);
1267 Builder.CreateMemCpy(Dest, Src, SizeVal,
false);
1271 case Builtin::BI__builtin_objc_memmove_collectable: {
1272 Address DestAddr = EmitPointerWithAlignment(E->getArg(0));
1273 Address SrcAddr = EmitPointerWithAlignment(E->getArg(1));
1274 Value *SizeVal = EmitScalarExpr(E->getArg(2));
1275 CGM.getObjCRuntime().EmitGCMemmoveCollectable(*
this,
1276 DestAddr, SrcAddr, SizeVal);
1280 case Builtin::BI__builtin___memmove_chk: {
1282 llvm::APSInt Size, DstSize;
1286 if (Size.ugt(DstSize))
1288 Address Dest = EmitPointerWithAlignment(E->getArg(0));
1289 Address Src = EmitPointerWithAlignment(E->getArg(1));
1290 Value *SizeVal = llvm::ConstantInt::get(
Builder.getContext(), Size);
1291 Builder.CreateMemMove(Dest, Src, SizeVal,
false);
1295 case Builtin::BImemmove:
1296 case Builtin::BI__builtin_memmove: {
1297 Address Dest = EmitPointerWithAlignment(E->getArg(0));
1298 Address Src = EmitPointerWithAlignment(E->getArg(1));
1299 Value *SizeVal = EmitScalarExpr(E->getArg(2));
1302 EmitNonNullArgCheck(RValue::get(Src.getPointer()), E->getArg(1)->
getType(),
1304 Builder.CreateMemMove(Dest, Src, SizeVal,
false);
1307 case Builtin::BImemset:
1308 case Builtin::BI__builtin_memset: {
1309 Address Dest = EmitPointerWithAlignment(E->getArg(0));
1310 Value *ByteVal =
Builder.CreateTrunc(EmitScalarExpr(E->getArg(1)),
1312 Value *SizeVal = EmitScalarExpr(E->getArg(2));
1315 Builder.CreateMemSet(Dest, ByteVal, SizeVal,
false);
1318 case Builtin::BI__builtin___memset_chk: {
1320 llvm::APSInt Size, DstSize;
1324 if (Size.ugt(DstSize))
1326 Address Dest = EmitPointerWithAlignment(E->getArg(0));
1327 Value *ByteVal =
Builder.CreateTrunc(EmitScalarExpr(E->getArg(1)),
1329 Value *SizeVal = llvm::ConstantInt::get(
Builder.getContext(), Size);
1330 Builder.CreateMemSet(Dest, ByteVal, SizeVal,
false);
1333 case Builtin::BI__builtin_dwarf_cfa: {
1344 Value *F = CGM.getIntrinsic(Intrinsic::eh_dwarf_cfa);
1345 return RValue::get(
Builder.CreateCall(F,
1346 llvm::ConstantInt::get(Int32Ty, Offset)));
1348 case Builtin::BI__builtin_return_address: {
1350 CGM.EmitConstantExpr(E->getArg(0), getContext().UnsignedIntTy,
this);
1351 Value *F = CGM.getIntrinsic(Intrinsic::returnaddress);
1352 return RValue::get(
Builder.CreateCall(F, Depth));
1354 case Builtin::BI_ReturnAddress: {
1355 Value *F = CGM.getIntrinsic(Intrinsic::returnaddress);
1358 case Builtin::BI__builtin_frame_address: {
1360 CGM.EmitConstantExpr(E->getArg(0), getContext().UnsignedIntTy,
this);
1361 Value *F = CGM.getIntrinsic(Intrinsic::frameaddress);
1362 return RValue::get(
Builder.CreateCall(F, Depth));
1364 case Builtin::BI__builtin_extract_return_addr: {
1366 Value *Result = getTargetHooks().decodeReturnAddress(*
this, Address);
1367 return RValue::get(Result);
1369 case Builtin::BI__builtin_frob_return_addr: {
1370 Value *Address = EmitScalarExpr(E->getArg(0));
1371 Value *Result = getTargetHooks().encodeReturnAddress(*
this, Address);
1372 return RValue::get(Result);
1374 case Builtin::BI__builtin_dwarf_sp_column: {
1375 llvm::IntegerType *Ty
1376 = cast<llvm::IntegerType>(ConvertType(E->
getType()));
1377 int Column = getTargetHooks().getDwarfEHStackPointer(CGM);
1379 CGM.ErrorUnsupported(E,
"__builtin_dwarf_sp_column");
1380 return RValue::get(llvm::UndefValue::get(Ty));
1382 return RValue::get(llvm::ConstantInt::get(Ty, Column,
true));
1384 case Builtin::BI__builtin_init_dwarf_reg_size_table: {
1385 Value *Address = EmitScalarExpr(E->getArg(0));
1386 if (getTargetHooks().initDwarfEHRegSizeTable(*
this, Address))
1387 CGM.ErrorUnsupported(E,
"__builtin_init_dwarf_reg_size_table");
1388 return RValue::get(llvm::UndefValue::get(ConvertType(E->
getType())));
1390 case Builtin::BI__builtin_eh_return: {
1391 Value *Int = EmitScalarExpr(E->getArg(0));
1392 Value *Ptr = EmitScalarExpr(E->getArg(1));
1394 llvm::IntegerType *IntTy = cast<llvm::IntegerType>(Int->getType());
1395 assert((IntTy->getBitWidth() == 32 || IntTy->getBitWidth() == 64) &&
1396 "LLVM's __builtin_eh_return only supports 32- and 64-bit variants");
1397 Value *F = CGM.getIntrinsic(IntTy->getBitWidth() == 32
1398 ? Intrinsic::eh_return_i32
1399 : Intrinsic::eh_return_i64);
1400 Builder.CreateCall(F, {Int, Ptr});
1404 EmitBlock(createBasicBlock(
"builtin_eh_return.cont"));
1406 return RValue::get(
nullptr);
1408 case Builtin::BI__builtin_unwind_init: {
1409 Value *F = CGM.getIntrinsic(Intrinsic::eh_unwind_init);
1410 return RValue::get(
Builder.CreateCall(F));
1412 case Builtin::BI__builtin_extend_pointer: {
1424 Value *Ptr = EmitScalarExpr(E->getArg(0));
1425 Value *Result =
Builder.CreatePtrToInt(Ptr, IntPtrTy,
"extend.cast");
1428 if (IntPtrTy->getBitWidth() == 64)
1429 return RValue::get(Result);
1432 if (getTargetHooks().extendPointerWithSExt())
1433 return RValue::get(
Builder.CreateSExt(Result, Int64Ty,
"extend.sext"));
1435 return RValue::get(
Builder.CreateZExt(Result, Int64Ty,
"extend.zext"));
1437 case Builtin::BI__builtin_setjmp: {
1439 Address Buf = EmitPointerWithAlignment(E->getArg(0));
1443 Builder.CreateCall(CGM.getIntrinsic(Intrinsic::frameaddress),
1444 ConstantInt::get(Int32Ty, 0));
1445 Builder.CreateStore(FrameAddr, Buf);
1449 Builder.CreateCall(CGM.getIntrinsic(Intrinsic::stacksave));
1450 Address StackSaveSlot =
1451 Builder.CreateConstInBoundsGEP(Buf, 2, getPointerSize());
1452 Builder.CreateStore(StackAddr, StackSaveSlot);
1455 Value *F = CGM.getIntrinsic(Intrinsic::eh_sjlj_setjmp);
1456 Buf =
Builder.CreateBitCast(Buf, Int8PtrTy);
1459 case Builtin::BI__builtin_longjmp: {
1460 Value *Buf = EmitScalarExpr(E->getArg(0));
1461 Buf =
Builder.CreateBitCast(Buf, Int8PtrTy);
1464 Builder.CreateCall(CGM.getIntrinsic(Intrinsic::eh_sjlj_longjmp), Buf);
1470 EmitBlock(createBasicBlock(
"longjmp.cont"));
1472 return RValue::get(
nullptr);
1474 case Builtin::BI__sync_fetch_and_add:
1475 case Builtin::BI__sync_fetch_and_sub:
1476 case Builtin::BI__sync_fetch_and_or:
1477 case Builtin::BI__sync_fetch_and_and:
1478 case Builtin::BI__sync_fetch_and_xor:
1479 case Builtin::BI__sync_fetch_and_nand:
1480 case Builtin::BI__sync_add_and_fetch:
1481 case Builtin::BI__sync_sub_and_fetch:
1482 case Builtin::BI__sync_and_and_fetch:
1483 case Builtin::BI__sync_or_and_fetch:
1484 case Builtin::BI__sync_xor_and_fetch:
1485 case Builtin::BI__sync_nand_and_fetch:
1486 case Builtin::BI__sync_val_compare_and_swap:
1487 case Builtin::BI__sync_bool_compare_and_swap:
1488 case Builtin::BI__sync_lock_test_and_set:
1489 case Builtin::BI__sync_lock_release:
1490 case Builtin::BI__sync_swap:
1491 llvm_unreachable(
"Shouldn't make it through sema");
1492 case Builtin::BI__sync_fetch_and_add_1:
1493 case Builtin::BI__sync_fetch_and_add_2:
1494 case Builtin::BI__sync_fetch_and_add_4:
1495 case Builtin::BI__sync_fetch_and_add_8:
1496 case Builtin::BI__sync_fetch_and_add_16:
1498 case Builtin::BI__sync_fetch_and_sub_1:
1499 case Builtin::BI__sync_fetch_and_sub_2:
1500 case Builtin::BI__sync_fetch_and_sub_4:
1501 case Builtin::BI__sync_fetch_and_sub_8:
1502 case Builtin::BI__sync_fetch_and_sub_16:
1504 case Builtin::BI__sync_fetch_and_or_1:
1505 case Builtin::BI__sync_fetch_and_or_2:
1506 case Builtin::BI__sync_fetch_and_or_4:
1507 case Builtin::BI__sync_fetch_and_or_8:
1508 case Builtin::BI__sync_fetch_and_or_16:
1510 case Builtin::BI__sync_fetch_and_and_1:
1511 case Builtin::BI__sync_fetch_and_and_2:
1512 case Builtin::BI__sync_fetch_and_and_4:
1513 case Builtin::BI__sync_fetch_and_and_8:
1514 case Builtin::BI__sync_fetch_and_and_16:
1516 case Builtin::BI__sync_fetch_and_xor_1:
1517 case Builtin::BI__sync_fetch_and_xor_2:
1518 case Builtin::BI__sync_fetch_and_xor_4:
1519 case Builtin::BI__sync_fetch_and_xor_8:
1520 case Builtin::BI__sync_fetch_and_xor_16:
1522 case Builtin::BI__sync_fetch_and_nand_1:
1523 case Builtin::BI__sync_fetch_and_nand_2:
1524 case Builtin::BI__sync_fetch_and_nand_4:
1525 case Builtin::BI__sync_fetch_and_nand_8:
1526 case Builtin::BI__sync_fetch_and_nand_16:
1530 case Builtin::BI__sync_fetch_and_min:
1532 case Builtin::BI__sync_fetch_and_max:
1534 case Builtin::BI__sync_fetch_and_umin:
1536 case Builtin::BI__sync_fetch_and_umax:
1539 case Builtin::BI__sync_add_and_fetch_1:
1540 case Builtin::BI__sync_add_and_fetch_2:
1541 case Builtin::BI__sync_add_and_fetch_4:
1542 case Builtin::BI__sync_add_and_fetch_8:
1543 case Builtin::BI__sync_add_and_fetch_16:
1545 llvm::Instruction::Add);
1546 case Builtin::BI__sync_sub_and_fetch_1:
1547 case Builtin::BI__sync_sub_and_fetch_2:
1548 case Builtin::BI__sync_sub_and_fetch_4:
1549 case Builtin::BI__sync_sub_and_fetch_8:
1550 case Builtin::BI__sync_sub_and_fetch_16:
1552 llvm::Instruction::Sub);
1553 case Builtin::BI__sync_and_and_fetch_1:
1554 case Builtin::BI__sync_and_and_fetch_2:
1555 case Builtin::BI__sync_and_and_fetch_4:
1556 case Builtin::BI__sync_and_and_fetch_8:
1557 case Builtin::BI__sync_and_and_fetch_16:
1560 case Builtin::BI__sync_or_and_fetch_1:
1561 case Builtin::BI__sync_or_and_fetch_2:
1562 case Builtin::BI__sync_or_and_fetch_4:
1563 case Builtin::BI__sync_or_and_fetch_8:
1564 case Builtin::BI__sync_or_and_fetch_16:
1566 llvm::Instruction::Or);
1567 case Builtin::BI__sync_xor_and_fetch_1:
1568 case Builtin::BI__sync_xor_and_fetch_2:
1569 case Builtin::BI__sync_xor_and_fetch_4:
1570 case Builtin::BI__sync_xor_and_fetch_8:
1571 case Builtin::BI__sync_xor_and_fetch_16:
1573 llvm::Instruction::Xor);
1574 case Builtin::BI__sync_nand_and_fetch_1:
1575 case Builtin::BI__sync_nand_and_fetch_2:
1576 case Builtin::BI__sync_nand_and_fetch_4:
1577 case Builtin::BI__sync_nand_and_fetch_8:
1578 case Builtin::BI__sync_nand_and_fetch_16:
1582 case Builtin::BI__sync_val_compare_and_swap_1:
1583 case Builtin::BI__sync_val_compare_and_swap_2:
1584 case Builtin::BI__sync_val_compare_and_swap_4:
1585 case Builtin::BI__sync_val_compare_and_swap_8:
1586 case Builtin::BI__sync_val_compare_and_swap_16:
1589 case Builtin::BI__sync_bool_compare_and_swap_1:
1590 case Builtin::BI__sync_bool_compare_and_swap_2:
1591 case Builtin::BI__sync_bool_compare_and_swap_4:
1592 case Builtin::BI__sync_bool_compare_and_swap_8:
1593 case Builtin::BI__sync_bool_compare_and_swap_16:
1596 case Builtin::BI__sync_swap_1:
1597 case Builtin::BI__sync_swap_2:
1598 case Builtin::BI__sync_swap_4:
1599 case Builtin::BI__sync_swap_8:
1600 case Builtin::BI__sync_swap_16:
1603 case Builtin::BI__sync_lock_test_and_set_1:
1604 case Builtin::BI__sync_lock_test_and_set_2:
1605 case Builtin::BI__sync_lock_test_and_set_4:
1606 case Builtin::BI__sync_lock_test_and_set_8:
1607 case Builtin::BI__sync_lock_test_and_set_16:
1610 case Builtin::BI__sync_lock_release_1:
1611 case Builtin::BI__sync_lock_release_2:
1612 case Builtin::BI__sync_lock_release_4:
1613 case Builtin::BI__sync_lock_release_8:
1614 case Builtin::BI__sync_lock_release_16: {
1615 Value *Ptr = EmitScalarExpr(E->getArg(0));
1617 CharUnits StoreSize = getContext().getTypeSizeInChars(ElTy);
1618 llvm::Type *ITy = llvm::IntegerType::get(getLLVMContext(),
1620 Ptr =
Builder.CreateBitCast(Ptr, ITy->getPointerTo());
1621 llvm::StoreInst *
Store =
1622 Builder.CreateAlignedStore(llvm::Constant::getNullValue(ITy), Ptr,
1624 Store->setAtomic(llvm::AtomicOrdering::Release);
1625 return RValue::get(
nullptr);
1628 case Builtin::BI__sync_synchronize: {
1636 Builder.CreateFence(llvm::AtomicOrdering::SequentiallyConsistent);
1637 return RValue::get(
nullptr);
1640 case Builtin::BI__builtin_nontemporal_load:
1642 case Builtin::BI__builtin_nontemporal_store:
1644 case Builtin::BI__c11_atomic_is_lock_free:
1645 case Builtin::BI__atomic_is_lock_free: {
1649 const char *LibCallName =
"__atomic_is_lock_free";
1651 Args.
add(RValue::get(EmitScalarExpr(E->getArg(0))),
1652 getContext().getSizeType());
1653 if (BuiltinID == Builtin::BI__atomic_is_lock_free)
1654 Args.
add(RValue::get(EmitScalarExpr(E->getArg(1))),
1655 getContext().VoidPtrTy);
1657 Args.
add(RValue::get(llvm::Constant::getNullValue(VoidPtrTy)),
1658 getContext().VoidPtrTy);
1660 CGM.getTypes().arrangeBuiltinFunctionCall(E->
getType(), Args);
1661 llvm::FunctionType *FTy = CGM.getTypes().GetFunctionType(FuncInfo);
1662 llvm::Constant *Func = CGM.CreateRuntimeFunction(FTy, LibCallName);
1663 return EmitCall(FuncInfo, CGCallee::forDirect(Func),
1667 case Builtin::BI__atomic_test_and_set: {
1674 Value *Ptr = EmitScalarExpr(E->getArg(0));
1675 unsigned AddrSpace = Ptr->getType()->getPointerAddressSpace();
1676 Ptr =
Builder.CreateBitCast(Ptr, Int8Ty->getPointerTo(AddrSpace));
1678 Value *Order = EmitScalarExpr(E->getArg(1));
1679 if (isa<llvm::ConstantInt>(Order)) {
1680 int ord = cast<llvm::ConstantInt>(Order)->getZExtValue();
1681 AtomicRMWInst *Result =
nullptr;
1685 Result =
Builder.CreateAtomicRMW(llvm::AtomicRMWInst::Xchg, Ptr, NewVal,
1686 llvm::AtomicOrdering::Monotonic);
1690 Result =
Builder.CreateAtomicRMW(llvm::AtomicRMWInst::Xchg, Ptr, NewVal,
1691 llvm::AtomicOrdering::Acquire);
1694 Result =
Builder.CreateAtomicRMW(llvm::AtomicRMWInst::Xchg, Ptr, NewVal,
1695 llvm::AtomicOrdering::Release);
1699 Result =
Builder.CreateAtomicRMW(llvm::AtomicRMWInst::Xchg, Ptr, NewVal,
1700 llvm::AtomicOrdering::AcquireRelease);
1703 Result =
Builder.CreateAtomicRMW(
1704 llvm::AtomicRMWInst::Xchg, Ptr, NewVal,
1705 llvm::AtomicOrdering::SequentiallyConsistent);
1708 Result->setVolatile(Volatile);
1709 return RValue::get(
Builder.CreateIsNotNull(Result,
"tobool"));
1712 llvm::BasicBlock *ContBB = createBasicBlock(
"atomic.continue", CurFn);
1714 llvm::BasicBlock *BBs[5] = {
1715 createBasicBlock(
"monotonic", CurFn),
1716 createBasicBlock(
"acquire", CurFn),
1717 createBasicBlock(
"release", CurFn),
1718 createBasicBlock(
"acqrel", CurFn),
1719 createBasicBlock(
"seqcst", CurFn)
1721 llvm::AtomicOrdering Orders[5] = {
1722 llvm::AtomicOrdering::Monotonic, llvm::AtomicOrdering::Acquire,
1723 llvm::AtomicOrdering::Release, llvm::AtomicOrdering::AcquireRelease,
1724 llvm::AtomicOrdering::SequentiallyConsistent};
1726 Order =
Builder.CreateIntCast(Order,
Builder.getInt32Ty(),
false);
1727 llvm::SwitchInst *SI =
Builder.CreateSwitch(Order, BBs[0]);
1729 Builder.SetInsertPoint(ContBB);
1730 PHINode *Result =
Builder.CreatePHI(Int8Ty, 5,
"was_set");
1732 for (
unsigned i = 0; i < 5; ++i) {
1733 Builder.SetInsertPoint(BBs[i]);
1734 AtomicRMWInst *RMW =
Builder.CreateAtomicRMW(llvm::AtomicRMWInst::Xchg,
1735 Ptr, NewVal, Orders[i]);
1736 RMW->setVolatile(Volatile);
1737 Result->addIncoming(RMW, BBs[i]);
1741 SI->addCase(
Builder.getInt32(0), BBs[0]);
1742 SI->addCase(
Builder.getInt32(1), BBs[1]);
1743 SI->addCase(
Builder.getInt32(2), BBs[1]);
1744 SI->addCase(
Builder.getInt32(3), BBs[2]);
1745 SI->addCase(
Builder.getInt32(4), BBs[3]);
1746 SI->addCase(
Builder.getInt32(5), BBs[4]);
1748 Builder.SetInsertPoint(ContBB);
1749 return RValue::get(
Builder.CreateIsNotNull(Result,
"tobool"));
1752 case Builtin::BI__atomic_clear: {
1757 Address Ptr = EmitPointerWithAlignment(E->getArg(0));
1758 unsigned AddrSpace = Ptr.
getPointer()->getType()->getPointerAddressSpace();
1759 Ptr =
Builder.CreateBitCast(Ptr, Int8Ty->getPointerTo(AddrSpace));
1761 Value *Order = EmitScalarExpr(E->getArg(1));
1762 if (isa<llvm::ConstantInt>(Order)) {
1763 int ord = cast<llvm::ConstantInt>(Order)->getZExtValue();
1764 StoreInst *Store =
Builder.CreateStore(NewVal, Ptr, Volatile);
1768 Store->setOrdering(llvm::AtomicOrdering::Monotonic);
1771 Store->setOrdering(llvm::AtomicOrdering::Release);
1774 Store->setOrdering(llvm::AtomicOrdering::SequentiallyConsistent);
1777 return RValue::get(
nullptr);
1780 llvm::BasicBlock *ContBB = createBasicBlock(
"atomic.continue", CurFn);
1782 llvm::BasicBlock *BBs[3] = {
1783 createBasicBlock(
"monotonic", CurFn),
1784 createBasicBlock(
"release", CurFn),
1785 createBasicBlock(
"seqcst", CurFn)
1787 llvm::AtomicOrdering Orders[3] = {
1788 llvm::AtomicOrdering::Monotonic, llvm::AtomicOrdering::Release,
1789 llvm::AtomicOrdering::SequentiallyConsistent};
1791 Order =
Builder.CreateIntCast(Order,
Builder.getInt32Ty(),
false);
1792 llvm::SwitchInst *SI =
Builder.CreateSwitch(Order, BBs[0]);
1794 for (
unsigned i = 0; i < 3; ++i) {
1795 Builder.SetInsertPoint(BBs[i]);
1796 StoreInst *Store =
Builder.CreateStore(NewVal, Ptr, Volatile);
1797 Store->setOrdering(Orders[i]);
1801 SI->addCase(
Builder.getInt32(0), BBs[0]);
1802 SI->addCase(
Builder.getInt32(3), BBs[1]);
1803 SI->addCase(
Builder.getInt32(5), BBs[2]);
1805 Builder.SetInsertPoint(ContBB);
1806 return RValue::get(
nullptr);
1809 case Builtin::BI__atomic_thread_fence:
1810 case Builtin::BI__atomic_signal_fence:
1811 case Builtin::BI__c11_atomic_thread_fence:
1812 case Builtin::BI__c11_atomic_signal_fence: {
1814 if (BuiltinID == Builtin::BI__atomic_signal_fence ||
1815 BuiltinID == Builtin::BI__c11_atomic_signal_fence)
1816 SSID = llvm::SyncScope::SingleThread;
1819 Value *Order = EmitScalarExpr(E->getArg(0));
1820 if (isa<llvm::ConstantInt>(Order)) {
1821 int ord = cast<llvm::ConstantInt>(Order)->getZExtValue();
1828 Builder.CreateFence(llvm::AtomicOrdering::Acquire, SSID);
1831 Builder.CreateFence(llvm::AtomicOrdering::Release, SSID);
1834 Builder.CreateFence(llvm::AtomicOrdering::AcquireRelease, SSID);
1837 Builder.CreateFence(llvm::AtomicOrdering::SequentiallyConsistent, SSID);
1840 return RValue::get(
nullptr);
1843 llvm::BasicBlock *AcquireBB, *ReleaseBB, *AcqRelBB, *SeqCstBB;
1844 AcquireBB = createBasicBlock(
"acquire", CurFn);
1845 ReleaseBB = createBasicBlock(
"release", CurFn);
1846 AcqRelBB = createBasicBlock(
"acqrel", CurFn);
1847 SeqCstBB = createBasicBlock(
"seqcst", CurFn);
1848 llvm::BasicBlock *ContBB = createBasicBlock(
"atomic.continue", CurFn);
1850 Order =
Builder.CreateIntCast(Order,
Builder.getInt32Ty(),
false);
1851 llvm::SwitchInst *SI =
Builder.CreateSwitch(Order, ContBB);
1853 Builder.SetInsertPoint(AcquireBB);
1854 Builder.CreateFence(llvm::AtomicOrdering::Acquire, SSID);
1856 SI->addCase(
Builder.getInt32(1), AcquireBB);
1857 SI->addCase(
Builder.getInt32(2), AcquireBB);
1859 Builder.SetInsertPoint(ReleaseBB);
1860 Builder.CreateFence(llvm::AtomicOrdering::Release, SSID);
1862 SI->addCase(
Builder.getInt32(3), ReleaseBB);
1864 Builder.SetInsertPoint(AcqRelBB);
1865 Builder.CreateFence(llvm::AtomicOrdering::AcquireRelease, SSID);
1867 SI->addCase(
Builder.getInt32(4), AcqRelBB);
1869 Builder.SetInsertPoint(SeqCstBB);
1870 Builder.CreateFence(llvm::AtomicOrdering::SequentiallyConsistent, SSID);
1872 SI->addCase(
Builder.getInt32(5), SeqCstBB);
1874 Builder.SetInsertPoint(ContBB);
1875 return RValue::get(
nullptr);
1879 case Builtin::BIsqrt:
1880 case Builtin::BIsqrtf:
1881 case Builtin::BIsqrtl: {
1886 if (!FD->hasAttr<ConstAttr>())
1888 if (!(CGM.getCodeGenOpts().UnsafeFPMath ||
1889 CGM.getCodeGenOpts().NoNaNsFPMath))
1891 Value *Arg0 = EmitScalarExpr(E->getArg(0));
1892 llvm::Type *ArgType = Arg0->getType();
1894 return RValue::get(
Builder.CreateCall(F, Arg0));
1897 case Builtin::BI__builtin_pow:
1898 case Builtin::BI__builtin_powf:
1899 case Builtin::BI__builtin_powl:
1900 case Builtin::BIpow:
1901 case Builtin::BIpowf:
1902 case Builtin::BIpowl: {
1904 if (!FD->hasAttr<ConstAttr>())
1906 Value *Base = EmitScalarExpr(E->getArg(0));
1907 Value *Exponent = EmitScalarExpr(E->getArg(1));
1908 llvm::Type *ArgType = Base->getType();
1910 return RValue::get(
Builder.CreateCall(F, {Base, Exponent}));
1913 case Builtin::BIfma:
1914 case Builtin::BIfmaf:
1915 case Builtin::BIfmal:
1916 case Builtin::BI__builtin_fma:
1917 case Builtin::BI__builtin_fmaf:
1918 case Builtin::BI__builtin_fmal: {
1920 Value *FirstArg = EmitScalarExpr(E->getArg(0));
1921 llvm::Type *ArgType = FirstArg->getType();
1924 Builder.CreateCall(F, {FirstArg, EmitScalarExpr(E->getArg(1)),
1925 EmitScalarExpr(E->getArg(2))}));
1928 case Builtin::BI__builtin_signbit:
1929 case Builtin::BI__builtin_signbitf:
1930 case Builtin::BI__builtin_signbitl: {
1935 case Builtin::BI__builtin_annotation: {
1936 llvm::Value *AnnVal = EmitScalarExpr(E->getArg(0));
1937 llvm::Value *F = CGM.getIntrinsic(llvm::Intrinsic::annotation,
1943 StringRef Str = cast<StringLiteral>(AnnotationStrExpr)->getString();
1944 return RValue::get(EmitAnnotationCall(F, AnnVal, Str, E->
getExprLoc()));
1946 case Builtin::BI__builtin_addcb:
1947 case Builtin::BI__builtin_addcs:
1948 case Builtin::BI__builtin_addc:
1949 case Builtin::BI__builtin_addcl:
1950 case Builtin::BI__builtin_addcll:
1951 case Builtin::BI__builtin_subcb:
1952 case Builtin::BI__builtin_subcs:
1953 case Builtin::BI__builtin_subc:
1954 case Builtin::BI__builtin_subcl:
1955 case Builtin::BI__builtin_subcll: {
1977 llvm::Value *Carryin = EmitScalarExpr(E->getArg(2));
1978 Address CarryOutPtr = EmitPointerWithAlignment(E->getArg(3));
1982 switch (BuiltinID) {
1983 default: llvm_unreachable(
"Unknown multiprecision builtin id.");
1984 case Builtin::BI__builtin_addcb:
1985 case Builtin::BI__builtin_addcs:
1986 case Builtin::BI__builtin_addc:
1987 case Builtin::BI__builtin_addcl:
1988 case Builtin::BI__builtin_addcll:
1989 IntrinsicId = llvm::Intrinsic::uadd_with_overflow;
1991 case Builtin::BI__builtin_subcb:
1992 case Builtin::BI__builtin_subcs:
1993 case Builtin::BI__builtin_subc:
1994 case Builtin::BI__builtin_subcl:
1995 case Builtin::BI__builtin_subcll:
1996 IntrinsicId = llvm::Intrinsic::usub_with_overflow;
2006 Sum1, Carryin, Carry2);
2009 Builder.CreateStore(CarryOut, CarryOutPtr);
2010 return RValue::get(Sum2);
2013 case Builtin::BI__builtin_add_overflow:
2014 case Builtin::BI__builtin_sub_overflow:
2015 case Builtin::BI__builtin_mul_overflow: {
2023 WidthAndSignedness LeftInfo =
2025 WidthAndSignedness RightInfo =
2027 WidthAndSignedness ResultInfo =
2029 WidthAndSignedness EncompassingInfo =
2032 llvm::Type *EncompassingLLVMTy =
2033 llvm::IntegerType::get(CGM.getLLVMContext(), EncompassingInfo.Width);
2035 llvm::Type *ResultLLVMTy = CGM.getTypes().ConvertType(ResultQTy);
2038 switch (BuiltinID) {
2040 llvm_unreachable(
"Unknown overflow builtin id.");
2041 case Builtin::BI__builtin_add_overflow:
2042 IntrinsicId = EncompassingInfo.Signed
2043 ? llvm::Intrinsic::sadd_with_overflow
2044 : llvm::Intrinsic::uadd_with_overflow;
2046 case Builtin::BI__builtin_sub_overflow:
2047 IntrinsicId = EncompassingInfo.Signed
2048 ? llvm::Intrinsic::ssub_with_overflow
2049 : llvm::Intrinsic::usub_with_overflow;
2051 case Builtin::BI__builtin_mul_overflow:
2052 IntrinsicId = EncompassingInfo.Signed
2053 ? llvm::Intrinsic::smul_with_overflow
2054 : llvm::Intrinsic::umul_with_overflow;
2060 Address ResultPtr = EmitPointerWithAlignment(ResultArg);
2063 Left =
Builder.CreateIntCast(Left, EncompassingLLVMTy, LeftInfo.Signed);
2064 Right =
Builder.CreateIntCast(Right, EncompassingLLVMTy, RightInfo.Signed);
2070 if (EncompassingInfo.Width > ResultInfo.Width) {
2078 ResultTrunc, EncompassingLLVMTy, ResultInfo.Signed);
2080 Builder.CreateICmpNE(Result, ResultTruncExt);
2082 Overflow =
Builder.CreateOr(Overflow, TruncationOverflow);
2083 Result = ResultTrunc;
2089 Builder.CreateStore(EmitToMemory(Result, ResultQTy), ResultPtr, isVolatile);
2091 return RValue::get(Overflow);
2094 case Builtin::BI__builtin_uadd_overflow:
2095 case Builtin::BI__builtin_uaddl_overflow:
2096 case Builtin::BI__builtin_uaddll_overflow:
2097 case Builtin::BI__builtin_usub_overflow:
2098 case Builtin::BI__builtin_usubl_overflow:
2099 case Builtin::BI__builtin_usubll_overflow:
2100 case Builtin::BI__builtin_umul_overflow:
2101 case Builtin::BI__builtin_umull_overflow:
2102 case Builtin::BI__builtin_umulll_overflow:
2103 case Builtin::BI__builtin_sadd_overflow:
2104 case Builtin::BI__builtin_saddl_overflow:
2105 case Builtin::BI__builtin_saddll_overflow:
2106 case Builtin::BI__builtin_ssub_overflow:
2107 case Builtin::BI__builtin_ssubl_overflow:
2108 case Builtin::BI__builtin_ssubll_overflow:
2109 case Builtin::BI__builtin_smul_overflow:
2110 case Builtin::BI__builtin_smull_overflow:
2111 case Builtin::BI__builtin_smulll_overflow: {
2118 Address SumOutPtr = EmitPointerWithAlignment(E->getArg(2));
2122 switch (BuiltinID) {
2123 default: llvm_unreachable(
"Unknown overflow builtin id.");
2124 case Builtin::BI__builtin_uadd_overflow:
2125 case Builtin::BI__builtin_uaddl_overflow:
2126 case Builtin::BI__builtin_uaddll_overflow:
2127 IntrinsicId = llvm::Intrinsic::uadd_with_overflow;
2129 case Builtin::BI__builtin_usub_overflow:
2130 case Builtin::BI__builtin_usubl_overflow:
2131 case Builtin::BI__builtin_usubll_overflow:
2132 IntrinsicId = llvm::Intrinsic::usub_with_overflow;
2134 case Builtin::BI__builtin_umul_overflow:
2135 case Builtin::BI__builtin_umull_overflow:
2136 case Builtin::BI__builtin_umulll_overflow:
2137 IntrinsicId = llvm::Intrinsic::umul_with_overflow;
2139 case Builtin::BI__builtin_sadd_overflow:
2140 case Builtin::BI__builtin_saddl_overflow:
2141 case Builtin::BI__builtin_saddll_overflow:
2142 IntrinsicId = llvm::Intrinsic::sadd_with_overflow;
2144 case Builtin::BI__builtin_ssub_overflow:
2145 case Builtin::BI__builtin_ssubl_overflow:
2146 case Builtin::BI__builtin_ssubll_overflow:
2147 IntrinsicId = llvm::Intrinsic::ssub_with_overflow;
2149 case Builtin::BI__builtin_smul_overflow:
2150 case Builtin::BI__builtin_smull_overflow:
2151 case Builtin::BI__builtin_smulll_overflow:
2152 IntrinsicId = llvm::Intrinsic::smul_with_overflow;
2159 Builder.CreateStore(Sum, SumOutPtr);
2161 return RValue::get(Carry);
2163 case Builtin::BI__builtin_addressof:
2164 return RValue::get(EmitLValue(E->getArg(0)).getPointer());
2165 case Builtin::BI__builtin_operator_new:
2167 E->getArg(0),
false);
2168 case Builtin::BI__builtin_operator_delete:
2170 E->getArg(0),
true);
2171 case Builtin::BI__noop:
2173 return RValue::get(ConstantInt::get(IntTy, 0));
2174 case Builtin::BI__builtin_call_with_static_chain: {
2176 const Expr *Chain = E->getArg(1);
2178 EmitCallee(Call->
getCallee()), Call, ReturnValue,
2179 EmitScalarExpr(Chain));
2181 case Builtin::BI_InterlockedExchange8:
2182 case Builtin::BI_InterlockedExchange16:
2183 case Builtin::BI_InterlockedExchange:
2184 case Builtin::BI_InterlockedExchangePointer:
2186 EmitMSVCBuiltinExpr(MSVCIntrin::_InterlockedExchange, E));
2187 case Builtin::BI_InterlockedCompareExchangePointer: {
2189 llvm::IntegerType *IntType =
2190 IntegerType::get(getLLVMContext(),
2191 getContext().getTypeSize(E->
getType()));
2192 llvm::Type *IntPtrType = IntType->getPointerTo();
2195 Builder.CreateBitCast(EmitScalarExpr(E->getArg(0)), IntPtrType);
2197 llvm::Value *Exchange = EmitScalarExpr(E->getArg(1));
2198 RTy = Exchange->getType();
2199 Exchange =
Builder.CreatePtrToInt(Exchange, IntType);
2202 Builder.CreatePtrToInt(EmitScalarExpr(E->getArg(2)), IntType);
2205 Builder.CreateAtomicCmpXchg(Destination, Comparand, Exchange,
2206 AtomicOrdering::SequentiallyConsistent,
2207 AtomicOrdering::SequentiallyConsistent);
2208 Result->setVolatile(
true);
2210 return RValue::get(
Builder.CreateIntToPtr(
Builder.CreateExtractValue(Result,
2214 case Builtin::BI_InterlockedCompareExchange8:
2215 case Builtin::BI_InterlockedCompareExchange16:
2216 case Builtin::BI_InterlockedCompareExchange:
2217 case Builtin::BI_InterlockedCompareExchange64: {
2218 AtomicCmpXchgInst *CXI =
Builder.CreateAtomicCmpXchg(
2219 EmitScalarExpr(E->getArg(0)),
2220 EmitScalarExpr(E->getArg(2)),
2221 EmitScalarExpr(E->getArg(1)),
2222 AtomicOrdering::SequentiallyConsistent,
2223 AtomicOrdering::SequentiallyConsistent);
2224 CXI->setVolatile(
true);
2225 return RValue::get(
Builder.CreateExtractValue(CXI, 0));
2227 case Builtin::BI_InterlockedIncrement16:
2228 case Builtin::BI_InterlockedIncrement:
2230 EmitMSVCBuiltinExpr(MSVCIntrin::_InterlockedIncrement, E));
2231 case Builtin::BI_InterlockedDecrement16:
2232 case Builtin::BI_InterlockedDecrement:
2234 EmitMSVCBuiltinExpr(MSVCIntrin::_InterlockedDecrement, E));
2235 case Builtin::BI_InterlockedAnd8:
2236 case Builtin::BI_InterlockedAnd16:
2237 case Builtin::BI_InterlockedAnd:
2238 return RValue::get(EmitMSVCBuiltinExpr(MSVCIntrin::_InterlockedAnd, E));
2239 case Builtin::BI_InterlockedExchangeAdd8:
2240 case Builtin::BI_InterlockedExchangeAdd16:
2241 case Builtin::BI_InterlockedExchangeAdd:
2243 EmitMSVCBuiltinExpr(MSVCIntrin::_InterlockedExchangeAdd, E));
2244 case Builtin::BI_InterlockedExchangeSub8:
2245 case Builtin::BI_InterlockedExchangeSub16:
2246 case Builtin::BI_InterlockedExchangeSub:
2248 EmitMSVCBuiltinExpr(MSVCIntrin::_InterlockedExchangeSub, E));
2249 case Builtin::BI_InterlockedOr8:
2250 case Builtin::BI_InterlockedOr16:
2251 case Builtin::BI_InterlockedOr:
2252 return RValue::get(EmitMSVCBuiltinExpr(MSVCIntrin::_InterlockedOr, E));
2253 case Builtin::BI_InterlockedXor8:
2254 case Builtin::BI_InterlockedXor16:
2255 case Builtin::BI_InterlockedXor:
2256 return RValue::get(EmitMSVCBuiltinExpr(MSVCIntrin::_InterlockedXor, E));
2257 case Builtin::BI_interlockedbittestandset:
2259 EmitMSVCBuiltinExpr(MSVCIntrin::_interlockedbittestandset, E));
2261 case Builtin::BI__exception_code:
2262 case Builtin::BI_exception_code:
2263 return RValue::get(EmitSEHExceptionCode());
2264 case Builtin::BI__exception_info:
2265 case Builtin::BI_exception_info:
2266 return RValue::get(EmitSEHExceptionInfo());
2267 case Builtin::BI__abnormal_termination:
2268 case Builtin::BI_abnormal_termination:
2269 return RValue::get(EmitSEHAbnormalTermination());
2270 case Builtin::BI_setjmpex: {
2271 if (getTarget().getTriple().isOSMSVCRT()) {
2272 llvm::Type *ArgTypes[] = {Int8PtrTy, Int8PtrTy};
2273 llvm::AttributeList ReturnsTwiceAttr = llvm::AttributeList::get(
2274 getLLVMContext(), llvm::AttributeList::FunctionIndex,
2275 llvm::Attribute::ReturnsTwice);
2276 llvm::Constant *SetJmpEx = CGM.CreateRuntimeFunction(
2277 llvm::FunctionType::get(IntTy, ArgTypes,
false),
2278 "_setjmpex", ReturnsTwiceAttr,
true);
2280 EmitScalarExpr(E->getArg(0)), Int8PtrTy);
2282 Builder.CreateCall(CGM.getIntrinsic(Intrinsic::frameaddress),
2283 ConstantInt::get(Int32Ty, 0));
2285 llvm::CallSite CS = EmitRuntimeCallOrInvoke(SetJmpEx, Args);
2286 CS.setAttributes(ReturnsTwiceAttr);
2287 return RValue::get(CS.getInstruction());
2291 case Builtin::BI_setjmp: {
2292 if (getTarget().getTriple().isOSMSVCRT()) {
2293 llvm::AttributeList ReturnsTwiceAttr = llvm::AttributeList::get(
2294 getLLVMContext(), llvm::AttributeList::FunctionIndex,
2295 llvm::Attribute::ReturnsTwice);
2297 EmitScalarExpr(E->getArg(0)), Int8PtrTy);
2299 if (getTarget().getTriple().getArch() == llvm::Triple::x86) {
2300 llvm::Type *ArgTypes[] = {Int8PtrTy, IntTy};
2301 llvm::Constant *SetJmp3 = CGM.CreateRuntimeFunction(
2302 llvm::FunctionType::get(IntTy, ArgTypes,
true),
2303 "_setjmp3", ReturnsTwiceAttr,
true);
2306 CS = EmitRuntimeCallOrInvoke(SetJmp3, Args);
2308 llvm::Type *ArgTypes[] = {Int8PtrTy, Int8PtrTy};
2309 llvm::Constant *SetJmp = CGM.CreateRuntimeFunction(
2310 llvm::FunctionType::get(IntTy, ArgTypes,
false),
2311 "_setjmp", ReturnsTwiceAttr,
true);
2313 Builder.CreateCall(CGM.getIntrinsic(Intrinsic::frameaddress),
2314 ConstantInt::get(Int32Ty, 0));
2316 CS = EmitRuntimeCallOrInvoke(SetJmp, Args);
2318 CS.setAttributes(ReturnsTwiceAttr);
2319 return RValue::get(CS.getInstruction());
2324 case Builtin::BI__GetExceptionInfo: {
2325 if (llvm::GlobalVariable *GV =
2326 CGM.getCXXABI().getThrowInfo(FD->getParamDecl(0)->getType()))
2327 return RValue::get(llvm::ConstantExpr::getBitCast(GV, CGM.Int8PtrTy));
2331 case Builtin::BI__fastfail:
2332 return RValue::get(EmitMSVCBuiltinExpr(MSVCIntrin::__fastfail, E));
2334 case Builtin::BI__builtin_coro_size: {
2335 auto &
Context = getContext();
2338 Value *F = CGM.getIntrinsic(Intrinsic::coro_size, T);
2339 return RValue::get(
Builder.CreateCall(F));
2342 case Builtin::BI__builtin_coro_id:
2343 return EmitCoroutineIntrinsic(E, Intrinsic::coro_id);
2344 case Builtin::BI__builtin_coro_promise:
2345 return EmitCoroutineIntrinsic(E, Intrinsic::coro_promise);
2346 case Builtin::BI__builtin_coro_resume:
2347 return EmitCoroutineIntrinsic(E, Intrinsic::coro_resume);
2348 case Builtin::BI__builtin_coro_frame:
2349 return EmitCoroutineIntrinsic(E, Intrinsic::coro_frame);
2350 case Builtin::BI__builtin_coro_free:
2351 return EmitCoroutineIntrinsic(E, Intrinsic::coro_free);
2352 case Builtin::BI__builtin_coro_destroy:
2353 return EmitCoroutineIntrinsic(E, Intrinsic::coro_destroy);
2354 case Builtin::BI__builtin_coro_done:
2355 return EmitCoroutineIntrinsic(E, Intrinsic::coro_done);
2356 case Builtin::BI__builtin_coro_alloc:
2357 return EmitCoroutineIntrinsic(E, Intrinsic::coro_alloc);
2358 case Builtin::BI__builtin_coro_begin:
2359 return EmitCoroutineIntrinsic(E, Intrinsic::coro_begin);
2360 case Builtin::BI__builtin_coro_end:
2361 return EmitCoroutineIntrinsic(E, Intrinsic::coro_end);
2362 case Builtin::BI__builtin_coro_suspend:
2363 return EmitCoroutineIntrinsic(E, Intrinsic::coro_suspend);
2364 case Builtin::BI__builtin_coro_param:
2365 return EmitCoroutineIntrinsic(E, Intrinsic::coro_param);
2368 case Builtin::BIread_pipe:
2369 case Builtin::BIwrite_pipe: {
2370 Value *Arg0 = EmitScalarExpr(E->getArg(0)),
2371 *Arg1 = EmitScalarExpr(E->getArg(1));
2373 Value *PacketSize = OpenCLRT.getPipeElemSize(E->getArg(0));
2374 Value *PacketAlign = OpenCLRT.getPipeElemAlign(E->getArg(0));
2377 unsigned GenericAS =
2379 llvm::Type *I8PTy = llvm::PointerType::get(
2380 llvm::Type::getInt8Ty(getLLVMContext()), GenericAS);
2383 if (2U == E->getNumArgs()) {
2384 const char *
Name = (BuiltinID == Builtin::BIread_pipe) ?
"__read_pipe_2"
2388 llvm::Type *ArgTys[] = {Arg0->getType(), I8PTy, Int32Ty, Int32Ty};
2389 llvm::FunctionType *FTy = llvm::FunctionType::get(
2393 Builder.CreateCall(CGM.CreateRuntimeFunction(FTy, Name),
2394 {Arg0, BCast, PacketSize, PacketAlign}));
2396 assert(4 == E->getNumArgs() &&
2397 "Illegal number of parameters to pipe function");
2398 const char *Name = (BuiltinID == Builtin::BIread_pipe) ?
"__read_pipe_4"
2401 llvm::Type *ArgTys[] = {Arg0->getType(), Arg1->getType(), Int32Ty, I8PTy,
2403 Value *Arg2 = EmitScalarExpr(E->getArg(2)),
2404 *Arg3 = EmitScalarExpr(E->getArg(3));
2405 llvm::FunctionType *FTy = llvm::FunctionType::get(
2410 if (Arg2->getType() != Int32Ty)
2411 Arg2 =
Builder.CreateZExtOrTrunc(Arg2, Int32Ty);
2412 return RValue::get(
Builder.CreateCall(
2413 CGM.CreateRuntimeFunction(FTy, Name),
2414 {Arg0, Arg1, Arg2, BCast, PacketSize, PacketAlign}));
2419 case Builtin::BIreserve_read_pipe:
2420 case Builtin::BIreserve_write_pipe:
2421 case Builtin::BIwork_group_reserve_read_pipe:
2422 case Builtin::BIwork_group_reserve_write_pipe:
2423 case Builtin::BIsub_group_reserve_read_pipe:
2424 case Builtin::BIsub_group_reserve_write_pipe: {
2427 if (BuiltinID == Builtin::BIreserve_read_pipe)
2428 Name =
"__reserve_read_pipe";
2429 else if (BuiltinID == Builtin::BIreserve_write_pipe)
2430 Name =
"__reserve_write_pipe";
2431 else if (BuiltinID == Builtin::BIwork_group_reserve_read_pipe)
2432 Name =
"__work_group_reserve_read_pipe";
2433 else if (BuiltinID == Builtin::BIwork_group_reserve_write_pipe)
2434 Name =
"__work_group_reserve_write_pipe";
2435 else if (BuiltinID == Builtin::BIsub_group_reserve_read_pipe)
2436 Name =
"__sub_group_reserve_read_pipe";
2438 Name =
"__sub_group_reserve_write_pipe";
2440 Value *Arg0 = EmitScalarExpr(E->getArg(0)),
2441 *Arg1 = EmitScalarExpr(E->getArg(1));
2442 llvm::Type *ReservedIDTy = ConvertType(getContext().OCLReserveIDTy);
2444 Value *PacketSize = OpenCLRT.getPipeElemSize(E->getArg(0));
2445 Value *PacketAlign = OpenCLRT.getPipeElemAlign(E->getArg(0));
2448 llvm::Type *ArgTys[] = {Arg0->getType(), Int32Ty, Int32Ty, Int32Ty};
2449 llvm::FunctionType *FTy = llvm::FunctionType::get(
2453 if (Arg1->getType() != Int32Ty)
2454 Arg1 =
Builder.CreateZExtOrTrunc(Arg1, Int32Ty);
2456 Builder.CreateCall(CGM.CreateRuntimeFunction(FTy, Name),
2457 {Arg0, Arg1, PacketSize, PacketAlign}));
2461 case Builtin::BIcommit_read_pipe:
2462 case Builtin::BIcommit_write_pipe:
2463 case Builtin::BIwork_group_commit_read_pipe:
2464 case Builtin::BIwork_group_commit_write_pipe:
2465 case Builtin::BIsub_group_commit_read_pipe:
2466 case Builtin::BIsub_group_commit_write_pipe: {
2468 if (BuiltinID == Builtin::BIcommit_read_pipe)
2469 Name =
"__commit_read_pipe";
2470 else if (BuiltinID == Builtin::BIcommit_write_pipe)
2471 Name =
"__commit_write_pipe";
2472 else if (BuiltinID == Builtin::BIwork_group_commit_read_pipe)
2473 Name =
"__work_group_commit_read_pipe";
2474 else if (BuiltinID == Builtin::BIwork_group_commit_write_pipe)
2475 Name =
"__work_group_commit_write_pipe";
2476 else if (BuiltinID == Builtin::BIsub_group_commit_read_pipe)
2477 Name =
"__sub_group_commit_read_pipe";
2479 Name =
"__sub_group_commit_write_pipe";
2481 Value *Arg0 = EmitScalarExpr(E->getArg(0)),
2482 *Arg1 = EmitScalarExpr(E->getArg(1));
2484 Value *PacketSize = OpenCLRT.getPipeElemSize(E->getArg(0));
2485 Value *PacketAlign = OpenCLRT.getPipeElemAlign(E->getArg(0));
2488 llvm::Type *ArgTys[] = {Arg0->getType(), Arg1->getType(), Int32Ty, Int32Ty};
2489 llvm::FunctionType *FTy =
2490 llvm::FunctionType::get(llvm::Type::getVoidTy(getLLVMContext()),
2494 Builder.CreateCall(CGM.CreateRuntimeFunction(FTy, Name),
2495 {Arg0, Arg1, PacketSize, PacketAlign}));
2498 case Builtin::BIget_pipe_num_packets:
2499 case Builtin::BIget_pipe_max_packets: {
2501 if (BuiltinID == Builtin::BIget_pipe_num_packets)
2502 Name =
"__get_pipe_num_packets";
2504 Name =
"__get_pipe_max_packets";
2507 Value *Arg0 = EmitScalarExpr(E->getArg(0));
2509 Value *PacketSize = OpenCLRT.getPipeElemSize(E->getArg(0));
2510 Value *PacketAlign = OpenCLRT.getPipeElemAlign(E->getArg(0));
2511 llvm::Type *ArgTys[] = {Arg0->getType(), Int32Ty, Int32Ty};
2512 llvm::FunctionType *FTy = llvm::FunctionType::get(
2515 return RValue::get(
Builder.CreateCall(CGM.CreateRuntimeFunction(FTy, Name),
2516 {Arg0, PacketSize, PacketAlign}));
2520 case Builtin::BIto_global:
2521 case Builtin::BIto_local:
2522 case Builtin::BIto_private: {
2523 auto Arg0 = EmitScalarExpr(E->getArg(0));
2524 auto NewArgT = llvm::PointerType::get(Int8Ty,
2526 auto NewRetT = llvm::PointerType::get(Int8Ty,
2527 CGM.getContext().getTargetAddressSpace(
2529 auto FTy = llvm::FunctionType::get(NewRetT, {NewArgT},
false);
2531 if (Arg0->getType()->getPointerAddressSpace() !=
2532 NewArgT->getPointerAddressSpace())
2533 NewArg =
Builder.CreateAddrSpaceCast(Arg0, NewArgT);
2535 NewArg =
Builder.CreateBitOrPointerCast(Arg0, NewArgT);
2536 auto NewName = std::string(
"__") + E->getDirectCallee()->getName().str();
2538 Builder.CreateCall(CGM.CreateRuntimeFunction(FTy, NewName), {NewArg});
2539 return RValue::get(
Builder.CreateBitOrPointerCast(NewCall,
2545 case Builtin::BIenqueue_kernel: {
2547 unsigned NumArgs = E->getNumArgs();
2549 llvm::Type *QueueTy = ConvertType(getContext().OCLQueueTy);
2550 llvm::Type *GenericVoidPtrTy =
Builder.getInt8PtrTy(
2553 llvm::Value *Queue = EmitScalarExpr(E->getArg(0));
2554 llvm::Value *Flags = EmitScalarExpr(E->getArg(1));
2555 LValue NDRangeL = EmitAggExprToLValue(E->getArg(2));
2556 llvm::Value *Range = NDRangeL.getAddress().getPointer();
2557 llvm::Type *RangeTy = NDRangeL.getAddress().getType();
2562 Name =
"__enqueue_kernel_basic";
2563 llvm::Type *ArgTys[] = {QueueTy, Int32Ty, RangeTy, GenericVoidPtrTy};
2564 llvm::FunctionType *FTy = llvm::FunctionType::get(
2568 EmitScalarExpr(E->getArg(3)), GenericVoidPtrTy);
2571 B.addAttribute(Attribute::ByVal);
2572 llvm::AttributeList ByValAttrSet =
2573 llvm::AttributeList::get(CGM.getModule().getContext(), 3U, B);
2576 Builder.CreateCall(CGM.CreateRuntimeFunction(FTy, Name, ByValAttrSet),
2577 {Queue, Flags, Range, Block});
2578 RTCall->setAttributes(ByValAttrSet);
2579 return RValue::get(RTCall);
2581 assert(NumArgs >= 5 &&
"Invalid enqueue_kernel signature");
2586 Name =
"__enqueue_kernel_vaargs";
2588 EmitScalarExpr(E->getArg(3)), GenericVoidPtrTy);
2591 std::vector<llvm::Value *> Args = {Queue, Flags, Range, Block,
2592 ConstantInt::get(IntTy, NumArgs - 4)};
2593 std::vector<llvm::Type *> ArgTys = {QueueTy, IntTy, RangeTy,
2594 GenericVoidPtrTy, IntTy};
2598 for (
unsigned I = 4;
I < NumArgs; ++
I)
2600 Builder.CreateZExtOrTrunc(EmitScalarExpr(E->getArg(
I)), SizeTy));
2602 llvm::FunctionType *FTy = llvm::FunctionType::get(
2605 Builder.CreateCall(CGM.CreateRuntimeFunction(FTy, Name),
2610 llvm::Type *EventTy = ConvertType(getContext().OCLClkEventTy);
2611 llvm::Type *EventPtrTy = EventTy->getPointerTo(
2615 Builder.CreateZExtOrTrunc(EmitScalarExpr(E->getArg(3)), Int32Ty);
2618 ? EmitArrayToPointerDecay(E->getArg(4)).getPointer()
2619 : EmitScalarExpr(E->getArg(4));
2620 llvm::Value *ClkEvent = EmitScalarExpr(E->getArg(5));
2622 EventList =
Builder.CreatePointerCast(EventList, EventPtrTy);
2623 ClkEvent =
Builder.CreatePointerCast(ClkEvent, EventPtrTy);
2625 EmitScalarExpr(E->getArg(6)), GenericVoidPtrTy);
2627 std::vector<llvm::Type *> ArgTys = {
2628 QueueTy, Int32Ty, RangeTy, Int32Ty,
2629 EventPtrTy, EventPtrTy, GenericVoidPtrTy};
2631 std::vector<llvm::Value *> Args = {Queue, Flags, Range, NumEvents,
2632 EventList, ClkEvent, Block};
2636 Name =
"__enqueue_kernel_basic_events";
2637 llvm::FunctionType *FTy = llvm::FunctionType::get(
2640 Builder.CreateCall(CGM.CreateRuntimeFunction(FTy, Name),
2645 Args.push_back(ConstantInt::get(Int32Ty, NumArgs - 7));
2646 ArgTys.push_back(Int32Ty);
2647 Name =
"__enqueue_kernel_events_vaargs";
2651 for (
unsigned I = 7;
I < NumArgs; ++
I)
2653 Builder.CreateZExtOrTrunc(EmitScalarExpr(E->getArg(
I)), SizeTy));
2655 llvm::FunctionType *FTy = llvm::FunctionType::get(
2658 Builder.CreateCall(CGM.CreateRuntimeFunction(FTy, Name),
2665 case Builtin::BIget_kernel_work_group_size: {
2666 llvm::Type *GenericVoidPtrTy =
Builder.getInt8PtrTy(
2668 Value *Arg = EmitScalarExpr(E->getArg(0));
2669 Arg =
Builder.CreatePointerCast(Arg, GenericVoidPtrTy);
2670 return RValue::get(
Builder.CreateCall(
2671 CGM.CreateRuntimeFunction(
2672 llvm::FunctionType::get(IntTy, GenericVoidPtrTy,
false),
2673 "__get_kernel_work_group_size_impl"),
2676 case Builtin::BIget_kernel_preferred_work_group_size_multiple: {
2677 llvm::Type *GenericVoidPtrTy =
Builder.getInt8PtrTy(
2679 Value *Arg = EmitScalarExpr(E->getArg(0));
2680 Arg =
Builder.CreatePointerCast(Arg, GenericVoidPtrTy);
2681 return RValue::get(
Builder.CreateCall(
2682 CGM.CreateRuntimeFunction(
2683 llvm::FunctionType::get(IntTy, GenericVoidPtrTy,
false),
2684 "__get_kernel_preferred_work_group_multiple_impl"),
2687 case Builtin::BIprintf:
2688 if (getTarget().getTriple().isNVPTX())
2689 return EmitNVPTXDevicePrintfCallExpr(E, ReturnValue);
2691 case Builtin::BI__builtin_canonicalize:
2692 case Builtin::BI__builtin_canonicalizef:
2693 case Builtin::BI__builtin_canonicalizel:
2696 case Builtin::BI__builtin_thread_pointer: {
2697 if (!getContext().getTargetInfo().isTLSSupported())
2698 CGM.ErrorUnsupported(E,
"__builtin_thread_pointer");
2702 case Builtin::BI__builtin_os_log_format: {
2703 assert(E->getNumArgs() >= 2 &&
2704 "__builtin_os_log_format takes at least 2 arguments");
2707 Address BufAddr = EmitPointerWithAlignment(E->getArg(0));
2711 Builder.getInt8(Layout.getSummaryByte()),
2712 Builder.CreateConstByteGEP(BufAddr, Offset++,
"summary"));
2714 Builder.getInt8(Layout.getNumArgsByte()),
2715 Builder.CreateConstByteGEP(BufAddr, Offset++,
"numArgs"));
2718 for (
const auto &Item : Layout.Items) {
2720 Builder.getInt8(Item.getDescriptorByte()),
2721 Builder.CreateConstByteGEP(BufAddr, Offset++,
"argDescriptor"));
2723 Builder.getInt8(Item.getSizeByte()),
2724 Builder.CreateConstByteGEP(BufAddr, Offset++,
"argSize"));
2725 Address Addr =
Builder.CreateConstByteGEP(BufAddr, Offset);
2726 if (
const Expr *TheExpr = Item.getExpr()) {
2727 Addr =
Builder.CreateElementBitCast(
2728 Addr, ConvertTypeForMem(TheExpr->getType()));
2730 if (TheExpr->getType()->isObjCRetainableType()) {
2731 assert(getEvaluationKind(TheExpr->getType()) ==
TEK_Scalar &&
2732 "Only scalar can be a ObjC retainable type");
2734 RValue RV = RValue::get(SV);
2736 EmitStoreThroughLValue(RV, LV);
2739 if (!isa<Constant>(SV))
2740 RetainableOperands.push_back(SV);
2742 EmitAnyExprToMem(TheExpr, Addr,
Qualifiers(),
true);
2745 Addr =
Builder.CreateElementBitCast(Addr, Int32Ty);
2747 Builder.getInt32(Item.getConstValue().getQuantity()), Addr);
2749 Offset += Item.size();
2758 if (!RetainableOperands.empty() && getLangOpts().ObjCAutoRefCount &&
2759 CGM.getCodeGenOpts().OptimizationLevel != 0)
2761 pushFullExprCleanup<CallObjCArcUse>(getARCCleanupKind(), object);
2763 return RValue::get(BufAddr.getPointer());
2766 case Builtin::BI__builtin_os_log_format_buffer_size: {
2769 return RValue::get(ConstantInt::get(ConvertType(E->
getType()),
2773 case Builtin::BI__xray_customevent: {
2774 if (!ShouldXRayInstrumentFunction())
2775 return RValue::getIgnored();
2776 if (
const auto *XRayAttr = CurFuncDecl->getAttr<XRayInstrumentAttr>()) {
2777 if (XRayAttr->neverXRayInstrument())
2778 return RValue::getIgnored();
2780 Function *F = CGM.getIntrinsic(Intrinsic::xray_customevent);
2781 auto FTy = F->getFunctionType();
2782 auto Arg0 = E->getArg(0);
2783 auto Arg0Val = EmitScalarExpr(Arg0);
2784 auto Arg0Ty = Arg0->getType();
2785 auto PTy0 = FTy->getParamType(0);
2786 if (PTy0 != Arg0Val->getType()) {
2787 if (Arg0Ty->isArrayType())
2788 Arg0Val = EmitArrayToPointerDecay(Arg0).getPointer();
2790 Arg0Val =
Builder.CreatePointerCast(Arg0Val, PTy0);
2792 auto Arg1 = EmitScalarExpr(E->getArg(1));
2793 auto PTy1 = FTy->getParamType(1);
2794 if (PTy1 != Arg1->getType())
2795 Arg1 =
Builder.CreateTruncOrBitCast(Arg1, PTy1);
2796 return RValue::get(
Builder.CreateCall(F, {Arg0Val, Arg1}));
2799 case Builtin::BI__builtin_ms_va_start:
2800 case Builtin::BI__builtin_ms_va_end:
2802 EmitVAStartEnd(EmitMSVAListRef(E->getArg(0)).getPointer(),
2803 BuiltinID == Builtin::BI__builtin_ms_va_start));
2805 case Builtin::BI__builtin_ms_va_copy: {
2812 Address DestAddr = EmitMSVAListRef(E->getArg(0));
2813 Address SrcAddr = EmitMSVAListRef(E->getArg(1));
2815 llvm::Type *BPP = Int8PtrPtrTy;
2823 return RValue::get(
Builder.CreateStore(ArgPtr, DestAddr));
2830 if (getContext().BuiltinInfo.isLibFunction(BuiltinID))
2832 CGM.getBuiltinLibFunction(FD, BuiltinID));
2836 if (getContext().BuiltinInfo.isPredefinedLibFunction(BuiltinID))
2838 cast<llvm::Constant>(EmitScalarExpr(E->getCallee())));
2845 checkTargetFeatures(E, FD);
2848 const char *Name = getContext().BuiltinInfo.getName(BuiltinID);
2851 llvm::Triple::getArchTypePrefix(getTarget().getTriple().getArch());
2852 if (!Prefix.empty()) {
2853 IntrinsicID = Intrinsic::getIntrinsicForGCCBuiltin(Prefix.data(),
Name);
2857 if (IntrinsicID == Intrinsic::not_intrinsic)
2858 IntrinsicID = Intrinsic::getIntrinsicForMSBuiltin(Prefix.data(),
Name);
2861 if (IntrinsicID != Intrinsic::not_intrinsic) {
2866 unsigned ICEArguments = 0;
2868 getContext().GetBuiltinType(BuiltinID, Error, &ICEArguments);
2869 assert(Error == ASTContext::GE_None &&
"Should not codegen an error");
2871 Function *F = CGM.getIntrinsic(IntrinsicID);
2872 llvm::FunctionType *FTy = F->getFunctionType();
2874 for (
unsigned i = 0, e = E->getNumArgs(); i != e; ++i) {
2877 if ((ICEArguments & (1 << i)) == 0) {
2878 ArgValue = EmitScalarExpr(E->getArg(i));
2884 assert(IsConst &&
"Constant arg isn't actually constant?");
2886 ArgValue = llvm::ConstantInt::get(getLLVMContext(), Result);
2891 llvm::Type *PTy = FTy->getParamType(i);
2892 if (PTy != ArgValue->getType()) {
2893 assert(PTy->canLosslesslyBitCastTo(FTy->getParamType(i)) &&
2894 "Must be able to losslessly bit cast to param");
2895 ArgValue =
Builder.CreateBitCast(ArgValue, PTy);
2898 Args.push_back(ArgValue);
2904 llvm::Type *RetTy = VoidTy;
2906 RetTy = ConvertType(BuiltinRetType);
2908 if (RetTy != V->getType()) {
2909 assert(V->getType()->canLosslesslyBitCastTo(RetTy) &&
2910 "Must be able to losslessly bit cast result type");
2911 V =
Builder.CreateBitCast(V, RetTy);
2914 return RValue::get(V);
2918 if (
Value *V = EmitTargetBuiltinExpr(BuiltinID, E))
2919 return RValue::get(V);
2921 ErrorUnsupported(E,
"builtin function");
2924 return GetUndefRValue(E->
getType());
2928 unsigned BuiltinID,
const CallExpr *E,
2929 llvm::Triple::ArchType Arch) {
2931 case llvm::Triple::arm:
2932 case llvm::Triple::armeb:
2933 case llvm::Triple::thumb:
2934 case llvm::Triple::thumbeb:
2936 case llvm::Triple::aarch64:
2937 case llvm::Triple::aarch64_be:
2939 case llvm::Triple::x86:
2940 case llvm::Triple::x86_64:
2942 case llvm::Triple::ppc:
2943 case llvm::Triple::ppc64:
2944 case llvm::Triple::ppc64le:
2946 case llvm::Triple::r600:
2947 case llvm::Triple::amdgcn:
2949 case llvm::Triple::systemz:
2951 case llvm::Triple::nvptx:
2952 case llvm::Triple::nvptx64:
2954 case llvm::Triple::wasm32:
2955 case llvm::Triple::wasm64:
2962 Value *CodeGenFunction::EmitTargetBuiltinExpr(
unsigned BuiltinID,
2965 assert(
getContext().getAuxTargetInfo() &&
"Missing aux target info");
2978 int IsQuad = TypeFlags.
isQuad();
2982 return llvm::VectorType::get(CGF->
Int8Ty, V1Ty ? 1 : (8 << IsQuad));
2986 return llvm::VectorType::get(CGF->
Int16Ty, V1Ty ? 1 : (4 << IsQuad));
2988 return llvm::VectorType::get(CGF->
Int32Ty, V1Ty ? 1 : (2 << IsQuad));
2991 return llvm::VectorType::get(CGF->
Int64Ty, V1Ty ? 1 : (1 << IsQuad));
2996 return llvm::VectorType::get(CGF->
Int8Ty, 16);
2998 return llvm::VectorType::get(CGF->
FloatTy, V1Ty ? 1 : (2 << IsQuad));
3000 return llvm::VectorType::get(CGF->
DoubleTy, V1Ty ? 1 : (1 << IsQuad));
3002 llvm_unreachable(
"Unknown vector element type!");
3007 int IsQuad = IntTypeFlags.
isQuad();
3010 return llvm::VectorType::get(CGF->
FloatTy, (2 << IsQuad));
3012 return llvm::VectorType::get(CGF->
DoubleTy, (1 << IsQuad));
3014 llvm_unreachable(
"Type can't be converted to floating-point!");
3019 unsigned nElts = V->getType()->getVectorNumElements();
3020 Value* SV = llvm::ConstantVector::getSplat(nElts, C);
3021 return Builder.CreateShuffleVector(V, V, SV,
"lane");
3026 unsigned shift,
bool rightshift) {
3028 for (Function::const_arg_iterator ai = F->arg_begin(), ae = F->arg_end();
3029 ai != ae; ++ai, ++j)
3030 if (shift > 0 && shift == j)
3035 return Builder.CreateCall(F, Ops, name);
3040 int SV = cast<ConstantInt>(V)->getSExtValue();
3041 return ConstantInt::get(Ty, neg ? -SV : SV);
3048 llvm::VectorType *VTy = cast<llvm::VectorType>(Ty);
3050 int ShiftAmt = cast<ConstantInt>(
Shift)->getSExtValue();
3051 int EltSize = VTy->getScalarSizeInBits();
3057 if (ShiftAmt == EltSize) {
3060 return llvm::ConstantAggregateZero::get(VTy);
3065 Shift = ConstantInt::get(VTy->getElementType(), ShiftAmt);
3071 return Builder.CreateLShr(Vec, Shift, name);
3073 return Builder.CreateAShr(Vec, Shift, name);
3099 struct NeonIntrinsicInfo {
3100 const char *NameHint;
3102 unsigned LLVMIntrinsic;
3103 unsigned AltLLVMIntrinsic;
3104 unsigned TypeModifier;
3106 bool operator<(
unsigned RHSBuiltinID)
const {
3107 return BuiltinID < RHSBuiltinID;
3109 bool operator<(
const NeonIntrinsicInfo &TE)
const {
3110 return BuiltinID < TE.BuiltinID;
3115 #define NEONMAP0(NameBase) \
3116 { #NameBase, NEON::BI__builtin_neon_ ## NameBase, 0, 0, 0 }
3118 #define NEONMAP1(NameBase, LLVMIntrinsic, TypeModifier) \
3119 { #NameBase, NEON:: BI__builtin_neon_ ## NameBase, \
3120 Intrinsic::LLVMIntrinsic, 0, TypeModifier }
3122 #define NEONMAP2(NameBase, LLVMIntrinsic, AltLLVMIntrinsic, TypeModifier) \
3123 { #NameBase, NEON:: BI__builtin_neon_ ## NameBase, \
3124 Intrinsic::LLVMIntrinsic, Intrinsic::AltLLVMIntrinsic, \
3130 NEONMAP1(vabs_v, arm_neon_vabs, 0),
3131 NEONMAP1(vabsq_v, arm_neon_vabs, 0),
3133 NEONMAP1(vaesdq_v, arm_neon_aesd, 0),
3134 NEONMAP1(vaeseq_v, arm_neon_aese, 0),
3135 NEONMAP1(vaesimcq_v, arm_neon_aesimc, 0),
3136 NEONMAP1(vaesmcq_v, arm_neon_aesmc, 0),
3139 NEONMAP1(vcage_v, arm_neon_vacge, 0),
3140 NEONMAP1(vcageq_v, arm_neon_vacge, 0),
3141 NEONMAP1(vcagt_v, arm_neon_vacgt, 0),
3142 NEONMAP1(vcagtq_v, arm_neon_vacgt, 0),
3143 NEONMAP1(vcale_v, arm_neon_vacge, 0),
3144 NEONMAP1(vcaleq_v, arm_neon_vacge, 0),
3145 NEONMAP1(vcalt_v, arm_neon_vacgt, 0),
3146 NEONMAP1(vcaltq_v, arm_neon_vacgt, 0),
3153 NEONMAP1(vcvt_f16_f32, arm_neon_vcvtfp2hf, 0),
3154 NEONMAP1(vcvt_f32_f16, arm_neon_vcvthf2fp, 0),
3156 NEONMAP2(vcvt_n_f32_v, arm_neon_vcvtfxu2fp, arm_neon_vcvtfxs2fp, 0),
3157 NEONMAP1(vcvt_n_s32_v, arm_neon_vcvtfp2fxs, 0),
3158 NEONMAP1(vcvt_n_s64_v, arm_neon_vcvtfp2fxs, 0),
3159 NEONMAP1(vcvt_n_u32_v, arm_neon_vcvtfp2fxu, 0),
3160 NEONMAP1(vcvt_n_u64_v, arm_neon_vcvtfp2fxu, 0),
3165 NEONMAP1(vcvta_s32_v, arm_neon_vcvtas, 0),
3166 NEONMAP1(vcvta_s64_v, arm_neon_vcvtas, 0),
3167 NEONMAP1(vcvta_u32_v, arm_neon_vcvtau, 0),
3168 NEONMAP1(vcvta_u64_v, arm_neon_vcvtau, 0),
3169 NEONMAP1(vcvtaq_s32_v, arm_neon_vcvtas, 0),
3170 NEONMAP1(vcvtaq_s64_v, arm_neon_vcvtas, 0),
3171 NEONMAP1(vcvtaq_u32_v, arm_neon_vcvtau, 0),
3172 NEONMAP1(vcvtaq_u64_v, arm_neon_vcvtau, 0),
3173 NEONMAP1(vcvtm_s32_v, arm_neon_vcvtms, 0),
3174 NEONMAP1(vcvtm_s64_v, arm_neon_vcvtms, 0),
3175 NEONMAP1(vcvtm_u32_v, arm_neon_vcvtmu, 0),
3176 NEONMAP1(vcvtm_u64_v, arm_neon_vcvtmu, 0),
3177 NEONMAP1(vcvtmq_s32_v, arm_neon_vcvtms, 0),
3178 NEONMAP1(vcvtmq_s64_v, arm_neon_vcvtms, 0),
3179 NEONMAP1(vcvtmq_u32_v, arm_neon_vcvtmu, 0),
3180 NEONMAP1(vcvtmq_u64_v, arm_neon_vcvtmu, 0),
3181 NEONMAP1(vcvtn_s32_v, arm_neon_vcvtns, 0),
3182 NEONMAP1(vcvtn_s64_v, arm_neon_vcvtns, 0),
3183 NEONMAP1(vcvtn_u32_v, arm_neon_vcvtnu, 0),
3184 NEONMAP1(vcvtn_u64_v, arm_neon_vcvtnu, 0),
3185 NEONMAP1(vcvtnq_s32_v, arm_neon_vcvtns, 0),
3186 NEONMAP1(vcvtnq_s64_v, arm_neon_vcvtns, 0),
3187 NEONMAP1(vcvtnq_u32_v, arm_neon_vcvtnu, 0),
3188 NEONMAP1(vcvtnq_u64_v, arm_neon_vcvtnu, 0),
3189 NEONMAP1(vcvtp_s32_v, arm_neon_vcvtps, 0),
3190 NEONMAP1(vcvtp_s64_v, arm_neon_vcvtps, 0),
3191 NEONMAP1(vcvtp_u32_v, arm_neon_vcvtpu, 0),
3192 NEONMAP1(vcvtp_u64_v, arm_neon_vcvtpu, 0),
3193 NEONMAP1(vcvtpq_s32_v, arm_neon_vcvtps, 0),
3194 NEONMAP1(vcvtpq_s64_v, arm_neon_vcvtps, 0),
3195 NEONMAP1(vcvtpq_u32_v, arm_neon_vcvtpu, 0),
3196 NEONMAP1(vcvtpq_u64_v, arm_neon_vcvtpu, 0),
3198 NEONMAP2(vcvtq_n_f32_v, arm_neon_vcvtfxu2fp, arm_neon_vcvtfxs2fp, 0),
3199 NEONMAP1(vcvtq_n_s32_v, arm_neon_vcvtfp2fxs, 0),
3200 NEONMAP1(vcvtq_n_s64_v, arm_neon_vcvtfp2fxs, 0),
3201 NEONMAP1(vcvtq_n_u32_v, arm_neon_vcvtfp2fxu, 0),
3202 NEONMAP1(vcvtq_n_u64_v, arm_neon_vcvtfp2fxu, 0),
3216 NEONMAP1(vld1_v, arm_neon_vld1, 0),
3218 NEONMAP1(vld1q_v, arm_neon_vld1, 0),
3219 NEONMAP1(vld2_lane_v, arm_neon_vld2lane, 0),
3220 NEONMAP1(vld2_v, arm_neon_vld2, 0),
3221 NEONMAP1(vld2q_lane_v, arm_neon_vld2lane, 0),
3222 NEONMAP1(vld2q_v, arm_neon_vld2, 0),
3223 NEONMAP1(vld3_lane_v, arm_neon_vld3lane, 0),
3224 NEONMAP1(vld3_v, arm_neon_vld3, 0),
3225 NEONMAP1(vld3q_lane_v, arm_neon_vld3lane, 0),
3226 NEONMAP1(vld3q_v, arm_neon_vld3, 0),
3227 NEONMAP1(vld4_lane_v, arm_neon_vld4lane, 0),
3228 NEONMAP1(vld4_v, arm_neon_vld4, 0),
3229 NEONMAP1(vld4q_lane_v, arm_neon_vld4lane, 0),
3230 NEONMAP1(vld4q_v, arm_neon_vld4, 0),
3256 NEONMAP2(vqdmlal_v, arm_neon_vqdmull, arm_neon_vqadds, 0),
3257 NEONMAP2(vqdmlsl_v, arm_neon_vqdmull, arm_neon_vqsubs, 0),
3273 NEONMAP1(vqshlu_n_v, arm_neon_vqshiftsu, 0),
3274 NEONMAP1(vqshluq_n_v, arm_neon_vqshiftsu, 0),
3278 NEONMAP2(vrecpe_v, arm_neon_vrecpe, arm_neon_vrecpe, 0),
3279 NEONMAP2(vrecpeq_v, arm_neon_vrecpe, arm_neon_vrecpe, 0),
3300 NEONMAP2(vrsqrte_v, arm_neon_vrsqrte, arm_neon_vrsqrte, 0),
3301 NEONMAP2(vrsqrteq_v, arm_neon_vrsqrte, arm_neon_vrsqrte, 0),
3305 NEONMAP1(vsha1su0q_v, arm_neon_sha1su0, 0),
3306 NEONMAP1(vsha1su1q_v, arm_neon_sha1su1, 0),
3307 NEONMAP1(vsha256h2q_v, arm_neon_sha256h2, 0),
3308 NEONMAP1(vsha256hq_v, arm_neon_sha256h, 0),
3309 NEONMAP1(vsha256su0q_v, arm_neon_sha256su0, 0),
3310 NEONMAP1(vsha256su1q_v, arm_neon_sha256su1, 0),
3319 NEONMAP1(vst1_v, arm_neon_vst1, 0),
3320 NEONMAP1(vst1q_v, arm_neon_vst1, 0),
3321 NEONMAP1(vst2_lane_v, arm_neon_vst2lane, 0),
3322 NEONMAP1(vst2_v, arm_neon_vst2, 0),
3323 NEONMAP1(vst2q_lane_v, arm_neon_vst2lane, 0),
3324 NEONMAP1(vst2q_v, arm_neon_vst2, 0),
3325 NEONMAP1(vst3_lane_v, arm_neon_vst3lane, 0),
3326 NEONMAP1(vst3_v, arm_neon_vst3, 0),
3327 NEONMAP1(vst3q_lane_v, arm_neon_vst3lane, 0),
3328 NEONMAP1(vst3q_v, arm_neon_vst3, 0),
3329 NEONMAP1(vst4_lane_v, arm_neon_vst4lane, 0),
3330 NEONMAP1(vst4_v, arm_neon_vst4, 0),
3331 NEONMAP1(vst4q_lane_v, arm_neon_vst4lane, 0),
3332 NEONMAP1(vst4q_v, arm_neon_vst4, 0),
3345 NEONMAP1(vabs_v, aarch64_neon_abs, 0),
3346 NEONMAP1(vabsq_v, aarch64_neon_abs, 0),
3348 NEONMAP1(vaesdq_v, aarch64_crypto_aesd, 0),
3349 NEONMAP1(vaeseq_v, aarch64_crypto_aese, 0),
3350 NEONMAP1(vaesimcq_v, aarch64_crypto_aesimc, 0),
3351 NEONMAP1(vaesmcq_v, aarch64_crypto_aesmc, 0),
3352 NEONMAP1(vcage_v, aarch64_neon_facge, 0),
3353 NEONMAP1(vcageq_v, aarch64_neon_facge, 0),
3354 NEONMAP1(vcagt_v, aarch64_neon_facgt, 0),
3355 NEONMAP1(vcagtq_v, aarch64_neon_facgt, 0),
3356 NEONMAP1(vcale_v, aarch64_neon_facge, 0),
3357 NEONMAP1(vcaleq_v, aarch64_neon_facge, 0),
3358 NEONMAP1(vcalt_v, aarch64_neon_facgt, 0),
3359 NEONMAP1(vcaltq_v, aarch64_neon_facgt, 0),
3366 NEONMAP1(vcvt_f16_f32, aarch64_neon_vcvtfp2hf, 0),
3367 NEONMAP1(vcvt_f32_f16, aarch64_neon_vcvthf2fp, 0),
3369 NEONMAP2(vcvt_n_f32_v, aarch64_neon_vcvtfxu2fp, aarch64_neon_vcvtfxs2fp, 0),
3370 NEONMAP2(vcvt_n_f64_v, aarch64_neon_vcvtfxu2fp, aarch64_neon_vcvtfxs2fp, 0),
3371 NEONMAP1(vcvt_n_s32_v, aarch64_neon_vcvtfp2fxs, 0),
3372 NEONMAP1(vcvt_n_s64_v, aarch64_neon_vcvtfp2fxs, 0),
3373 NEONMAP1(vcvt_n_u32_v, aarch64_neon_vcvtfp2fxu, 0),
3374 NEONMAP1(vcvt_n_u64_v, aarch64_neon_vcvtfp2fxu, 0),
3376 NEONMAP2(vcvtq_n_f32_v, aarch64_neon_vcvtfxu2fp, aarch64_neon_vcvtfxs2fp, 0),
3377 NEONMAP2(vcvtq_n_f64_v, aarch64_neon_vcvtfxu2fp, aarch64_neon_vcvtfxs2fp, 0),
3378 NEONMAP1(vcvtq_n_s32_v, aarch64_neon_vcvtfp2fxs, 0),
3379 NEONMAP1(vcvtq_n_s64_v, aarch64_neon_vcvtfp2fxs, 0),
3380 NEONMAP1(vcvtq_n_u32_v, aarch64_neon_vcvtfp2fxu, 0),
3381 NEONMAP1(vcvtq_n_u64_v, aarch64_neon_vcvtfp2fxu, 0),
3403 NEONMAP2(vqdmlal_v, aarch64_neon_sqdmull, aarch64_neon_sqadd, 0),
3404 NEONMAP2(vqdmlsl_v, aarch64_neon_sqdmull, aarch64_neon_sqsub, 0),
3420 NEONMAP1(vqshlu_n_v, aarch64_neon_sqshlu, 0),
3421 NEONMAP1(vqshluq_n_v, aarch64_neon_sqshlu, 0),
3425 NEONMAP2(vrecpe_v, aarch64_neon_frecpe, aarch64_neon_urecpe, 0),
3426 NEONMAP2(vrecpeq_v, aarch64_neon_frecpe, aarch64_neon_urecpe, 0),
3435 NEONMAP2(vrsqrte_v, aarch64_neon_frsqrte, aarch64_neon_ursqrte, 0),
3436 NEONMAP2(vrsqrteq_v, aarch64_neon_frsqrte, aarch64_neon_ursqrte, 0),
3440 NEONMAP1(vsha1su0q_v, aarch64_crypto_sha1su0, 0),
3441 NEONMAP1(vsha1su1q_v, aarch64_crypto_sha1su1, 0),
3442 NEONMAP1(vsha256h2q_v, aarch64_crypto_sha256h2, 0),
3443 NEONMAP1(vsha256hq_v, aarch64_crypto_sha256h, 0),
3444 NEONMAP1(vsha256su0q_v, aarch64_crypto_sha256su0, 0),
3445 NEONMAP1(vsha256su1q_v, aarch64_crypto_sha256su1, 0),
3508 NEONMAP1(vcvtxd_f32_f64, aarch64_sisd_fcvtxn, 0),
3529 NEONMAP1(vmull_p64, aarch64_neon_pmull64, 0),
3557 NEONMAP1(vqdmulls_s32, aarch64_neon_sqdmulls_scalar, 0),
3634 NEONMAP1(vsha1cq_u32, aarch64_crypto_sha1c, 0),
3635 NEONMAP1(vsha1h_u32, aarch64_crypto_sha1h, 0),
3636 NEONMAP1(vsha1mq_u32, aarch64_crypto_sha1m, 0),
3637 NEONMAP1(vsha1pq_u32, aarch64_crypto_sha1p, 0),
3664 static const NeonIntrinsicInfo *
3666 unsigned BuiltinID,
bool &MapProvenSorted) {
3669 if (!MapProvenSorted) {
3670 assert(std::is_sorted(std::begin(IntrinsicMap), std::end(IntrinsicMap)));
3671 MapProvenSorted =
true;
3675 const NeonIntrinsicInfo *Builtin =
3676 std::lower_bound(IntrinsicMap.begin(), IntrinsicMap.end(), BuiltinID);
3678 if (Builtin != IntrinsicMap.end() && Builtin->BuiltinID == BuiltinID)
3699 Ty = llvm::VectorType::get(
3700 Ty, VectorSize ? VectorSize / Ty->getPrimitiveSizeInBits() : 1);
3707 int Elts = VectorSize ? VectorSize / ArgType->getPrimitiveSizeInBits() : 1;
3708 ArgType = llvm::VectorType::get(ArgType, Elts);
3712 Tys.push_back(ArgType);
3715 Tys.push_back(ArgType);
3724 const NeonIntrinsicInfo &SISDInfo,
3727 unsigned BuiltinID = SISDInfo.BuiltinID;
3728 unsigned int Int = SISDInfo.LLVMIntrinsic;
3729 unsigned Modifier = SISDInfo.TypeModifier;
3730 const char *s = SISDInfo.NameHint;
3732 switch (BuiltinID) {
3733 case NEON::BI__builtin_neon_vcled_s64:
3734 case NEON::BI__builtin_neon_vcled_u64:
3735 case NEON::BI__builtin_neon_vcles_f32:
3736 case NEON::BI__builtin_neon_vcled_f64:
3737 case NEON::BI__builtin_neon_vcltd_s64:
3738 case NEON::BI__builtin_neon_vcltd_u64:
3739 case NEON::BI__builtin_neon_vclts_f32:
3740 case NEON::BI__builtin_neon_vcltd_f64:
3741 case NEON::BI__builtin_neon_vcales_f32:
3742 case NEON::BI__builtin_neon_vcaled_f64:
3743 case NEON::BI__builtin_neon_vcalts_f32:
3744 case NEON::BI__builtin_neon_vcaltd_f64:
3748 std::swap(Ops[0], Ops[1]);
3752 assert(Int &&
"Generic code assumes a valid intrinsic");
3760 ConstantInt *C0 = ConstantInt::get(CGF.
SizeTy, 0);
3761 for (Function::const_arg_iterator ai = F->arg_begin(), ae = F->arg_end();
3762 ai != ae; ++ai, ++j) {
3764 if (Ops[j]->getType()->getPrimitiveSizeInBits() ==
3765 ArgTy->getPrimitiveSizeInBits())
3768 assert(ArgTy->isVectorTy() && !Ops[j]->getType()->isVectorTy());
3772 CGF.
Builder.CreateTruncOrBitCast(Ops[j], ArgTy->getVectorElementType());
3774 CGF.
Builder.CreateInsertElement(UndefValue::get(ArgTy), Ops[j], C0);
3779 if (ResultType->getPrimitiveSizeInBits() <
3780 Result->getType()->getPrimitiveSizeInBits())
3781 return CGF.
Builder.CreateExtractElement(Result, C0);
3787 unsigned BuiltinID,
unsigned LLVMIntrinsic,
unsigned AltLLVMIntrinsic,
3791 llvm::APSInt NeonTypeConst;
3798 bool Usgn = Type.isUnsigned();
3799 bool Quad = Type.isQuad();
3806 auto getAlignmentValue32 = [&](Address addr) ->
Value* {
3807 return Builder.getInt32(addr.getAlignment().getQuantity());
3810 unsigned Int = LLVMIntrinsic;
3812 Int = AltLLVMIntrinsic;
3814 switch (BuiltinID) {
3816 case NEON::BI__builtin_neon_vabs_v:
3817 case NEON::BI__builtin_neon_vabsq_v:
3818 if (VTy->getElementType()->isFloatingPointTy())
3821 case NEON::BI__builtin_neon_vaddhn_v: {
3822 llvm::VectorType *SrcTy =
3823 llvm::VectorType::getExtendedElementVectorType(VTy);
3828 Ops[0] =
Builder.CreateAdd(Ops[0], Ops[1],
"vaddhn");
3831 Constant *ShiftAmt =
3832 ConstantInt::get(SrcTy, SrcTy->getScalarSizeInBits() / 2);
3833 Ops[0] =
Builder.CreateLShr(Ops[0], ShiftAmt,
"vaddhn");
3836 return Builder.CreateTrunc(Ops[0], VTy,
"vaddhn");
3838 case NEON::BI__builtin_neon_vcale_v:
3839 case NEON::BI__builtin_neon_vcaleq_v:
3840 case NEON::BI__builtin_neon_vcalt_v:
3841 case NEON::BI__builtin_neon_vcaltq_v:
3842 std::swap(Ops[0], Ops[1]);
3844 case NEON::BI__builtin_neon_vcage_v:
3845 case NEON::BI__builtin_neon_vcageq_v:
3846 case NEON::BI__builtin_neon_vcagt_v:
3847 case NEON::BI__builtin_neon_vcagtq_v: {
3850 VTy->getNumElements());
3855 case NEON::BI__builtin_neon_vclz_v:
3856 case NEON::BI__builtin_neon_vclzq_v:
3861 case NEON::BI__builtin_neon_vcvt_f32_v:
3862 case NEON::BI__builtin_neon_vcvtq_f32_v:
3865 return Usgn ?
Builder.CreateUIToFP(Ops[0], Ty,
"vcvt")
3866 :
Builder.CreateSIToFP(Ops[0], Ty,
"vcvt");
3867 case NEON::BI__builtin_neon_vcvt_n_f32_v:
3868 case NEON::BI__builtin_neon_vcvt_n_f64_v:
3869 case NEON::BI__builtin_neon_vcvtq_n_f32_v:
3870 case NEON::BI__builtin_neon_vcvtq_n_f64_v: {
3872 Int = Usgn ? LLVMIntrinsic : AltLLVMIntrinsic;
3876 case NEON::BI__builtin_neon_vcvt_n_s32_v:
3877 case NEON::BI__builtin_neon_vcvt_n_u32_v:
3878 case NEON::BI__builtin_neon_vcvt_n_s64_v:
3879 case NEON::BI__builtin_neon_vcvt_n_u64_v:
3880 case NEON::BI__builtin_neon_vcvtq_n_s32_v:
3881 case NEON::BI__builtin_neon_vcvtq_n_u32_v:
3882 case NEON::BI__builtin_neon_vcvtq_n_s64_v:
3883 case NEON::BI__builtin_neon_vcvtq_n_u64_v: {
3888 case NEON::BI__builtin_neon_vcvt_s32_v:
3889 case NEON::BI__builtin_neon_vcvt_u32_v:
3890 case NEON::BI__builtin_neon_vcvt_s64_v:
3891 case NEON::BI__builtin_neon_vcvt_u64_v:
3892 case NEON::BI__builtin_neon_vcvtq_s32_v:
3893 case NEON::BI__builtin_neon_vcvtq_u32_v:
3894 case NEON::BI__builtin_neon_vcvtq_s64_v:
3895 case NEON::BI__builtin_neon_vcvtq_u64_v: {
3897 return Usgn ?
Builder.CreateFPToUI(Ops[0], Ty,
"vcvt")
3898 :
Builder.CreateFPToSI(Ops[0], Ty,
"vcvt");
3900 case NEON::BI__builtin_neon_vcvta_s32_v:
3901 case NEON::BI__builtin_neon_vcvta_s64_v:
3902 case NEON::BI__builtin_neon_vcvta_u32_v:
3903 case NEON::BI__builtin_neon_vcvta_u64_v:
3904 case NEON::BI__builtin_neon_vcvtaq_s32_v:
3905 case NEON::BI__builtin_neon_vcvtaq_s64_v:
3906 case NEON::BI__builtin_neon_vcvtaq_u32_v:
3907 case NEON::BI__builtin_neon_vcvtaq_u64_v:
3908 case NEON::BI__builtin_neon_vcvtn_s32_v:
3909 case NEON::BI__builtin_neon_vcvtn_s64_v:
3910 case NEON::BI__builtin_neon_vcvtn_u32_v:
3911 case NEON::BI__builtin_neon_vcvtn_u64_v:
3912 case NEON::BI__builtin_neon_vcvtnq_s32_v:
3913 case NEON::BI__builtin_neon_vcvtnq_s64_v:
3914 case NEON::BI__builtin_neon_vcvtnq_u32_v:
3915 case NEON::BI__builtin_neon_vcvtnq_u64_v:
3916 case NEON::BI__builtin_neon_vcvtp_s32_v:
3917 case NEON::BI__builtin_neon_vcvtp_s64_v:
3918 case NEON::BI__builtin_neon_vcvtp_u32_v:
3919 case NEON::BI__builtin_neon_vcvtp_u64_v:
3920 case NEON::BI__builtin_neon_vcvtpq_s32_v:
3921 case NEON::BI__builtin_neon_vcvtpq_s64_v:
3922 case NEON::BI__builtin_neon_vcvtpq_u32_v:
3923 case NEON::BI__builtin_neon_vcvtpq_u64_v:
3924 case NEON::BI__builtin_neon_vcvtm_s32_v:
3925 case NEON::BI__builtin_neon_vcvtm_s64_v:
3926 case NEON::BI__builtin_neon_vcvtm_u32_v:
3927 case NEON::BI__builtin_neon_vcvtm_u64_v:
3928 case NEON::BI__builtin_neon_vcvtmq_s32_v:
3929 case NEON::BI__builtin_neon_vcvtmq_s64_v:
3930 case NEON::BI__builtin_neon_vcvtmq_u32_v:
3931 case NEON::BI__builtin_neon_vcvtmq_u64_v: {
3935 case NEON::BI__builtin_neon_vext_v:
3936 case NEON::BI__builtin_neon_vextq_v: {
3937 int CV = cast<ConstantInt>(Ops[2])->getSExtValue();
3939 for (
unsigned i = 0, e = VTy->getNumElements(); i != e; ++i)
3940 Indices.push_back(i+CV);
3944 return Builder.CreateShuffleVector(Ops[0], Ops[1], Indices,
"vext");
3946 case NEON::BI__builtin_neon_vfma_v:
3947 case NEON::BI__builtin_neon_vfmaq_v: {
3954 return Builder.CreateCall(F, {Ops[1], Ops[2], Ops[0]});
3956 case NEON::BI__builtin_neon_vld1_v:
3957 case NEON::BI__builtin_neon_vld1q_v: {
3959 Ops.push_back(getAlignmentValue32(PtrOp0));
3962 case NEON::BI__builtin_neon_vld2_v:
3963 case NEON::BI__builtin_neon_vld2q_v:
3964 case NEON::BI__builtin_neon_vld3_v:
3965 case NEON::BI__builtin_neon_vld3q_v:
3966 case NEON::BI__builtin_neon_vld4_v:
3967 case NEON::BI__builtin_neon_vld4q_v: {
3970 Value *Align = getAlignmentValue32(PtrOp1);
3971 Ops[1] =
Builder.CreateCall(F, {Ops[1], Align}, NameHint);
3972 Ty = llvm::PointerType::getUnqual(Ops[1]->getType());
3976 case NEON::BI__builtin_neon_vld1_dup_v:
3977 case NEON::BI__builtin_neon_vld1q_dup_v: {
3978 Value *V = UndefValue::get(Ty);
3979 Ty = llvm::PointerType::getUnqual(VTy->getElementType());
3982 llvm::Constant *CI = ConstantInt::get(
SizeTy, 0);
3983 Ops[0] =
Builder.CreateInsertElement(V, Ld, CI);
3986 case NEON::BI__builtin_neon_vld2_lane_v:
3987 case NEON::BI__builtin_neon_vld2q_lane_v:
3988 case NEON::BI__builtin_neon_vld3_lane_v:
3989 case NEON::BI__builtin_neon_vld3q_lane_v:
3990 case NEON::BI__builtin_neon_vld4_lane_v:
3991 case NEON::BI__builtin_neon_vld4q_lane_v: {
3994 for (
unsigned I = 2;
I < Ops.size() - 1; ++
I)
3996 Ops.push_back(getAlignmentValue32(PtrOp1));
3997 Ops[1] =
Builder.CreateCall(F, makeArrayRef(Ops).slice(1), NameHint);
3998 Ty = llvm::PointerType::getUnqual(Ops[1]->getType());
4002 case NEON::BI__builtin_neon_vmovl_v: {
4003 llvm::Type *DTy =llvm::VectorType::getTruncatedElementVectorType(VTy);
4006 return Builder.CreateZExt(Ops[0], Ty,
"vmovl");
4007 return Builder.CreateSExt(Ops[0], Ty,
"vmovl");
4009 case NEON::BI__builtin_neon_vmovn_v: {
4010 llvm::Type *QTy = llvm::VectorType::getExtendedElementVectorType(VTy);
4012 return Builder.CreateTrunc(Ops[0], Ty,
"vmovn");
4014 case NEON::BI__builtin_neon_vmull_v:
4020 Int = Usgn ? Intrinsic::arm_neon_vmullu : Intrinsic::arm_neon_vmulls;
4021 Int = Type.isPoly() ? (
unsigned)Intrinsic::arm_neon_vmullp : Int;
4023 case NEON::BI__builtin_neon_vpadal_v:
4024 case NEON::BI__builtin_neon_vpadalq_v: {
4026 unsigned EltBits = VTy->getElementType()->getPrimitiveSizeInBits();
4030 llvm::VectorType::get(EltTy, VTy->getNumElements() * 2);
4034 case NEON::BI__builtin_neon_vpaddl_v:
4035 case NEON::BI__builtin_neon_vpaddlq_v: {
4037 unsigned EltBits = VTy->getElementType()->getPrimitiveSizeInBits();
4040 llvm::VectorType::get(EltTy, VTy->getNumElements() * 2);
4044 case NEON::BI__builtin_neon_vqdmlal_v:
4045 case NEON::BI__builtin_neon_vqdmlsl_v: {
4052 case NEON::BI__builtin_neon_vqshl_n_v:
4053 case NEON::BI__builtin_neon_vqshlq_n_v:
4056 case NEON::BI__builtin_neon_vqshlu_n_v:
4057 case NEON::BI__builtin_neon_vqshluq_n_v:
4060 case NEON::BI__builtin_neon_vrecpe_v:
4061 case NEON::BI__builtin_neon_vrecpeq_v:
4062 case NEON::BI__builtin_neon_vrsqrte_v:
4063 case NEON::BI__builtin_neon_vrsqrteq_v:
4064 Int = Ty->isFPOrFPVectorTy() ? LLVMIntrinsic : AltLLVMIntrinsic;
4067 case NEON::BI__builtin_neon_vrshr_n_v:
4068 case NEON::BI__builtin_neon_vrshrq_n_v:
4071 case NEON::BI__builtin_neon_vshl_n_v:
4072 case NEON::BI__builtin_neon_vshlq_n_v:
4076 case NEON::BI__builtin_neon_vshll_n_v: {
4077 llvm::Type *SrcTy = llvm::VectorType::getTruncatedElementVectorType(VTy);
4080 Ops[0] =
Builder.CreateZExt(Ops[0], VTy);
4082 Ops[0] =
Builder.CreateSExt(Ops[0], VTy);
4084 return Builder.CreateShl(Ops[0], Ops[1],
"vshll_n");
4086 case NEON::BI__builtin_neon_vshrn_n_v: {
4087 llvm::Type *SrcTy = llvm::VectorType::getExtendedElementVectorType(VTy);
4091 Ops[0] =
Builder.CreateLShr(Ops[0], Ops[1]);
4093 Ops[0] =
Builder.CreateAShr(Ops[0], Ops[1]);
4094 return Builder.CreateTrunc(Ops[0], Ty,
"vshrn_n");
4096 case NEON::BI__builtin_neon_vshr_n_v:
4097 case NEON::BI__builtin_neon_vshrq_n_v:
4099 case NEON::BI__builtin_neon_vst1_v:
4100 case NEON::BI__builtin_neon_vst1q_v:
4101 case NEON::BI__builtin_neon_vst2_v:
4102 case NEON::BI__builtin_neon_vst2q_v:
4103 case NEON::BI__builtin_neon_vst3_v:
4104 case NEON::BI__builtin_neon_vst3q_v:
4105 case NEON::BI__builtin_neon_vst4_v:
4106 case NEON::BI__builtin_neon_vst4q_v:
4107 case NEON::BI__builtin_neon_vst2_lane_v:
4108 case NEON::BI__builtin_neon_vst2q_lane_v:
4109 case NEON::BI__builtin_neon_vst3_lane_v:
4110 case NEON::BI__builtin_neon_vst3q_lane_v:
4111 case NEON::BI__builtin_neon_vst4_lane_v:
4112 case NEON::BI__builtin_neon_vst4q_lane_v: {
4114 Ops.push_back(getAlignmentValue32(PtrOp0));
4117 case NEON::BI__builtin_neon_vsubhn_v: {
4118 llvm::VectorType *SrcTy =
4119 llvm::VectorType::getExtendedElementVectorType(VTy);
4124 Ops[0] =
Builder.CreateSub(Ops[0], Ops[1],
"vsubhn");
4127 Constant *ShiftAmt =
4128 ConstantInt::get(SrcTy, SrcTy->getScalarSizeInBits() / 2);
4129 Ops[0] =
Builder.CreateLShr(Ops[0], ShiftAmt,
"vsubhn");
4132 return Builder.CreateTrunc(Ops[0], VTy,
"vsubhn");
4134 case NEON::BI__builtin_neon_vtrn_v:
4135 case NEON::BI__builtin_neon_vtrnq_v: {
4139 Value *SV =
nullptr;
4141 for (
unsigned vi = 0; vi != 2; ++vi) {
4143 for (
unsigned i = 0, e = VTy->getNumElements(); i != e; i += 2) {
4144 Indices.push_back(i+vi);
4145 Indices.push_back(i+e+vi);
4147 Value *Addr =
Builder.CreateConstInBoundsGEP1_32(Ty, Ops[0], vi);
4148 SV =
Builder.CreateShuffleVector(Ops[1], Ops[2], Indices,
"vtrn");
4153 case NEON::BI__builtin_neon_vtst_v:
4154 case NEON::BI__builtin_neon_vtstq_v: {
4157 Ops[0] =
Builder.CreateAnd(Ops[0], Ops[1]);
4158 Ops[0] =
Builder.CreateICmp(ICmpInst::ICMP_NE, Ops[0],
4159 ConstantAggregateZero::get(Ty));
4160 return Builder.CreateSExt(Ops[0], Ty,
"vtst");
4162 case NEON::BI__builtin_neon_vuzp_v:
4163 case NEON::BI__builtin_neon_vuzpq_v: {
4167 Value *SV =
nullptr;
4169 for (
unsigned vi = 0; vi != 2; ++vi) {
4171 for (
unsigned i = 0, e = VTy->getNumElements(); i != e; ++i)
4172 Indices.push_back(2*i+vi);
4174 Value *Addr =
Builder.CreateConstInBoundsGEP1_32(Ty, Ops[0], vi);
4175 SV =
Builder.CreateShuffleVector(Ops[1], Ops[2], Indices,
"vuzp");
4180 case NEON::BI__builtin_neon_vzip_v:
4181 case NEON::BI__builtin_neon_vzipq_v: {
4185 Value *SV =
nullptr;
4187 for (
unsigned vi = 0; vi != 2; ++vi) {
4189 for (
unsigned i = 0, e = VTy->getNumElements(); i != e; i += 2) {
4190 Indices.push_back((i + vi*e) >> 1);
4191 Indices.push_back(((i + vi*e) >> 1)+e);
4193 Value *Addr =
Builder.CreateConstInBoundsGEP1_32(Ty, Ops[0], vi);
4194 SV =
Builder.CreateShuffleVector(Ops[1], Ops[2], Indices,
"vzip");
4201 assert(Int &&
"Expected valid intrinsic number");
4215 const CmpInst::Predicate Ip,
const Twine &Name) {
4222 if (BitCastInst *BI = dyn_cast<BitCastInst>(Op))
4223 OTy = BI->getOperand(0)->getType();
4226 if (OTy->getScalarType()->isFloatingPointTy()) {
4227 Op =
Builder.CreateFCmp(Fp, Op, Constant::getNullValue(OTy));
4229 Op =
Builder.CreateICmp(Ip, Op, Constant::getNullValue(OTy));
4231 return Builder.CreateSExt(Op, Ty, Name);
4240 TblOps.push_back(ExtOp);
4244 llvm::VectorType *TblTy = cast<llvm::VectorType>(Ops[0]->getType());
4245 for (
unsigned i = 0, e = TblTy->getNumElements(); i != e; ++i) {
4246 Indices.push_back(2*i);
4247 Indices.push_back(2*i+1);
4250 int PairPos = 0, End = Ops.size() - 1;
4251 while (PairPos < End) {
4252 TblOps.push_back(CGF.
Builder.CreateShuffleVector(Ops[PairPos],
4253 Ops[PairPos+1], Indices,
4260 if (PairPos == End) {
4261 Value *ZeroTbl = ConstantAggregateZero::get(TblTy);
4262 TblOps.push_back(CGF.
Builder.CreateShuffleVector(Ops[PairPos],
4263 ZeroTbl, Indices, Name));
4267 TblOps.push_back(IndexOp);
4273 Value *CodeGenFunction::GetValueForARMHint(
unsigned BuiltinID) {
4275 switch (BuiltinID) {
4278 case ARM::BI__builtin_arm_nop:
4281 case ARM::BI__builtin_arm_yield:
4282 case ARM::BI__yield:
4285 case ARM::BI__builtin_arm_wfe:
4289 case ARM::BI__builtin_arm_wfi:
4293 case ARM::BI__builtin_arm_sev:
4297 case ARM::BI__builtin_arm_sevl:
4304 llvm::ConstantInt::get(
Int32Ty, Value));
4315 StringRef SysReg =
"") {
4317 assert((RegisterType->isIntegerTy(32) || RegisterType->isIntegerTy(64))
4318 &&
"Unsupported size for register.");
4324 if (SysReg.empty()) {
4326 SysReg = cast<clang::StringLiteral>(SysRegStrExpr)->getString();
4329 llvm::Metadata *Ops[] = { llvm::MDString::get(Context, SysReg) };
4330 llvm::MDNode *RegName = llvm::MDNode::get(Context, Ops);
4331 llvm::Value *Metadata = llvm::MetadataAsValue::get(Context, RegName);
4335 bool MixedTypes = RegisterType->isIntegerTy(64) && ValueType->isIntegerTy(32);
4336 assert(!(RegisterType->isIntegerTy(32) && ValueType->isIntegerTy(64))
4337 &&
"Can't fit 64-bit value in 32-bit register");
4341 llvm::Value *Call = Builder.CreateCall(F, Metadata);
4345 return Builder.CreateTrunc(Call, ValueType);
4347 if (ValueType->isPointerTy())
4349 return Builder.CreateIntToPtr(Call, ValueType);
4358 ArgValue = Builder.CreateZExt(ArgValue, RegisterType);
4359 return Builder.CreateCall(F, { Metadata, ArgValue });
4362 if (ValueType->isPointerTy()) {
4364 ArgValue = Builder.CreatePtrToInt(ArgValue, RegisterType);
4365 return Builder.CreateCall(F, { Metadata, ArgValue });
4368 return Builder.CreateCall(F, { Metadata, ArgValue });
4374 switch (BuiltinID) {
4376 case NEON::BI__builtin_neon_vget_lane_i8:
4377 case NEON::BI__builtin_neon_vget_lane_i16:
4378 case NEON::BI__builtin_neon_vget_lane_i32:
4379 case NEON::BI__builtin_neon_vget_lane_i64:
4380 case NEON::BI__builtin_neon_vget_lane_f32:
4381 case NEON::BI__builtin_neon_vgetq_lane_i8:
4382 case NEON::BI__builtin_neon_vgetq_lane_i16:
4383 case NEON::BI__builtin_neon_vgetq_lane_i32:
4384 case NEON::BI__builtin_neon_vgetq_lane_i64:
4385 case NEON::BI__builtin_neon_vgetq_lane_f32:
4386 case NEON::BI__builtin_neon_vset_lane_i8:
4387 case NEON::BI__builtin_neon_vset_lane_i16:
4388 case NEON::BI__builtin_neon_vset_lane_i32:
4389 case NEON::BI__builtin_neon_vset_lane_i64:
4390 case NEON::BI__builtin_neon_vset_lane_f32:
4391 case NEON::BI__builtin_neon_vsetq_lane_i8:
4392 case NEON::BI__builtin_neon_vsetq_lane_i16:
4393 case NEON::BI__builtin_neon_vsetq_lane_i32:
4394 case NEON::BI__builtin_neon_vsetq_lane_i64:
4395 case NEON::BI__builtin_neon_vsetq_lane_f32:
4396 case NEON::BI__builtin_neon_vsha1h_u32:
4397 case NEON::BI__builtin_neon_vsha1cq_u32:
4398 case NEON::BI__builtin_neon_vsha1pq_u32:
4399 case NEON::BI__builtin_neon_vsha1mq_u32:
4400 case ARM::BI_MoveToCoprocessor:
4401 case ARM::BI_MoveToCoprocessor2:
4409 if (
auto Hint = GetValueForARMHint(BuiltinID))
4412 if (BuiltinID == ARM::BI__emit) {
4414 llvm::FunctionType *FTy =
4415 llvm::FunctionType::get(
VoidTy,
false);
4419 llvm_unreachable(
"Sema will ensure that the parameter is constant");
4421 uint64_t ZExtValue = Value.zextOrTrunc(IsThumb ? 16 : 32).getZExtValue();
4423 llvm::InlineAsm *Emit =
4424 IsThumb ? InlineAsm::get(FTy,
".inst.n 0x" + utohexstr(ZExtValue),
"",
4426 : InlineAsm::get(FTy,
".inst 0x" + utohexstr(ZExtValue),
"",
4429 return Builder.CreateCall(Emit);
4432 if (BuiltinID == ARM::BI__builtin_arm_dbg) {
4437 if (BuiltinID == ARM::BI__builtin_arm_prefetch) {
4443 Value *Locality = llvm::ConstantInt::get(
Int32Ty, 3);
4446 return Builder.CreateCall(F, {Address, RW, Locality, IsData});
4449 if (BuiltinID == ARM::BI__builtin_arm_rbit) {
4452 CGM.
getIntrinsic(Intrinsic::bitreverse, Arg->getType()), Arg,
"rbit");
4455 if (BuiltinID == ARM::BI__clear_cache) {
4456 assert(E->
getNumArgs() == 2 &&
"__clear_cache takes 2 arguments");
4459 for (
unsigned i = 0; i < 2; i++)
4462 llvm::FunctionType *FTy = cast<llvm::FunctionType>(Ty);
4463 StringRef Name = FD->getName();
4467 if (BuiltinID == ARM::BI__builtin_arm_mcrr ||
4468 BuiltinID == ARM::BI__builtin_arm_mcrr2) {
4471 switch (BuiltinID) {
4472 default: llvm_unreachable(
"unexpected builtin");
4473 case ARM::BI__builtin_arm_mcrr:
4476 case ARM::BI__builtin_arm_mcrr2:
4493 Value *C1 = llvm::ConstantInt::get(
Int64Ty, 32);
4495 Value *Rt2 =
Builder.CreateLShr(RtAndRt2, C1);
4498 return Builder.CreateCall(F, {Coproc, Opc1, Rt, Rt2, CRm});
4501 if (BuiltinID == ARM::BI__builtin_arm_mrrc ||
4502 BuiltinID == ARM::BI__builtin_arm_mrrc2) {
4505 switch (BuiltinID) {
4506 default: llvm_unreachable(
"unexpected builtin");
4507 case ARM::BI__builtin_arm_mrrc:
4510 case ARM::BI__builtin_arm_mrrc2:
4518 Value *RtAndRt2 =
Builder.CreateCall(F, {Coproc, Opc1, CRm});
4523 Value *Rt =
Builder.CreateExtractValue(RtAndRt2, 1);
4524 Value *Rt1 =
Builder.CreateExtractValue(RtAndRt2, 0);
4528 Value *ShiftCast = llvm::ConstantInt::get(
Int64Ty, 32);
4529 RtAndRt2 =
Builder.CreateShl(Rt, ShiftCast,
"shl",
true);
4530 RtAndRt2 =
Builder.CreateOr(RtAndRt2, Rt1);
4535 if (BuiltinID == ARM::BI__builtin_arm_ldrexd ||
4536 ((BuiltinID == ARM::BI__builtin_arm_ldrex ||
4537 BuiltinID == ARM::BI__builtin_arm_ldaex) &&
4539 BuiltinID == ARM::BI__ldrexd) {
4542 switch (BuiltinID) {
4543 default: llvm_unreachable(
"unexpected builtin");
4544 case ARM::BI__builtin_arm_ldaex:
4547 case ARM::BI__builtin_arm_ldrexd:
4548 case ARM::BI__builtin_arm_ldrex:
4549 case ARM::BI__ldrexd:
4558 Value *Val0 =
Builder.CreateExtractValue(Val, 1);
4559 Value *Val1 =
Builder.CreateExtractValue(Val, 0);
4563 Value *ShiftCst = llvm::ConstantInt::get(
Int64Ty, 32);
4564 Val =
Builder.CreateShl(Val0, ShiftCst,
"shl",
true );
4565 Val =
Builder.CreateOr(Val, Val1);
4569 if (BuiltinID == ARM::BI__builtin_arm_ldrex ||
4570 BuiltinID == ARM::BI__builtin_arm_ldaex) {
4579 Function *F =
CGM.
getIntrinsic(BuiltinID == ARM::BI__builtin_arm_ldaex
4580 ? Intrinsic::arm_ldaex
4581 : Intrinsic::arm_ldrex,
4583 Value *Val =
Builder.CreateCall(F, LoadAddr,
"ldrex");
4585 if (RealResTy->isPointerTy())
4586 return Builder.CreateIntToPtr(Val, RealResTy);
4588 llvm::Type *IntResTy = llvm::IntegerType::get(
4590 Val =
Builder.CreateTruncOrBitCast(Val, IntResTy);
4595 if (BuiltinID == ARM::BI__builtin_arm_strexd ||
4596 ((BuiltinID == ARM::BI__builtin_arm_stlex ||
4597 BuiltinID == ARM::BI__builtin_arm_strex) &&
4599 Function *F =
CGM.
getIntrinsic(BuiltinID == ARM::BI__builtin_arm_stlex
4600 ? Intrinsic::arm_stlexd
4601 : Intrinsic::arm_strexd);
4611 Value *Arg0 =
Builder.CreateExtractValue(Val, 0);
4612 Value *Arg1 =
Builder.CreateExtractValue(Val, 1);
4614 return Builder.CreateCall(F, {Arg0, Arg1, StPtr},
"strexd");
4617 if (BuiltinID == ARM::BI__builtin_arm_strex ||
4618 BuiltinID == ARM::BI__builtin_arm_stlex) {
4627 if (StoreVal->getType()->isPointerTy())
4637 Function *F =
CGM.
getIntrinsic(BuiltinID == ARM::BI__builtin_arm_stlex
4638 ? Intrinsic::arm_stlex
4639 : Intrinsic::arm_strex,
4640 StoreAddr->getType());
4641 return Builder.CreateCall(F, {StoreVal, StoreAddr},
"strex");
4644 switch (BuiltinID) {
4645 case ARM::BI__iso_volatile_load8:
4646 case ARM::BI__iso_volatile_load16:
4647 case ARM::BI__iso_volatile_load32:
4648 case ARM::BI__iso_volatile_load64: {
4655 llvm::LoadInst *Load =
4657 Load->setVolatile(
true);
4660 case ARM::BI__iso_volatile_store8:
4661 case ARM::BI__iso_volatile_store16:
4662 case ARM::BI__iso_volatile_store32:
4663 case ARM::BI__iso_volatile_store64: {
4671 llvm::StoreInst *Store =
4674 Store->setVolatile(
true);
4679 if (BuiltinID == ARM::BI__builtin_arm_clrex) {
4686 switch (BuiltinID) {
4687 case ARM::BI__builtin_arm_crc32b:
4688 CRCIntrinsicID = Intrinsic::arm_crc32b;
break;
4689 case ARM::BI__builtin_arm_crc32cb:
4690 CRCIntrinsicID = Intrinsic::arm_crc32cb;
break;
4691 case ARM::BI__builtin_arm_crc32h:
4692 CRCIntrinsicID = Intrinsic::arm_crc32h;
break;
4693 case ARM::BI__builtin_arm_crc32ch:
4694 CRCIntrinsicID = Intrinsic::arm_crc32ch;
break;
4695 case ARM::BI__builtin_arm_crc32w:
4696 case ARM::BI__builtin_arm_crc32d:
4697 CRCIntrinsicID = Intrinsic::arm_crc32w;
break;
4698 case ARM::BI__builtin_arm_crc32cw:
4699 case ARM::BI__builtin_arm_crc32cd:
4700 CRCIntrinsicID = Intrinsic::arm_crc32cw;
break;
4703 if (CRCIntrinsicID != Intrinsic::not_intrinsic) {
4709 if (BuiltinID == ARM::BI__builtin_arm_crc32d ||
4710 BuiltinID == ARM::BI__builtin_arm_crc32cd) {
4711 Value *C1 = llvm::ConstantInt::get(
Int64Ty, 32);
4713 Value *Arg1b =
Builder.CreateLShr(Arg1, C1);
4717 Value *Res =
Builder.CreateCall(F, {Arg0, Arg1a});
4718 return Builder.CreateCall(F, {Res, Arg1b});
4723 return Builder.CreateCall(F, {Arg0, Arg1});
4727 if (BuiltinID == ARM::BI__builtin_arm_rsr ||
4728 BuiltinID == ARM::BI__builtin_arm_rsr64 ||
4729 BuiltinID == ARM::BI__builtin_arm_rsrp ||
4730 BuiltinID == ARM::BI__builtin_arm_wsr ||
4731 BuiltinID == ARM::BI__builtin_arm_wsr64 ||
4732 BuiltinID == ARM::BI__builtin_arm_wsrp) {
4734 bool IsRead = BuiltinID == ARM::BI__builtin_arm_rsr ||
4735 BuiltinID == ARM::BI__builtin_arm_rsr64 ||
4736 BuiltinID == ARM::BI__builtin_arm_rsrp;
4738 bool IsPointerBuiltin = BuiltinID == ARM::BI__builtin_arm_rsrp ||
4739 BuiltinID == ARM::BI__builtin_arm_wsrp;
4741 bool Is64Bit = BuiltinID == ARM::BI__builtin_arm_rsr64 ||
4742 BuiltinID == ARM::BI__builtin_arm_wsr64;
4746 if (IsPointerBuiltin) {
4749 }
else if (Is64Bit) {
4750 ValueType = RegisterType =
Int64Ty;
4752 ValueType = RegisterType =
Int32Ty;
4760 unsigned ICEArguments = 0;
4765 auto getAlignmentValue32 = [&](Address addr) -> Value* {
4766 return Builder.getInt32(addr.getAlignment().getQuantity());
4773 unsigned NumArgs = E->
getNumArgs() - (HasExtraArg ? 1 : 0);
4774 for (
unsigned i = 0, e = NumArgs; i != e; i++) {
4776 switch (BuiltinID) {
4777 case NEON::BI__builtin_neon_vld1_v:
4778 case NEON::BI__builtin_neon_vld1q_v:
4779 case NEON::BI__builtin_neon_vld1q_lane_v:
4780 case NEON::BI__builtin_neon_vld1_lane_v:
4781 case NEON::BI__builtin_neon_vld1_dup_v:
4782 case NEON::BI__builtin_neon_vld1q_dup_v:
4783 case NEON::BI__builtin_neon_vst1_v:
4784 case NEON::BI__builtin_neon_vst1q_v:
4785 case NEON::BI__builtin_neon_vst1q_lane_v:
4786 case NEON::BI__builtin_neon_vst1_lane_v:
4787 case NEON::BI__builtin_neon_vst2_v:
4788 case NEON::BI__builtin_neon_vst2q_v:
4789 case NEON::BI__builtin_neon_vst2_lane_v:
4790 case NEON::BI__builtin_neon_vst2q_lane_v:
4791 case NEON::BI__builtin_neon_vst3_v:
4792 case NEON::BI__builtin_neon_vst3q_v:
4793 case NEON::BI__builtin_neon_vst3_lane_v:
4794 case NEON::BI__builtin_neon_vst3q_lane_v:
4795 case NEON::BI__builtin_neon_vst4_v:
4796 case NEON::BI__builtin_neon_vst4q_v:
4797 case NEON::BI__builtin_neon_vst4_lane_v:
4798 case NEON::BI__builtin_neon_vst4q_lane_v:
4807 switch (BuiltinID) {
4808 case NEON::BI__builtin_neon_vld2_v:
4809 case NEON::BI__builtin_neon_vld2q_v:
4810 case NEON::BI__builtin_neon_vld3_v:
4811 case NEON::BI__builtin_neon_vld3q_v:
4812 case NEON::BI__builtin_neon_vld4_v:
4813 case NEON::BI__builtin_neon_vld4q_v:
4814 case NEON::BI__builtin_neon_vld2_lane_v:
4815 case NEON::BI__builtin_neon_vld2q_lane_v:
4816 case NEON::BI__builtin_neon_vld3_lane_v:
4817 case NEON::BI__builtin_neon_vld3q_lane_v:
4818 case NEON::BI__builtin_neon_vld4_lane_v:
4819 case NEON::BI__builtin_neon_vld4q_lane_v:
4820 case NEON::BI__builtin_neon_vld2_dup_v:
4821 case NEON::BI__builtin_neon_vld3_dup_v:
4822 case NEON::BI__builtin_neon_vld4_dup_v:
4831 if ((ICEArguments & (1 << i)) == 0) {
4836 llvm::APSInt Result;
4838 assert(IsConst &&
"Constant arg isn't actually constant?"); (void)IsConst;
4843 switch (BuiltinID) {
4846 case NEON::BI__builtin_neon_vget_lane_i8:
4847 case NEON::BI__builtin_neon_vget_lane_i16:
4848 case NEON::BI__builtin_neon_vget_lane_i32:
4849 case NEON::BI__builtin_neon_vget_lane_i64:
4850 case NEON::BI__builtin_neon_vget_lane_f32:
4851 case NEON::BI__builtin_neon_vgetq_lane_i8:
4852 case NEON::BI__builtin_neon_vgetq_lane_i16:
4853 case NEON::BI__builtin_neon_vgetq_lane_i32:
4854 case NEON::BI__builtin_neon_vgetq_lane_i64:
4855 case NEON::BI__builtin_neon_vgetq_lane_f32:
4856 return Builder.CreateExtractElement(Ops[0], Ops[1],
"vget_lane");
4858 case NEON::BI__builtin_neon_vset_lane_i8:
4859 case NEON::BI__builtin_neon_vset_lane_i16:
4860 case NEON::BI__builtin_neon_vset_lane_i32:
4861 case NEON::BI__builtin_neon_vset_lane_i64:
4862 case NEON::BI__builtin_neon_vset_lane_f32:
4863 case NEON::BI__builtin_neon_vsetq_lane_i8:
4864 case NEON::BI__builtin_neon_vsetq_lane_i16:
4865 case NEON::BI__builtin_neon_vsetq_lane_i32:
4866 case NEON::BI__builtin_neon_vsetq_lane_i64:
4867 case NEON::BI__builtin_neon_vsetq_lane_f32:
4868 return Builder.CreateInsertElement(Ops[1], Ops[0], Ops[2],
"vset_lane");
4870 case NEON::BI__builtin_neon_vsha1h_u32:
4873 case NEON::BI__builtin_neon_vsha1cq_u32:
4876 case NEON::BI__builtin_neon_vsha1pq_u32:
4879 case NEON::BI__builtin_neon_vsha1mq_u32:
4885 case ARM::BI_MoveToCoprocessor:
4886 case ARM::BI_MoveToCoprocessor2: {
4887 Function *F =
CGM.
getIntrinsic(BuiltinID == ARM::BI_MoveToCoprocessor ?
4888 Intrinsic::arm_mcr : Intrinsic::arm_mcr2);
4889 return Builder.CreateCall(F, {Ops[1], Ops[2], Ops[0],
4890 Ops[3], Ops[4], Ops[5]});
4892 case ARM::BI_BitScanForward:
4893 case ARM::BI_BitScanForward64:
4895 case ARM::BI_BitScanReverse:
4896 case ARM::BI_BitScanReverse64:
4899 case ARM::BI_InterlockedAnd64:
4901 case ARM::BI_InterlockedExchange64:
4903 case ARM::BI_InterlockedExchangeAdd64:
4905 case ARM::BI_InterlockedExchangeSub64:
4907 case ARM::BI_InterlockedOr64:
4909 case ARM::BI_InterlockedXor64:
4911 case ARM::BI_InterlockedDecrement64:
4913 case ARM::BI_InterlockedIncrement64:
4918 assert(HasExtraArg);
4919 llvm::APSInt Result;
4924 if (BuiltinID == ARM::BI__builtin_arm_vcvtr_f ||
4925 BuiltinID == ARM::BI__builtin_arm_vcvtr_d) {
4928 if (BuiltinID == ARM::BI__builtin_arm_vcvtr_f)
4934 bool usgn = Result.getZExtValue() == 1;
4935 unsigned Int = usgn ? Intrinsic::arm_vcvtru : Intrinsic::arm_vcvtr;
4939 return Builder.CreateCall(F, Ops,
"vcvtr");
4944 bool usgn = Type.isUnsigned();
4945 bool rightShift =
false;
4954 auto IntrinsicMap = makeArrayRef(ARMSIMDIntrinsicMap);
4956 IntrinsicMap, BuiltinID, NEONSIMDIntrinsicsProvenSorted);
4959 Builtin->BuiltinID, Builtin->LLVMIntrinsic, Builtin->AltLLVMIntrinsic,
4960 Builtin->NameHint, Builtin->TypeModifier, E, Ops, PtrOp0, PtrOp1);
4963 switch (BuiltinID) {
4964 default:
return nullptr;
4965 case NEON::BI__builtin_neon_vld1q_lane_v:
4968 if (VTy->getElementType()->isIntegerTy(64)) {
4971 uint32_t Lane = cast<ConstantInt>(Ops[2])->getZExtValue();
4972 Value *SV = llvm::ConstantVector::get(ConstantInt::get(
Int32Ty, 1-Lane));
4973 Ops[1] =
Builder.CreateShuffleVector(Ops[1], Ops[1], SV);
4975 Ty = llvm::VectorType::get(VTy->getElementType(), 1);
4978 Value *Align = getAlignmentValue32(PtrOp0);
4979 Value *Ld =
Builder.CreateCall(F, {Ops[0], Align});
4981 uint32_t Indices[] = {1 - Lane, Lane};
4983 return Builder.CreateShuffleVector(Ops[1], Ld, SV,
"vld1q_lane");
4986 case NEON::BI__builtin_neon_vld1_lane_v: {
4990 return Builder.CreateInsertElement(Ops[1], Ld, Ops[2],
"vld1_lane");
4992 case NEON::BI__builtin_neon_vld2_dup_v:
4993 case NEON::BI__builtin_neon_vld3_dup_v:
4994 case NEON::BI__builtin_neon_vld4_dup_v: {
4996 if (VTy->getElementType()->getPrimitiveSizeInBits() == 64) {
4997 switch (BuiltinID) {
4998 case NEON::BI__builtin_neon_vld2_dup_v:
4999 Int = Intrinsic::arm_neon_vld2;
5001 case NEON::BI__builtin_neon_vld3_dup_v:
5002 Int = Intrinsic::arm_neon_vld3;
5004 case NEON::BI__builtin_neon_vld4_dup_v:
5005 Int = Intrinsic::arm_neon_vld4;
5007 default: llvm_unreachable(
"unknown vld_dup intrinsic?");
5012 Ops[1] =
Builder.CreateCall(F, {Ops[1], Align},
"vld_dup");
5013 Ty = llvm::PointerType::getUnqual(Ops[1]->getType());
5017 switch (BuiltinID) {
5018 case NEON::BI__builtin_neon_vld2_dup_v:
5019 Int = Intrinsic::arm_neon_vld2lane;
5021 case NEON::BI__builtin_neon_vld3_dup_v:
5022 Int = Intrinsic::arm_neon_vld3lane;
5024 case NEON::BI__builtin_neon_vld4_dup_v:
5025 Int = Intrinsic::arm_neon_vld4lane;
5027 default: llvm_unreachable(
"unknown vld_dup intrinsic?");
5031 llvm::StructType *STy = cast<llvm::StructType>(F->getReturnType());
5034 Args.push_back(Ops[1]);
5035 Args.append(STy->getNumElements(), UndefValue::get(Ty));
5037 llvm::Constant *CI = ConstantInt::get(
Int32Ty, 0);
5039 Args.push_back(getAlignmentValue32(PtrOp1));
5041 Ops[1] =
Builder.CreateCall(F, Args,
"vld_dup");
5043 for (
unsigned i = 0, e = STy->getNumElements(); i != e; ++i) {
5044 Value *Val =
Builder.CreateExtractValue(Ops[1], i);
5048 Ops[1] =
Builder.CreateInsertValue(Ops[1], Elt, i);
5050 Ty = llvm::PointerType::getUnqual(Ops[1]->getType());
5054 case NEON::BI__builtin_neon_vqrshrn_n_v:
5056 usgn ? Intrinsic::arm_neon_vqrshiftnu : Intrinsic::arm_neon_vqrshiftns;
5059 case NEON::BI__builtin_neon_vqrshrun_n_v:
5061 Ops,
"vqrshrun_n", 1,
true);
5062 case NEON::BI__builtin_neon_vqshrn_n_v:
5063 Int = usgn ? Intrinsic::arm_neon_vqshiftnu : Intrinsic::arm_neon_vqshiftns;
5066 case NEON::BI__builtin_neon_vqshrun_n_v:
5068 Ops,
"vqshrun_n", 1,
true);
5069 case NEON::BI__builtin_neon_vrecpe_v:
5070 case NEON::BI__builtin_neon_vrecpeq_v:
5073 case NEON::BI__builtin_neon_vrshrn_n_v:
5075 Ops,
"vrshrn_n", 1,
true);
5076 case NEON::BI__builtin_neon_vrsra_n_v:
5077 case NEON::BI__builtin_neon_vrsraq_n_v:
5081 Int = usgn ? Intrinsic::arm_neon_vrshiftu : Intrinsic::arm_neon_vrshifts;
5083 return Builder.CreateAdd(Ops[0], Ops[1],
"vrsra_n");
5084 case NEON::BI__builtin_neon_vsri_n_v:
5085 case NEON::BI__builtin_neon_vsriq_n_v:
5088 case NEON::BI__builtin_neon_vsli_n_v:
5089 case NEON::BI__builtin_neon_vsliq_n_v:
5093 case NEON::BI__builtin_neon_vsra_n_v:
5094 case NEON::BI__builtin_neon_vsraq_n_v:
5097 return Builder.CreateAdd(Ops[0], Ops[1]);
5098 case NEON::BI__builtin_neon_vst1q_lane_v:
5101 if (VTy->getElementType()->isIntegerTy(64)) {
5103 Value *SV = llvm::ConstantVector::get(cast<llvm::Constant>(Ops[2]));
5104 Ops[1] =
Builder.CreateShuffleVector(Ops[1], Ops[1], SV);
5105 Ops[2] = getAlignmentValue32(PtrOp0);
5111 case NEON::BI__builtin_neon_vst1_lane_v: {
5113 Ops[1] =
Builder.CreateExtractElement(Ops[1], Ops[2]);
5114 Ty = llvm::PointerType::getUnqual(Ops[1]->getType());
5118 case NEON::BI__builtin_neon_vtbl1_v:
5121 case NEON::BI__builtin_neon_vtbl2_v:
5124 case NEON::BI__builtin_neon_vtbl3_v:
5127 case NEON::BI__builtin_neon_vtbl4_v:
5130 case NEON::BI__builtin_neon_vtbx1_v:
5133 case NEON::BI__builtin_neon_vtbx2_v:
5136 case NEON::BI__builtin_neon_vtbx3_v:
5139 case NEON::BI__builtin_neon_vtbx4_v:
5148 unsigned int Int = 0;
5149 const char *s =
nullptr;
5151 switch (BuiltinID) {
5154 case NEON::BI__builtin_neon_vtbl1_v:
5155 case NEON::BI__builtin_neon_vqtbl1_v:
5156 case NEON::BI__builtin_neon_vqtbl1q_v:
5157 case NEON::BI__builtin_neon_vtbl2_v:
5158 case NEON::BI__builtin_neon_vqtbl2_v:
5159 case NEON::BI__builtin_neon_vqtbl2q_v:
5160 case NEON::BI__builtin_neon_vtbl3_v:
5161 case NEON::BI__builtin_neon_vqtbl3_v:
5162 case NEON::BI__builtin_neon_vqtbl3q_v:
5163 case NEON::BI__builtin_neon_vtbl4_v:
5164 case NEON::BI__builtin_neon_vqtbl4_v:
5165 case NEON::BI__builtin_neon_vqtbl4q_v:
5167 case NEON::BI__builtin_neon_vtbx1_v:
5168 case NEON::BI__builtin_neon_vqtbx1_v:
5169 case NEON::BI__builtin_neon_vqtbx1q_v:
5170 case NEON::BI__builtin_neon_vtbx2_v:
5171 case NEON::BI__builtin_neon_vqtbx2_v:
5172 case NEON::BI__builtin_neon_vqtbx2q_v:
5173 case NEON::BI__builtin_neon_vtbx3_v:
5174 case NEON::BI__builtin_neon_vqtbx3_v:
5175 case NEON::BI__builtin_neon_vqtbx3q_v:
5176 case NEON::BI__builtin_neon_vtbx4_v:
5177 case NEON::BI__builtin_neon_vqtbx4_v:
5178 case NEON::BI__builtin_neon_vqtbx4q_v:
5185 llvm::APSInt Result;
5200 switch (BuiltinID) {
5201 case NEON::BI__builtin_neon_vtbl1_v: {
5203 Ops[1], Ty, Intrinsic::aarch64_neon_tbl1,
5206 case NEON::BI__builtin_neon_vtbl2_v: {
5208 Ops[2], Ty, Intrinsic::aarch64_neon_tbl1,
5211 case NEON::BI__builtin_neon_vtbl3_v: {
5213 Ops[3], Ty, Intrinsic::aarch64_neon_tbl2,
5216 case NEON::BI__builtin_neon_vtbl4_v: {
5218 Ops[4], Ty, Intrinsic::aarch64_neon_tbl2,
5221 case NEON::BI__builtin_neon_vtbx1_v: {
5224 Ty, Intrinsic::aarch64_neon_tbl1,
"vtbl1");
5226 llvm::Constant *EightV = ConstantInt::get(Ty, 8);
5227 Value *CmpRes = Builder.CreateICmp(ICmpInst::ICMP_UGE, Ops[2], EightV);
5228 CmpRes = Builder.CreateSExt(CmpRes, Ty);
5230 Value *EltsFromInput = Builder.CreateAnd(CmpRes, Ops[0]);
5231 Value *EltsFromTbl = Builder.CreateAnd(Builder.CreateNot(CmpRes), TblRes);
5232 return Builder.CreateOr(EltsFromInput, EltsFromTbl,
"vtbx");
5234 case NEON::BI__builtin_neon_vtbx2_v: {
5236 Ops[3], Ty, Intrinsic::aarch64_neon_tbx1,
5239 case NEON::BI__builtin_neon_vtbx3_v: {
5242 Ty, Intrinsic::aarch64_neon_tbl2,
"vtbl2");
5244 llvm::Constant *TwentyFourV = ConstantInt::get(Ty, 24);
5245 Value *CmpRes = Builder.CreateICmp(ICmpInst::ICMP_UGE, Ops[4],
5247 CmpRes = Builder.CreateSExt(CmpRes, Ty);
5249 Value *EltsFromInput = Builder.CreateAnd(CmpRes, Ops[0]);
5250 Value *EltsFromTbl = Builder.CreateAnd(Builder.CreateNot(CmpRes), TblRes);
5251 return Builder.CreateOr(EltsFromInput, EltsFromTbl,
"vtbx");
5253 case NEON::BI__builtin_neon_vtbx4_v: {
5255 Ops[5], Ty, Intrinsic::aarch64_neon_tbx2,
5258 case NEON::BI__builtin_neon_vqtbl1_v:
5259 case NEON::BI__builtin_neon_vqtbl1q_v:
5260 Int = Intrinsic::aarch64_neon_tbl1; s =
"vtbl1";
break;
5261 case NEON::BI__builtin_neon_vqtbl2_v:
5262 case NEON::BI__builtin_neon_vqtbl2q_v: {
5263 Int = Intrinsic::aarch64_neon_tbl2; s =
"vtbl2";
break;
5264 case NEON::BI__builtin_neon_vqtbl3_v:
5265 case NEON::BI__builtin_neon_vqtbl3q_v:
5266 Int = Intrinsic::aarch64_neon_tbl3; s =
"vtbl3";
break;
5267 case NEON::BI__builtin_neon_vqtbl4_v:
5268 case NEON::BI__builtin_neon_vqtbl4q_v:
5269 Int = Intrinsic::aarch64_neon_tbl4; s =
"vtbl4";
break;
5270 case NEON::BI__builtin_neon_vqtbx1_v:
5271 case NEON::BI__builtin_neon_vqtbx1q_v:
5272 Int = Intrinsic::aarch64_neon_tbx1; s =
"vtbx1";
break;
5273 case NEON::BI__builtin_neon_vqtbx2_v:
5274 case NEON::BI__builtin_neon_vqtbx2q_v:
5275 Int = Intrinsic::aarch64_neon_tbx2; s =
"vtbx2";
break;
5276 case NEON::BI__builtin_neon_vqtbx3_v:
5277 case NEON::BI__builtin_neon_vqtbx3q_v:
5278 Int = Intrinsic::aarch64_neon_tbx3; s =
"vtbx3";
break;
5279 case NEON::BI__builtin_neon_vqtbx4_v:
5280 case NEON::BI__builtin_neon_vqtbx4q_v:
5281 Int = Intrinsic::aarch64_neon_tbx4; s =
"vtbx4";
break;
5295 Value *V = UndefValue::get(VTy);
5296 llvm::Constant *CI = ConstantInt::get(
SizeTy, 0);
5297 Op =
Builder.CreateInsertElement(V, Op, CI);
5303 unsigned HintID =
static_cast<unsigned>(-1);
5304 switch (BuiltinID) {
5306 case AArch64::BI__builtin_arm_nop:
5309 case AArch64::BI__builtin_arm_yield:
5312 case AArch64::BI__builtin_arm_wfe:
5315 case AArch64::BI__builtin_arm_wfi:
5318 case AArch64::BI__builtin_arm_sev:
5321 case AArch64::BI__builtin_arm_sevl:
5326 if (HintID != static_cast<unsigned>(-1)) {
5328 return Builder.CreateCall(F, llvm::ConstantInt::get(
Int32Ty, HintID));
5331 if (BuiltinID == AArch64::BI__builtin_arm_prefetch) {
5338 Value *Locality =
nullptr;
5339 if (cast<llvm::ConstantInt>(RetentionPolicy)->isZero()) {
5341 Locality = llvm::ConstantInt::get(
Int32Ty,
5342 -cast<llvm::ConstantInt>(CacheLevel)->
getValue() + 3);
5345 Locality = llvm::ConstantInt::get(
Int32Ty, 0);
5351 return Builder.CreateCall(F, {Address, RW, Locality, IsData});
5354 if (BuiltinID == AArch64::BI__builtin_arm_rbit) {
5356 "rbit of unusual size!");
5359 CGM.
getIntrinsic(Intrinsic::bitreverse, Arg->getType()), Arg,
"rbit");
5361 if (BuiltinID == AArch64::BI__builtin_arm_rbit64) {
5363 "rbit of unusual size!");
5366 CGM.
getIntrinsic(Intrinsic::bitreverse, Arg->getType()), Arg,
"rbit");
5369 if (BuiltinID == AArch64::BI__clear_cache) {
5370 assert(E->
getNumArgs() == 2 &&
"__clear_cache takes 2 arguments");
5373 for (
unsigned i = 0; i < 2; i++)
5376 llvm::FunctionType *FTy = cast<llvm::FunctionType>(Ty);
5377 StringRef Name = FD->getName();
5381 if ((BuiltinID == AArch64::BI__builtin_arm_ldrex ||
5382 BuiltinID == AArch64::BI__builtin_arm_ldaex) &&
5384 Function *F =
CGM.
getIntrinsic(BuiltinID == AArch64::BI__builtin_arm_ldaex
5385 ? Intrinsic::aarch64_ldaxp
5386 : Intrinsic::aarch64_ldxp);
5392 Value *Val0 =
Builder.CreateExtractValue(Val, 1);
5393 Value *Val1 =
Builder.CreateExtractValue(Val, 0);
5395 Val0 =
Builder.CreateZExt(Val0, Int128Ty);
5396 Val1 =
Builder.CreateZExt(Val1, Int128Ty);
5398 Value *ShiftCst = llvm::ConstantInt::get(Int128Ty, 64);
5399 Val =
Builder.CreateShl(Val0, ShiftCst,
"shl",
true );
5400 Val =
Builder.CreateOr(Val, Val1);
5402 }
else if (BuiltinID == AArch64::BI__builtin_arm_ldrex ||
5403 BuiltinID == AArch64::BI__builtin_arm_ldaex) {
5412 Function *F =
CGM.
getIntrinsic(BuiltinID == AArch64::BI__builtin_arm_ldaex
5413 ? Intrinsic::aarch64_ldaxr
5414 : Intrinsic::aarch64_ldxr,
5416 Value *Val =
Builder.CreateCall(F, LoadAddr,
"ldxr");
5418 if (RealResTy->isPointerTy())
5419 return Builder.CreateIntToPtr(Val, RealResTy);
5421 llvm::Type *IntResTy = llvm::IntegerType::get(
5423 Val =
Builder.CreateTruncOrBitCast(Val, IntResTy);
5427 if ((BuiltinID == AArch64::BI__builtin_arm_strex ||
5428 BuiltinID == AArch64::BI__builtin_arm_stlex) &&
5430 Function *F =
CGM.
getIntrinsic(BuiltinID == AArch64::BI__builtin_arm_stlex
5431 ? Intrinsic::aarch64_stlxp
5432 : Intrinsic::aarch64_stxp);
5441 Value *Arg0 =
Builder.CreateExtractValue(Val, 0);
5442 Value *Arg1 =
Builder.CreateExtractValue(Val, 1);
5445 return Builder.CreateCall(F, {Arg0, Arg1, StPtr},
"stxp");
5448 if (BuiltinID == AArch64::BI__builtin_arm_strex ||
5449 BuiltinID == AArch64::BI__builtin_arm_stlex) {
5458 if (StoreVal->getType()->isPointerTy())
5468 Function *F =
CGM.
getIntrinsic(BuiltinID == AArch64::BI__builtin_arm_stlex
5469 ? Intrinsic::aarch64_stlxr
5470 : Intrinsic::aarch64_stxr,
5471 StoreAddr->getType());
5472 return Builder.CreateCall(F, {StoreVal, StoreAddr},
"stxr");
5475 if (BuiltinID == AArch64::BI__builtin_arm_clrex) {
5482 switch (BuiltinID) {
5483 case AArch64::BI__builtin_arm_crc32b:
5484 CRCIntrinsicID = Intrinsic::aarch64_crc32b;
break;
5485 case AArch64::BI__builtin_arm_crc32cb:
5486 CRCIntrinsicID = Intrinsic::aarch64_crc32cb;
break;
5487 case AArch64::BI__builtin_arm_crc32h:
5488 CRCIntrinsicID = Intrinsic::aarch64_crc32h;
break;
5489 case AArch64::BI__builtin_arm_crc32ch:
5490 CRCIntrinsicID = Intrinsic::aarch64_crc32ch;
break;
5491 case AArch64::BI__builtin_arm_crc32w:
5492 CRCIntrinsicID = Intrinsic::aarch64_crc32w;
break;
5493 case AArch64::BI__builtin_arm_crc32cw:
5494 CRCIntrinsicID = Intrinsic::aarch64_crc32cw;
break;
5495 case AArch64::BI__builtin_arm_crc32d:
5496 CRCIntrinsicID = Intrinsic::aarch64_crc32x;
break;
5497 case AArch64::BI__builtin_arm_crc32cd:
5498 CRCIntrinsicID = Intrinsic::aarch64_crc32cx;
break;
5501 if (CRCIntrinsicID != Intrinsic::not_intrinsic) {
5506 llvm::Type *DataTy = F->getFunctionType()->getParamType(1);
5507 Arg1 =
Builder.CreateZExtOrBitCast(Arg1, DataTy);
5509 return Builder.CreateCall(F, {Arg0, Arg1});
5512 if (BuiltinID == AArch64::BI__builtin_arm_rsr ||
5513 BuiltinID == AArch64::BI__builtin_arm_rsr64 ||
5514 BuiltinID == AArch64::BI__builtin_arm_rsrp ||
5515 BuiltinID == AArch64::BI__builtin_arm_wsr ||
5516 BuiltinID == AArch64::BI__builtin_arm_wsr64 ||
5517 BuiltinID == AArch64::BI__builtin_arm_wsrp) {
5519 bool IsRead = BuiltinID == AArch64::BI__builtin_arm_rsr ||
5520 BuiltinID == AArch64::BI__builtin_arm_rsr64 ||
5521 BuiltinID == AArch64::BI__builtin_arm_rsrp;
5523 bool IsPointerBuiltin = BuiltinID == AArch64::BI__builtin_arm_rsrp ||
5524 BuiltinID == AArch64::BI__builtin_arm_wsrp;
5526 bool Is64Bit = BuiltinID != AArch64::BI__builtin_arm_rsr &&
5527 BuiltinID != AArch64::BI__builtin_arm_wsr;
5531 if (IsPointerBuiltin) {
5533 }
else if (Is64Bit) {
5544 unsigned ICEArguments = 0;
5550 for (
unsigned i = 0, e = E->
getNumArgs() - 1; i != e; i++) {
5551 if ((ICEArguments & (1 << i)) == 0) {
5556 llvm::APSInt Result;
5558 assert(IsConst &&
"Constant arg isn't actually constant?");
5564 auto SISDMap = makeArrayRef(AArch64SISDIntrinsicMap);
5566 SISDMap, BuiltinID, AArch64SISDIntrinsicsProvenSorted);
5571 assert(Result &&
"SISD intrinsic should have been handled");
5575 llvm::APSInt Result;
5583 bool quad = Type.
isQuad();
5586 switch (BuiltinID) {
5588 case NEON::BI__builtin_neon_vldrq_p128: {
5590 llvm::Type *Int128PTy = llvm::PointerType::get(Int128Ty, 0);
5595 case NEON::BI__builtin_neon_vstrq_p128: {
5600 case NEON::BI__builtin_neon_vcvts_u32_f32:
5601 case NEON::BI__builtin_neon_vcvtd_u64_f64:
5604 case NEON::BI__builtin_neon_vcvts_s32_f32:
5605 case NEON::BI__builtin_neon_vcvtd_s64_f64: {
5607 bool Is64 = Ops[0]->getType()->getPrimitiveSizeInBits() == 64;
5612 return Builder.CreateFPToUI(Ops[0], InTy);
5613 return Builder.CreateFPToSI(Ops[0], InTy);
5615 case NEON::BI__builtin_neon_vcvts_f32_u32:
5616 case NEON::BI__builtin_neon_vcvtd_f64_u64:
5619 case NEON::BI__builtin_neon_vcvts_f32_s32:
5620 case NEON::BI__builtin_neon_vcvtd_f64_s64: {
5622 bool Is64 = Ops[0]->getType()->getPrimitiveSizeInBits() == 64;
5627 return Builder.CreateUIToFP(Ops[0], FTy);
5628 return Builder.CreateSIToFP(Ops[0], FTy);
5630 case NEON::BI__builtin_neon_vpaddd_s64: {
5637 Value *Op0 =
Builder.CreateExtractElement(Vec, Idx0,
"lane0");
5638 Value *Op1 =
Builder.CreateExtractElement(Vec, Idx1,
"lane1");
5640 return Builder.CreateAdd(Op0, Op1,
"vpaddd");
5642 case NEON::BI__builtin_neon_vpaddd_f64: {
5644 llvm::VectorType::get(
DoubleTy, 2);
5650 Value *Op0 =
Builder.CreateExtractElement(Vec, Idx0,
"lane0");
5651 Value *Op1 =
Builder.CreateExtractElement(Vec, Idx1,
"lane1");
5653 return Builder.CreateFAdd(Op0, Op1,
"vpaddd");
5655 case NEON::BI__builtin_neon_vpadds_f32: {
5657 llvm::VectorType::get(
FloatTy, 2);
5663 Value *Op0 =
Builder.CreateExtractElement(Vec, Idx0,
"lane0");
5664 Value *Op1 =
Builder.CreateExtractElement(Vec, Idx1,
"lane1");
5666 return Builder.CreateFAdd(Op0, Op1,
"vpaddd");
5668 case NEON::BI__builtin_neon_vceqzd_s64:
5669 case NEON::BI__builtin_neon_vceqzd_f64:
5670 case NEON::BI__builtin_neon_vceqzs_f32:
5674 ICmpInst::FCMP_OEQ, ICmpInst::ICMP_EQ,
"vceqz");
5675 case NEON::BI__builtin_neon_vcgezd_s64:
5676 case NEON::BI__builtin_neon_vcgezd_f64:
5677 case NEON::BI__builtin_neon_vcgezs_f32:
5681 ICmpInst::FCMP_OGE, ICmpInst::ICMP_SGE,
"vcgez");
5682 case NEON::BI__builtin_neon_vclezd_s64:
5683 case NEON::BI__builtin_neon_vclezd_f64:
5684 case NEON::BI__builtin_neon_vclezs_f32:
5688 ICmpInst::FCMP_OLE, ICmpInst::ICMP_SLE,
"vclez");
5689 case NEON::BI__builtin_neon_vcgtzd_s64:
5690 case NEON::BI__builtin_neon_vcgtzd_f64:
5691 case NEON::BI__builtin_neon_vcgtzs_f32:
5695 ICmpInst::FCMP_OGT, ICmpInst::ICMP_SGT,
"vcgtz");
5696 case NEON::BI__builtin_neon_vcltzd_s64:
5697 case NEON::BI__builtin_neon_vcltzd_f64:
5698 case NEON::BI__builtin_neon_vcltzs_f32:
5702 ICmpInst::FCMP_OLT, ICmpInst::ICMP_SLT,
"vcltz");
5704 case NEON::BI__builtin_neon_vceqzd_u64: {
5708 Builder.CreateICmpEQ(Ops[0], llvm::Constant::getNullValue(
Int64Ty));
5711 case NEON::BI__builtin_neon_vceqd_f64:
5712 case NEON::BI__builtin_neon_vcled_f64:
5713 case NEON::BI__builtin_neon_vcltd_f64:
5714 case NEON::BI__builtin_neon_vcged_f64:
5715 case NEON::BI__builtin_neon_vcgtd_f64: {
5716 llvm::CmpInst::Predicate
P;
5717 switch (BuiltinID) {
5718 default: llvm_unreachable(
"missing builtin ID in switch!");
5719 case NEON::BI__builtin_neon_vceqd_f64: P = llvm::FCmpInst::FCMP_OEQ;
break;
5720 case NEON::BI__builtin_neon_vcled_f64: P = llvm::FCmpInst::FCMP_OLE;
break;
5721 case NEON::BI__builtin_neon_vcltd_f64: P = llvm::FCmpInst::FCMP_OLT;
break;
5722 case NEON::BI__builtin_neon_vcged_f64: P = llvm::FCmpInst::FCMP_OGE;
break;
5723 case NEON::BI__builtin_neon_vcgtd_f64: P = llvm::FCmpInst::FCMP_OGT;
break;
5728 Ops[0] =
Builder.CreateFCmp(P, Ops[0], Ops[1]);
5731 case NEON::BI__builtin_neon_vceqs_f32:
5732 case NEON::BI__builtin_neon_vcles_f32:
5733 case NEON::BI__builtin_neon_vclts_f32:
5734 case NEON::BI__builtin_neon_vcges_f32:
5735 case NEON::BI__builtin_neon_vcgts_f32: {
5736 llvm::CmpInst::Predicate
P;
5737 switch (BuiltinID) {
5738 default: llvm_unreachable(
"missing builtin ID in switch!");
5739 case NEON::BI__builtin_neon_vceqs_f32: P = llvm::FCmpInst::FCMP_OEQ;
break;
5740 case NEON::BI__builtin_neon_vcles_f32: P = llvm::FCmpInst::FCMP_OLE;
break;
5741 case NEON::BI__builtin_neon_vclts_f32: P = llvm::FCmpInst::FCMP_OLT;
break;
5742 case NEON::BI__builtin_neon_vcges_f32: P = llvm::FCmpInst::FCMP_OGE;
break;
5743 case NEON::BI__builtin_neon_vcgts_f32: P = llvm::FCmpInst::FCMP_OGT;
break;
5748 Ops[0] =
Builder.CreateFCmp(P, Ops[0], Ops[1]);
5751 case NEON::BI__builtin_neon_vceqd_s64:
5752 case NEON::BI__builtin_neon_vceqd_u64:
5753 case NEON::BI__builtin_neon_vcgtd_s64:
5754 case NEON::BI__builtin_neon_vcgtd_u64:
5755 case NEON::BI__builtin_neon_vcltd_s64:
5756 case NEON::BI__builtin_neon_vcltd_u64:
5757 case NEON::BI__builtin_neon_vcged_u64:
5758 case NEON::BI__builtin_neon_vcged_s64:
5759 case NEON::BI__builtin_neon_vcled_u64:
5760 case NEON::BI__builtin_neon_vcled_s64: {
5761 llvm::CmpInst::Predicate
P;
5762 switch (BuiltinID) {
5763 default: llvm_unreachable(
"missing builtin ID in switch!");
5764 case NEON::BI__builtin_neon_vceqd_s64:
5765 case NEON::BI__builtin_neon_vceqd_u64:P = llvm::ICmpInst::ICMP_EQ;
break;
5766 case NEON::BI__builtin_neon_vcgtd_s64:P = llvm::ICmpInst::ICMP_SGT;
break;
5767 case NEON::BI__builtin_neon_vcgtd_u64:P = llvm::ICmpInst::ICMP_UGT;
break;
5768 case NEON::BI__builtin_neon_vcltd_s64:P = llvm::ICmpInst::ICMP_SLT;
break;
5769 case NEON::BI__builtin_neon_vcltd_u64:P = llvm::ICmpInst::ICMP_ULT;
break;
5770 case NEON::BI__builtin_neon_vcged_u64:P = llvm::ICmpInst::ICMP_UGE;
break;
5771 case NEON::BI__builtin_neon_vcged_s64:P = llvm::ICmpInst::ICMP_SGE;
break;
5772 case NEON::BI__builtin_neon_vcled_u64:P = llvm::ICmpInst::ICMP_ULE;
break;
5773 case NEON::BI__builtin_neon_vcled_s64:P = llvm::ICmpInst::ICMP_SLE;
break;
5778 Ops[0] =
Builder.CreateICmp(P, Ops[0], Ops[1]);
5781 case NEON::BI__builtin_neon_vtstd_s64:
5782 case NEON::BI__builtin_neon_vtstd_u64: {
5786 Ops[0] =
Builder.CreateAnd(Ops[0], Ops[1]);
5787 Ops[0] =
Builder.CreateICmp(ICmpInst::ICMP_NE, Ops[0],
5788 llvm::Constant::getNullValue(
Int64Ty));
5791 case NEON::BI__builtin_neon_vset_lane_i8:
5792 case NEON::BI__builtin_neon_vset_lane_i16:
5793 case NEON::BI__builtin_neon_vset_lane_i32:
5794 case NEON::BI__builtin_neon_vset_lane_i64:
5795 case NEON::BI__builtin_neon_vset_lane_f32:
5796 case NEON::BI__builtin_neon_vsetq_lane_i8:
5797 case NEON::BI__builtin_neon_vsetq_lane_i16:
5798 case NEON::BI__builtin_neon_vsetq_lane_i32:
5799 case NEON::BI__builtin_neon_vsetq_lane_i64:
5800 case NEON::BI__builtin_neon_vsetq_lane_f32:
5802 return Builder.CreateInsertElement(Ops[1], Ops[0], Ops[2],
"vset_lane");
5803 case NEON::BI__builtin_neon_vset_lane_f64:
5806 llvm::VectorType::get(
DoubleTy, 1));
5808 return Builder.CreateInsertElement(Ops[1], Ops[0], Ops[2],
"vset_lane");
5809 case NEON::BI__builtin_neon_vsetq_lane_f64:
5812 llvm::VectorType::get(
DoubleTy, 2));
5814 return Builder.CreateInsertElement(Ops[1], Ops[0], Ops[2],
"vset_lane");
5816 case NEON::BI__builtin_neon_vget_lane_i8:
5817 case NEON::BI__builtin_neon_vdupb_lane_i8:
5821 case NEON::BI__builtin_neon_vgetq_lane_i8:
5822 case NEON::BI__builtin_neon_vdupb_laneq_i8:
5826 case NEON::BI__builtin_neon_vget_lane_i16:
5827 case NEON::BI__builtin_neon_vduph_lane_i16:
5831 case NEON::BI__builtin_neon_vgetq_lane_i16:
5832 case NEON::BI__builtin_neon_vduph_laneq_i16:
5836 case NEON::BI__builtin_neon_vget_lane_i32:
5837 case NEON::BI__builtin_neon_vdups_lane_i32:
5841 case NEON::BI__builtin_neon_vdups_lane_f32:
5843 llvm::VectorType::get(
FloatTy, 2));
5846 case NEON::BI__builtin_neon_vgetq_lane_i32:
5847 case NEON::BI__builtin_neon_vdups_laneq_i32:
5851 case NEON::BI__builtin_neon_vget_lane_i64:
5852 case NEON::BI__builtin_neon_vdupd_lane_i64:
5856 case NEON::BI__builtin_neon_vdupd_lane_f64:
5858 llvm::VectorType::get(
DoubleTy, 1));
5861 case NEON::BI__builtin_neon_vgetq_lane_i64:
5862 case NEON::BI__builtin_neon_vdupd_laneq_i64:
5866 case NEON::BI__builtin_neon_vget_lane_f32:
5868 llvm::VectorType::get(
FloatTy, 2));
5871 case NEON::BI__builtin_neon_vget_lane_f64:
5873 llvm::VectorType::get(
DoubleTy, 1));
5876 case NEON::BI__builtin_neon_vgetq_lane_f32:
5877 case NEON::BI__builtin_neon_vdups_laneq_f32:
5879 llvm::VectorType::get(
FloatTy, 4));
5882 case NEON::BI__builtin_neon_vgetq_lane_f64:
5883 case NEON::BI__builtin_neon_vdupd_laneq_f64:
5885 llvm::VectorType::get(
DoubleTy, 2));
5888 case NEON::BI__builtin_neon_vaddd_s64:
5889 case NEON::BI__builtin_neon_vaddd_u64:
5891 case NEON::BI__builtin_neon_vsubd_s64:
5892 case NEON::BI__builtin_neon_vsubd_u64:
5894 case NEON::BI__builtin_neon_vqdmlalh_s16:
5895 case NEON::BI__builtin_neon_vqdmlslh_s16: {
5901 ProductOps,
"vqdmlXl");
5902 Constant *CI = ConstantInt::get(
SizeTy, 0);
5903 Ops[1] =
Builder.CreateExtractElement(Ops[1], CI,
"lane0");
5905 unsigned AccumInt = BuiltinID == NEON::BI__builtin_neon_vqdmlalh_s16
5906 ? Intrinsic::aarch64_neon_sqadd
5907 : Intrinsic::aarch64_neon_sqsub;
5910 case NEON::BI__builtin_neon_vqshlud_n_s64: {
5916 case NEON::BI__builtin_neon_vqshld_n_u64:
5917 case NEON::BI__builtin_neon_vqshld_n_s64: {
5918 unsigned Int = BuiltinID == NEON::BI__builtin_neon_vqshld_n_u64
5919 ? Intrinsic::aarch64_neon_uqshl
5920 : Intrinsic::aarch64_neon_sqshl;
5925 case NEON::BI__builtin_neon_vrshrd_n_u64:
5926 case NEON::BI__builtin_neon_vrshrd_n_s64: {
5927 unsigned Int = BuiltinID == NEON::BI__builtin_neon_vrshrd_n_u64
5928 ? Intrinsic::aarch64_neon_urshl
5929 : Intrinsic::aarch64_neon_srshl;
5931 int SV = cast<ConstantInt>(Ops[1])->getSExtValue();
5932 Ops[1] = ConstantInt::get(
Int64Ty, -SV);
5935 case NEON::BI__builtin_neon_vrsrad_n_u64:
5936 case NEON::BI__builtin_neon_vrsrad_n_s64: {
5937 unsigned Int = BuiltinID == NEON::BI__builtin_neon_vrsrad_n_u64
5938 ? Intrinsic::aarch64_neon_urshl
5939 : Intrinsic::aarch64_neon_srshl;
5946 case NEON::BI__builtin_neon_vshld_n_s64:
5947 case NEON::BI__builtin_neon_vshld_n_u64: {
5950 Ops[0], ConstantInt::get(
Int64Ty, Amt->getZExtValue()),
"shld_n");
5952 case NEON::BI__builtin_neon_vshrd_n_s64: {
5956 Amt->getZExtValue())),
5959 case NEON::BI__builtin_neon_vshrd_n_u64: {
5961 uint64_t ShiftAmt = Amt->getZExtValue();
5964 return ConstantInt::get(
Int64Ty, 0);
5965 return Builder.CreateLShr(Ops[0], ConstantInt::get(
Int64Ty, ShiftAmt),
5968 case NEON::BI__builtin_neon_vsrad_n_s64: {
5972 Amt->getZExtValue())),
5974 return Builder.CreateAdd(Ops[0], Ops[1]);
5976 case NEON::BI__builtin_neon_vsrad_n_u64: {
5978 uint64_t ShiftAmt = Amt->getZExtValue();
5983 Ops[1] =
Builder.CreateLShr(Ops[1], ConstantInt::get(
Int64Ty, ShiftAmt),
5985 return Builder.CreateAdd(Ops[0], Ops[1]);
5987 case NEON::BI__builtin_neon_vqdmlalh_lane_s16:
5988 case NEON::BI__builtin_neon_vqdmlalh_laneq_s16:
5989 case NEON::BI__builtin_neon_vqdmlslh_lane_s16:
5990 case NEON::BI__builtin_neon_vqdmlslh_laneq_s16: {
5998 ProductOps,
"vqdmlXl");
5999 Constant *CI = ConstantInt::get(
SizeTy, 0);
6000 Ops[1] =
Builder.CreateExtractElement(Ops[1], CI,
"lane0");
6003 unsigned AccInt = (BuiltinID == NEON::BI__builtin_neon_vqdmlalh_lane_s16 ||
6004 BuiltinID == NEON::BI__builtin_neon_vqdmlalh_laneq_s16)
6005 ? Intrinsic::aarch64_neon_sqadd
6006 : Intrinsic::aarch64_neon_sqsub;
6009 case NEON::BI__builtin_neon_vqdmlals_s32:
6010 case NEON::BI__builtin_neon_vqdmlsls_s32: {
6012 ProductOps.push_back(Ops[1]);
6016 ProductOps,
"vqdmlXl");
6018 unsigned AccumInt = BuiltinID == NEON::BI__builtin_neon_vqdmlals_s32
6019 ? Intrinsic::aarch64_neon_sqadd
6020 : Intrinsic::aarch64_neon_sqsub;
6023 case NEON::BI__builtin_neon_vqdmlals_lane_s32:
6024 case NEON::BI__builtin_neon_vqdmlals_laneq_s32:
6025 case NEON::BI__builtin_neon_vqdmlsls_lane_s32:
6026 case NEON::BI__builtin_neon_vqdmlsls_laneq_s32: {
6030 ProductOps.push_back(Ops[1]);
6031 ProductOps.push_back(Ops[2]);
6034 ProductOps,
"vqdmlXl");
6037 unsigned AccInt = (BuiltinID == NEON::BI__builtin_neon_vqdmlals_lane_s32 ||
6038 BuiltinID == NEON::BI__builtin_neon_vqdmlals_laneq_s32)
6039 ? Intrinsic::aarch64_neon_sqadd
6040 : Intrinsic::aarch64_neon_sqsub;
6053 AArch64SIMDIntrinsicsProvenSorted);
6057 Builtin->BuiltinID, Builtin->LLVMIntrinsic, Builtin->AltLLVMIntrinsic,
6058 Builtin->NameHint, Builtin->TypeModifier, E, Ops,
6065 switch (BuiltinID) {
6066 default:
return nullptr;
6067 case NEON::BI__builtin_neon_vbsl_v:
6068 case NEON::BI__builtin_neon_vbslq_v: {
6069 llvm::Type *BitTy = llvm::VectorType::getInteger(VTy);
6074 Ops[1] =
Builder.CreateAnd(Ops[0], Ops[1],
"vbsl");
6075 Ops[2] =
Builder.CreateAnd(
Builder.CreateNot(Ops[0]), Ops[2],
"vbsl");
6076 Ops[0] =
Builder.CreateOr(Ops[1], Ops[2],
"vbsl");
6079 case NEON::BI__builtin_neon_vfma_lane_v:
6080 case NEON::BI__builtin_neon_vfmaq_lane_v: {
6083 Value *Addend = Ops[0];
6084 Value *Multiplicand = Ops[1];
6085 Value *LaneSource = Ops[2];
6086 Ops[0] = Multiplicand;
6087 Ops[1] = LaneSource;
6091 llvm::Type *SourceTy = BuiltinID == NEON::BI__builtin_neon_vfmaq_lane_v ?
6092 llvm::VectorType::get(VTy->getElementType(), VTy->getNumElements() / 2) :
6094 llvm::Constant *cst = cast<Constant>(Ops[3]);
6095 Value *SV = llvm::ConstantVector::getSplat(VTy->getNumElements(), cst);
6097 Ops[1] =
Builder.CreateShuffleVector(Ops[1], Ops[1], SV,
"lane");
6103 case NEON::BI__builtin_neon_vfma_laneq_v: {
6104 llvm::VectorType *VTy = cast<llvm::VectorType>(Ty);
6106 if (VTy && VTy->getElementType() ==
DoubleTy) {
6112 Ops[2] =
Builder.CreateExtractElement(Ops[2], Ops[3],
"extract");
6114 Value *Result =
Builder.CreateCall(F, {Ops[1], Ops[2], Ops[0]});
6121 llvm::Type *STy = llvm::VectorType::get(VTy->getElementType(),
6122 VTy->getNumElements() * 2);
6124 Value* SV = llvm::ConstantVector::getSplat(VTy->getNumElements(),
6125 cast<ConstantInt>(Ops[3]));
6126 Ops[2] =
Builder.CreateShuffleVector(Ops[2], Ops[2], SV,
"lane");
6128 return Builder.CreateCall(F, {Ops[2], Ops[1], Ops[0]});
6130 case NEON::BI__builtin_neon_vfmaq_laneq_v: {
6137 return Builder.CreateCall(F, {Ops[2], Ops[1], Ops[0]});
6139 case NEON::BI__builtin_neon_vfmas_lane_f32:
6140 case NEON::BI__builtin_neon_vfmas_laneq_f32:
6141 case NEON::BI__builtin_neon_vfmad_lane_f64:
6142 case NEON::BI__builtin_neon_vfmad_laneq_f64: {
6146 Ops[2] =
Builder.CreateExtractElement(Ops[2], Ops[3],
"extract");
6147 return Builder.CreateCall(F, {Ops[1], Ops[2], Ops[0]});
6149 case NEON::BI__builtin_neon_vmull_v:
6151 Int = usgn ? Intrinsic::aarch64_neon_umull : Intrinsic::aarch64_neon_smull;
6152 if (Type.
isPoly()) Int = Intrinsic::aarch64_neon_pmull;
6154 case NEON::BI__builtin_neon_vmax_v:
6155 case NEON::BI__builtin_neon_vmaxq_v:
6157 Int = usgn ? Intrinsic::aarch64_neon_umax : Intrinsic::aarch64_neon_smax;
6158 if (Ty->isFPOrFPVectorTy()) Int = Intrinsic::aarch64_neon_fmax;
6160 case NEON::BI__builtin_neon_vmin_v:
6161 case NEON::BI__builtin_neon_vminq_v:
6163 Int = usgn ? Intrinsic::aarch64_neon_umin : Intrinsic::aarch64_neon_smin;
6164 if (Ty->isFPOrFPVectorTy()) Int = Intrinsic::aarch64_neon_fmin;
6166 case NEON::BI__builtin_neon_vabd_v:
6167 case NEON::BI__builtin_neon_vabdq_v:
6169 Int = usgn ? Intrinsic::aarch64_neon_uabd : Intrinsic::aarch64_neon_sabd;
6170 if (Ty->isFPOrFPVectorTy()) Int = Intrinsic::aarch64_neon_fabd;
6172 case NEON::BI__builtin_neon_vpadal_v:
6173 case NEON::BI__builtin_neon_vpadalq_v: {
6174 unsigned ArgElts = VTy->getNumElements();
6175 llvm::IntegerType *EltTy = cast<IntegerType>(VTy->getElementType());
6176 unsigned BitWidth = EltTy->getBitWidth();
6178 llvm::IntegerType::get(
getLLVMContext(), BitWidth/2), 2*ArgElts);
6180 Int = usgn ? Intrinsic::aarch64_neon_uaddlp : Intrinsic::aarch64_neon_saddlp;
6182 TmpOps.push_back(Ops[1]);
6186 return Builder.CreateAdd(tmp, addend);
6188 case NEON::BI__builtin_neon_vpmin_v:
6189 case NEON::BI__builtin_neon_vpminq_v:
6191 Int = usgn ? Intrinsic::aarch64_neon_uminp : Intrinsic::aarch64_neon_sminp;
6192 if (Ty->isFPOrFPVectorTy()) Int = Intrinsic::aarch64_neon_fminp;
6194 case NEON::BI__builtin_neon_vpmax_v:
6195 case NEON::BI__builtin_neon_vpmaxq_v:
6197 Int = usgn ? Intrinsic::aarch64_neon_umaxp : Intrinsic::aarch64_neon_smaxp;
6198 if (Ty->isFPOrFPVectorTy()) Int = Intrinsic::aarch64_neon_fmaxp;
6200 case NEON::BI__builtin_neon_vminnm_v:
6201 case NEON::BI__builtin_neon_vminnmq_v:
6202 Int = Intrinsic::aarch64_neon_fminnm;
6204 case NEON::BI__builtin_neon_vmaxnm_v:
6205 case NEON::BI__builtin_neon_vmaxnmq_v:
6206 Int = Intrinsic::aarch64_neon_fmaxnm;
6208 case NEON::BI__builtin_neon_vrecpss_f32: {
6213 case NEON::BI__builtin_neon_vrecpsd_f64: {
6218 case NEON::BI__builtin_neon_vqshrun_n_v:
6219 Int = Intrinsic::aarch64_neon_sqshrun;
6221 case NEON::BI__builtin_neon_vqrshrun_n_v:
6222 Int = Intrinsic::aarch64_neon_sqrshrun;
6224 case NEON::BI__builtin_neon_vqshrn_n_v:
6225 Int = usgn ? Intrinsic::aarch64_neon_uqshrn : Intrinsic::aarch64_neon_sqshrn;
6227 case NEON::BI__builtin_neon_vrshrn_n_v:
6228 Int = Intrinsic::aarch64_neon_rshrn;
6230 case NEON::BI__builtin_neon_vqrshrn_n_v:
6231 Int = usgn ? Intrinsic::aarch64_neon_uqrshrn : Intrinsic::aarch64_neon_sqrshrn;
6233 case NEON::BI__builtin_neon_vrnda_v:
6234 case NEON::BI__builtin_neon_vrndaq_v: {
6238 case NEON::BI__builtin_neon_vrndi_v:
6239 case NEON::BI__builtin_neon_vrndiq_v: {
6243 case NEON::BI__builtin_neon_vrndm_v:
6244 case NEON::BI__builtin_neon_vrndmq_v: {
6248 case NEON::BI__builtin_neon_vrndn_v:
6249 case NEON::BI__builtin_neon_vrndnq_v: {
6250 Int = Intrinsic::aarch64_neon_frintn;
6253 case NEON::BI__builtin_neon_vrndp_v:
6254 case NEON::BI__builtin_neon_vrndpq_v: {
6258 case NEON::BI__builtin_neon_vrndx_v:
6259 case NEON::BI__builtin_neon_vrndxq_v: {
6263 case NEON::BI__builtin_neon_vrnd_v:
6264 case NEON::BI__builtin_neon_vrndq_v: {
6268 case NEON::BI__builtin_neon_vceqz_v:
6269 case NEON::BI__builtin_neon_vceqzq_v:
6271 ICmpInst::ICMP_EQ,
"vceqz");
6272 case NEON::BI__builtin_neon_vcgez_v:
6273 case NEON::BI__builtin_neon_vcgezq_v:
6275 ICmpInst::ICMP_SGE,
"vcgez");
6276 case NEON::BI__builtin_neon_vclez_v:
6277 case NEON::BI__builtin_neon_vclezq_v:
6279 ICmpInst::ICMP_SLE,
"vclez");
6280 case NEON::BI__builtin_neon_vcgtz_v:
6281 case NEON::BI__builtin_neon_vcgtzq_v:
6283 ICmpInst::ICMP_SGT,
"vcgtz");
6284 case NEON::BI__builtin_neon_vcltz_v:
6285 case NEON::BI__builtin_neon_vcltzq_v:
6287 ICmpInst::ICMP_SLT,
"vcltz");
6288 case NEON::BI__builtin_neon_vcvt_f64_v:
6289 case NEON::BI__builtin_neon_vcvtq_f64_v:
6292 return usgn ?
Builder.CreateUIToFP(Ops[0], Ty,
"vcvt")
6293 :
Builder.CreateSIToFP(Ops[0], Ty,
"vcvt");
6294 case NEON::BI__builtin_neon_vcvt_f64_f32: {
6296 "unexpected vcvt_f64_f32 builtin");
6300 return Builder.CreateFPExt(Ops[0], Ty,
"vcvt");
6302 case NEON::BI__builtin_neon_vcvt_f32_f64: {
6304 "unexpected vcvt_f32_f64 builtin");
6308 return Builder.CreateFPTrunc(Ops[0], Ty,
"vcvt");
6310 case NEON::BI__builtin_neon_vcvt_s32_v:
6311 case NEON::BI__builtin_neon_vcvt_u32_v:
6312 case NEON::BI__builtin_neon_vcvt_s64_v:
6313 case NEON::BI__builtin_neon_vcvt_u64_v:
6314 case NEON::BI__builtin_neon_vcvtq_s32_v:
6315 case NEON::BI__builtin_neon_vcvtq_u32_v:
6316 case NEON::BI__builtin_neon_vcvtq_s64_v:
6317 case NEON::BI__builtin_neon_vcvtq_u64_v: {
6320 return Builder.CreateFPToUI(Ops[0], Ty);
6321 return Builder.CreateFPToSI(Ops[0], Ty);
6323 case NEON::BI__builtin_neon_vcvta_s32_v:
6324 case NEON::BI__builtin_neon_vcvtaq_s32_v:
6325 case NEON::BI__builtin_neon_vcvta_u32_v:
6326 case NEON::BI__builtin_neon_vcvtaq_u32_v:
6327 case NEON::BI__builtin_neon_vcvta_s64_v:
6328 case NEON::BI__builtin_neon_vcvtaq_s64_v:
6329 case NEON::BI__builtin_neon_vcvta_u64_v:
6330 case NEON::BI__builtin_neon_vcvtaq_u64_v: {
6331 Int = usgn ? Intrinsic::aarch64_neon_fcvtau : Intrinsic::aarch64_neon_fcvtas;
6335 case NEON::BI__builtin_neon_vcvtm_s32_v:
6336 case NEON::BI__builtin_neon_vcvtmq_s32_v:
6337 case NEON::BI__builtin_neon_vcvtm_u32_v:
6338 case NEON::BI__builtin_neon_vcvtmq_u32_v:
6339 case NEON::BI__builtin_neon_vcvtm_s64_v:
6340 case NEON::BI__builtin_neon_vcvtmq_s64_v:
6341 case NEON::BI__builtin_neon_vcvtm_u64_v:
6342 case NEON::BI__builtin_neon_vcvtmq_u64_v: {
6343 Int = usgn ? Intrinsic::aarch64_neon_fcvtmu : Intrinsic::aarch64_neon_fcvtms;
6347 case NEON::BI__builtin_neon_vcvtn_s32_v:
6348 case NEON::BI__builtin_neon_vcvtnq_s32_v:
6349 case NEON::BI__builtin_neon_vcvtn_u32_v:
6350 case NEON::BI__builtin_neon_vcvtnq_u32_v:
6351 case NEON::BI__builtin_neon_vcvtn_s64_v:
6352 case NEON::BI__builtin_neon_vcvtnq_s64_v:
6353 case NEON::BI__builtin_neon_vcvtn_u64_v:
6354 case NEON::BI__builtin_neon_vcvtnq_u64_v: {
6355 Int = usgn ? Intrinsic::aarch64_neon_fcvtnu : Intrinsic::aarch64_neon_fcvtns;
6359 case NEON::BI__builtin_neon_vcvtp_s32_v:
6360 case NEON::BI__builtin_neon_vcvtpq_s32_v:
6361 case NEON::BI__builtin_neon_vcvtp_u32_v:
6362 case NEON::BI__builtin_neon_vcvtpq_u32_v:
6363 case NEON::BI__builtin_neon_vcvtp_s64_v:
6364 case NEON::BI__builtin_neon_vcvtpq_s64_v:
6365 case NEON::BI__builtin_neon_vcvtp_u64_v:
6366 case NEON::BI__builtin_neon_vcvtpq_u64_v: {
6367 Int = usgn ? Intrinsic::aarch64_neon_fcvtpu : Intrinsic::aarch64_neon_fcvtps;
6371 case NEON::BI__builtin_neon_vmulx_v:
6372 case NEON::BI__builtin_neon_vmulxq_v: {
6373 Int = Intrinsic::aarch64_neon_fmulx;
6376 case NEON::BI__builtin_neon_vmul_lane_v:
6377 case NEON::BI__builtin_neon_vmul_laneq_v: {
6380 if (BuiltinID == NEON::BI__builtin_neon_vmul_laneq_v)
6386 Ops[1] =
Builder.CreateExtractElement(Ops[1], Ops[2],
"extract");
6387 Value *Result =
Builder.CreateFMul(Ops[0], Ops[1]);
6390 case NEON::BI__builtin_neon_vnegd_s64:
6392 case NEON::BI__builtin_neon_vpmaxnm_v:
6393 case NEON::BI__builtin_neon_vpmaxnmq_v: {
6394 Int = Intrinsic::aarch64_neon_fmaxnmp;
6397 case NEON::BI__builtin_neon_vpminnm_v:
6398 case NEON::BI__builtin_neon_vpminnmq_v: {
6399 Int = Intrinsic::aarch64_neon_fminnmp;
6402 case NEON::BI__builtin_neon_vsqrt_v:
6403 case NEON::BI__builtin_neon_vsqrtq_v: {
6408 case NEON::BI__builtin_neon_vrbit_v:
6409 case NEON::BI__builtin_neon_vrbitq_v: {
6410 Int = Intrinsic::aarch64_neon_rbit;
6413 case NEON::BI__builtin_neon_vaddv_u8:
6417 case NEON::BI__builtin_neon_vaddv_s8: {
6418 Int = usgn ? Intrinsic::aarch64_neon_uaddv : Intrinsic::aarch64_neon_saddv;
6420 VTy = llvm::VectorType::get(
Int8Ty, 8);
6426 case NEON::BI__builtin_neon_vaddv_u16:
6429 case NEON::BI__builtin_neon_vaddv_s16: {
6430 Int = usgn ? Intrinsic::aarch64_neon_uaddv : Intrinsic::aarch64_neon_saddv;
6432 VTy = llvm::VectorType::get(
Int16Ty, 4);
6438 case NEON::BI__builtin_neon_vaddvq_u8:
6441 case NEON::BI__builtin_neon_vaddvq_s8: {
6442 Int = usgn ? Intrinsic::aarch64_neon_uaddv : Intrinsic::aarch64_neon_saddv;
6444 VTy = llvm::VectorType::get(
Int8Ty, 16);
6450 case NEON::BI__builtin_neon_vaddvq_u16:
6453 case NEON::BI__builtin_neon_vaddvq_s16: {
6454 Int = usgn ? Intrinsic::aarch64_neon_uaddv : Intrinsic::aarch64_neon_saddv;
6456 VTy = llvm::VectorType::get(
Int16Ty, 8);
6462 case NEON::BI__builtin_neon_vmaxv_u8: {
6463 Int = Intrinsic::aarch64_neon_umaxv;
6465 VTy = llvm::VectorType::get(
Int8Ty, 8);
6471 case NEON::BI__builtin_neon_vmaxv_u16: {
6472 Int = Intrinsic::aarch64_neon_umaxv;
6474 VTy = llvm::VectorType::get(
Int16Ty, 4);
6480 case NEON::BI__builtin_neon_vmaxvq_u8: {
6481 Int = Intrinsic::aarch64_neon_umaxv;
6483 VTy = llvm::VectorType::get(
Int8Ty, 16);
6489 case NEON::BI__builtin_neon_vmaxvq_u16: {
6490 Int = Intrinsic::aarch64_neon_umaxv;
6492 VTy = llvm::VectorType::get(
Int16Ty, 8);
6498 case NEON::BI__builtin_neon_vmaxv_s8: {
6499 Int = Intrinsic::aarch64_neon_smaxv;
6501 VTy = llvm::VectorType::get(
Int8Ty, 8);
6507 case NEON::BI__builtin_neon_vmaxv_s16: {
6508 Int = Intrinsic::aarch64_neon_smaxv;
6510 VTy = llvm::VectorType::get(
Int16Ty, 4);
6516 case NEON::BI__builtin_neon_vmaxvq_s8: {
6517 Int = Intrinsic::aarch64_neon_smaxv;
6519 VTy = llvm::VectorType::get(
Int8Ty, 16);
6525 case NEON::BI__builtin_neon_vmaxvq_s16: {
6526 Int = Intrinsic::aarch64_neon_smaxv;
6528 VTy = llvm::VectorType::get(
Int16Ty, 8);
6534 case NEON::BI__builtin_neon_vminv_u8: {
6535 Int = Intrinsic::aarch64_neon_uminv;
6537 VTy = llvm::VectorType::get(
Int8Ty, 8);
6543 case NEON::BI__builtin_neon_vminv_u16: {
6544 Int = Intrinsic::aarch64_neon_uminv;
6546 VTy = llvm::VectorType::get(
Int16Ty, 4);
6552 case NEON::BI__builtin_neon_vminvq_u8: {
6553 Int = Intrinsic::aarch64_neon_uminv;
6555 VTy = llvm::VectorType::get(
Int8Ty, 16);
6561 case NEON::BI__builtin_neon_vminvq_u16: {
6562 Int = Intrinsic::aarch64_neon_uminv;
6564 VTy = llvm::VectorType::get(
Int16Ty, 8);
6570 case NEON::BI__builtin_neon_vminv_s8: {
6571 Int = Intrinsic::aarch64_neon_sminv;
6573 VTy = llvm::VectorType::get(
Int8Ty, 8);
6579 case NEON::BI__builtin_neon_vminv_s16: {
6580 Int = Intrinsic::aarch64_neon_sminv;
6582 VTy = llvm::VectorType::get(
Int16Ty, 4);
6588 case NEON::BI__builtin_neon_vminvq_s8: {
6589 Int = Intrinsic::aarch64_neon_sminv;
6591 VTy = llvm::VectorType::get(
Int8Ty, 16);
6597 case NEON::BI__builtin_neon_vminvq_s16: {
6598 Int = Intrinsic::aarch64_neon_sminv;
6600 VTy = llvm::VectorType::get(
Int16Ty, 8);
6606 case NEON::BI__builtin_neon_vmul_n_f64: {
6609 return Builder.CreateFMul(Ops[0], RHS);
6611 case NEON::BI__builtin_neon_vaddlv_u8: {
6612 Int = Intrinsic::aarch64_neon_uaddlv;
6614 VTy = llvm::VectorType::get(
Int8Ty, 8);
6620 case NEON::BI__builtin_neon_vaddlv_u16: {
6621 Int = Intrinsic::aarch64_neon_uaddlv;
6623 VTy = llvm::VectorType::get(
Int16Ty, 4);
6628 case NEON::BI__builtin_neon_vaddlvq_u8: {
6629 Int = Intrinsic::aarch64_neon_uaddlv;
6631 VTy = llvm::VectorType::get(
Int8Ty, 16);
6637 case NEON::BI__builtin_neon_vaddlvq_u16: {
6638 Int = Intrinsic::aarch64_neon_uaddlv;
6640 VTy = llvm::VectorType::get(
Int16Ty, 8);
6645 case NEON::BI__builtin_neon_vaddlv_s8: {
6646 Int = Intrinsic::aarch64_neon_saddlv;
6648 VTy = llvm::VectorType::get(
Int8Ty, 8);
6654 case NEON::BI__builtin_neon_vaddlv_s16: {
6655 Int = Intrinsic::aarch64_neon_saddlv;
6657 VTy = llvm::VectorType::get(
Int16Ty, 4);
6662 case NEON::BI__builtin_neon_vaddlvq_s8: {
6663 Int = Intrinsic::aarch64_neon_saddlv;
6665 VTy = llvm::VectorType::get(
Int8Ty, 16);
6671 case NEON::BI__builtin_neon_vaddlvq_s16: {
6672 Int = Intrinsic::aarch64_neon_saddlv;
6674 VTy = llvm::VectorType::get(
Int16Ty, 8);
6679 case NEON::BI__builtin_neon_vsri_n_v:
6680 case NEON::BI__builtin_neon_vsriq_n_v: {
6681 Int = Intrinsic::aarch64_neon_vsri;
6685 case NEON::BI__builtin_neon_vsli_n_v:
6686 case NEON::BI__builtin_neon_vsliq_n_v: {
6687 Int = Intrinsic::aarch64_neon_vsli;
6691 case NEON::BI__builtin_neon_vsra_n_v:
6692 case NEON::BI__builtin_neon_vsraq_n_v:
6695 return Builder.CreateAdd(Ops[0], Ops[1]);
6696 case NEON::BI__builtin_neon_vrsra_n_v:
6697 case NEON::BI__builtin_neon_vrsraq_n_v: {
6698 Int = usgn ? Intrinsic::aarch64_neon_urshl : Intrinsic::aarch64_neon_srshl;
6700 TmpOps.push_back(Ops[1]);
6701 TmpOps.push_back(Ops[2]);
6705 return Builder.CreateAdd(Ops[0], tmp);
6709 case NEON::BI__builtin_neon_vld1_x2_v:
6710 case NEON::BI__builtin_neon_vld1q_x2_v:
6711 case NEON::BI__builtin_neon_vld1_x3_v:
6712 case NEON::BI__builtin_neon_vld1q_x3_v:
6713 case NEON::BI__builtin_neon_vld1_x4_v:
6714 case NEON::BI__builtin_neon_vld1q_x4_v: {
6715 llvm::Type *PTy = llvm::PointerType::getUnqual(VTy->getVectorElementType());
6719 switch (BuiltinID) {
6720 case NEON::BI__builtin_neon_vld1_x2_v:
6721 case NEON::BI__builtin_neon_vld1q_x2_v:
6722 Int = Intrinsic::aarch64_neon_ld1x2;
6724 case NEON::BI__builtin_neon_vld1_x3_v:
6725 case NEON::BI__builtin_neon_vld1q_x3_v:
6726 Int = Intrinsic::aarch64_neon_ld1x3;
6728 case NEON::BI__builtin_neon_vld1_x4_v:
6729 case NEON::BI__builtin_neon_vld1q_x4_v:
6730 Int = Intrinsic::aarch64_neon_ld1x4;
6734 Ops[1] =
Builder.CreateCall(F, Ops[1],
"vld1xN");
6735 Ty = llvm::PointerType::getUnqual(Ops[1]->getType());
6739 case NEON::BI__builtin_neon_vst1_x2_v:
6740 case NEON::BI__builtin_neon_vst1q_x2_v:
6741 case NEON::BI__builtin_neon_vst1_x3_v:
6742 case NEON::BI__builtin_neon_vst1q_x3_v:
6743 case NEON::BI__builtin_neon_vst1_x4_v:
6744 case NEON::BI__builtin_neon_vst1q_x4_v: {
6745 llvm::Type *PTy = llvm::PointerType::getUnqual(VTy->getVectorElementType());
6748 switch (BuiltinID) {
6749 case NEON::BI__builtin_neon_vst1_x2_v:
6750 case NEON::BI__builtin_neon_vst1q_x2_v:
6751 Int = Intrinsic::aarch64_neon_st1x2;
6753 case NEON::BI__builtin_neon_vst1_x3_v:
6754 case NEON::BI__builtin_neon_vst1q_x3_v:
6755 Int = Intrinsic::aarch64_neon_st1x3;
6757 case NEON::BI__builtin_neon_vst1_x4_v:
6758 case NEON::BI__builtin_neon_vst1q_x4_v:
6759 Int = Intrinsic::aarch64_neon_st1x4;
6762 std::rotate(Ops.begin(), Ops.begin() + 1, Ops.end());
6765 case NEON::BI__builtin_neon_vld1_v:
6766 case NEON::BI__builtin_neon_vld1q_v: {
6769 BuiltinID == NEON::BI__builtin_neon_vld1_v ? 8 : 16);
6772 case NEON::BI__builtin_neon_vst1_v:
6773 case NEON::BI__builtin_neon_vst1q_v:
6777 case NEON::BI__builtin_neon_vld1_lane_v:
6778 case NEON::BI__builtin_neon_vld1q_lane_v: {
6780 Ty = llvm::PointerType::getUnqual(VTy->getElementType());
6783 BuiltinID == NEON::BI__builtin_neon_vld1_lane_v ? 8 : 16);
6786 return Builder.CreateInsertElement(Ops[1], Ops[0], Ops[2],
"vld1_lane");
6788 case NEON::BI__builtin_neon_vld1_dup_v:
6789 case NEON::BI__builtin_neon_vld1q_dup_v: {
6790 Value *V = UndefValue::get(Ty);
6791 Ty = llvm::PointerType::getUnqual(VTy->getElementType());
6794 BuiltinID == NEON::BI__builtin_neon_vld1_dup_v ? 8 : 16);
6797 llvm::Constant *CI = ConstantInt::get(
Int32Ty, 0);
6798 Ops[0] =
Builder.CreateInsertElement(V, Ops[0], CI);
6801 case NEON::BI__builtin_neon_vst1_lane_v:
6802 case NEON::BI__builtin_neon_vst1q_lane_v:
6804 Ops[1] =
Builder.CreateExtractElement(Ops[1], Ops[2]);
6805 Ty = llvm::PointerType::getUnqual(Ops[1]->getType());
6808 case NEON::BI__builtin_neon_vld2_v:
6809 case NEON::BI__builtin_neon_vld2q_v: {
6810 llvm::Type *PTy = llvm::PointerType::getUnqual(VTy);
6814 Ops[1] =
Builder.CreateCall(F, Ops[1],
"vld2");
6816 llvm::PointerType::getUnqual(Ops[1]->getType()));
6819 case NEON::BI__builtin_neon_vld3_v:
6820 case NEON::BI__builtin_neon_vld3q_v: {
6821 llvm::Type *PTy = llvm::PointerType::getUnqual(VTy);
6825 Ops[1] =
Builder.CreateCall(F, Ops[1],
"vld3");
6827 llvm::PointerType::getUnqual(Ops[1]->getType()));
6830 case NEON::BI__builtin_neon_vld4_v:
6831 case NEON::BI__builtin_neon_vld4q_v: {
6832 llvm::Type *PTy = llvm::PointerType::getUnqual(VTy);
6836 Ops[1] =
Builder.CreateCall(F, Ops[1],
"vld4");
6838 llvm::PointerType::getUnqual(Ops[1]->getType()));
6841 case NEON::BI__builtin_neon_vld2_dup_v:
6842 case NEON::BI__builtin_neon_vld2q_dup_v: {
6844 llvm::PointerType::getUnqual(VTy->getElementType());
6848 Ops[1] =
Builder.CreateCall(F, Ops[1],
"vld2");
6850 llvm::PointerType::getUnqual(Ops[1]->getType()));
6853 case NEON::BI__builtin_neon_vld3_dup_v:
6854 case NEON::BI__builtin_neon_vld3q_dup_v: {
6856 llvm::PointerType::getUnqual(VTy->getElementType());
6860 Ops[1] =
Builder.CreateCall(F, Ops[1],
"vld3");
6862 llvm::PointerType::getUnqual(Ops[1]->getType()));
6865 case NEON::BI__builtin_neon_vld4_dup_v:
6866 case NEON::BI__builtin_neon_vld4q_dup_v: {
6868 llvm::PointerType::getUnqual(VTy->getElementType());
6872 Ops[1] =
Builder.CreateCall(F, Ops[1],
"vld4");
6874 llvm::PointerType::getUnqual(Ops[1]->getType()));
6877 case NEON::BI__builtin_neon_vld2_lane_v:
6878 case NEON::BI__builtin_neon_vld2q_lane_v: {
6879 llvm::Type *Tys[2] = { VTy, Ops[1]->getType() };
6881 Ops.push_back(Ops[1]);
6882 Ops.erase(Ops.begin()+1);
6886 Ops[1] =
Builder.CreateCall(F, makeArrayRef(Ops).slice(1),
"vld2_lane");
6887 Ty = llvm::PointerType::getUnqual(Ops[1]->getType());
6891 case NEON::BI__builtin_neon_vld3_lane_v:
6892 case NEON::BI__builtin_neon_vld3q_lane_v: {
6893 llvm::Type *Tys[2] = { VTy, Ops[1]->getType() };
6895 Ops.push_back(Ops[1]);
6896 Ops.erase(Ops.begin()+1);
6901 Ops[1] =
Builder.CreateCall(F, makeArrayRef(Ops).slice(1),
"vld3_lane");
6902 Ty = llvm::PointerType::getUnqual(Ops[1]->getType());
6906 case NEON::BI__builtin_neon_vld4_lane_v:
6907 case NEON::BI__builtin_neon_vld4q_lane_v: {
6908 llvm::Type *Tys[2] = { VTy, Ops[1]->getType() };
6910 Ops.push_back(Ops[1]);
6911 Ops.erase(Ops.begin()+1);
6917 Ops[1] =
Builder.CreateCall(F, makeArrayRef(Ops).slice(1),
"vld4_lane");
6918 Ty = llvm::PointerType::getUnqual(Ops[1]->getType());
6922 case NEON::BI__builtin_neon_vst2_v:
6923 case NEON::BI__builtin_neon_vst2q_v: {
6924 Ops.push_back(Ops[0]);
6925 Ops.erase(Ops.begin());
6926 llvm::Type *Tys[2] = { VTy, Ops[2]->getType() };
6930 case NEON::BI__builtin_neon_vst2_lane_v:
6931 case NEON::BI__builtin_neon_vst2q_lane_v: {
6932 Ops.push_back(Ops[0]);
6933 Ops.erase(Ops.begin());
6935 llvm::Type *Tys[2] = { VTy, Ops[3]->getType() };
6939 case NEON::BI__builtin_neon_vst3_v:
6940 case NEON::BI__builtin_neon_vst3q_v: {
6941 Ops.push_back(Ops[0]);
6942 Ops.erase(Ops.begin());
6943 llvm::Type *Tys[2] = { VTy, Ops[3]->getType() };
6947 case NEON::BI__builtin_neon_vst3_lane_v:
6948 case NEON::BI__builtin_neon_vst3q_lane_v: {
6949 Ops.push_back(Ops[0]);
6950 Ops.erase(Ops.begin());
6952 llvm::Type *Tys[2] = { VTy, Ops[4]->getType() };
6956 case NEON::BI__builtin_neon_vst4_v:
6957 case NEON::BI__builtin_neon_vst4q_v: {
6958 Ops.push_back(Ops[0]);
6959 Ops.erase(Ops.begin());
6960 llvm::Type *Tys[2] = { VTy, Ops[4]->getType() };
6964 case NEON::BI__builtin_neon_vst4_lane_v:
6965 case NEON::BI__builtin_neon_vst4q_lane_v: {
6966 Ops.push_back(Ops[0]);
6967 Ops.erase(Ops.begin());
6969 llvm::Type *Tys[2] = { VTy, Ops[5]->getType() };
6973 case NEON::BI__builtin_neon_vtrn_v:
6974 case NEON::BI__builtin_neon_vtrnq_v: {
6978 Value *SV =
nullptr;
6980 for (
unsigned vi = 0; vi != 2; ++vi) {
6982 for (
unsigned i = 0, e = VTy->getNumElements(); i != e; i += 2) {
6983 Indices.push_back(i+vi);
6984 Indices.push_back(i+e+vi);
6986 Value *Addr =
Builder.CreateConstInBoundsGEP1_32(Ty, Ops[0], vi);
6987 SV =
Builder.CreateShuffleVector(Ops[1], Ops[2], Indices,
"vtrn");
6992 case NEON::BI__builtin_neon_vuzp_v:
6993 case NEON::BI__builtin_neon_vuzpq_v: {
6997 Value *SV =
nullptr;
6999 for (
unsigned vi = 0; vi != 2; ++vi) {
7001 for (
unsigned i = 0, e = VTy->getNumElements(); i != e; ++i)
7002 Indices.push_back(2*i+vi);
7004 Value *Addr =
Builder.CreateConstInBoundsGEP1_32(Ty, Ops[0], vi);
7005 SV =
Builder.CreateShuffleVector(Ops[1], Ops[2], Indices,
"vuzp");
7010 case NEON::BI__builtin_neon_vzip_v:
7011 case NEON::BI__builtin_neon_vzipq_v: {
7015 Value *SV =
nullptr;
7017 for (
unsigned vi = 0; vi != 2; ++vi) {
7019 for (
unsigned i = 0, e = VTy->getNumElements(); i != e; i += 2) {
7020 Indices.push_back((i + vi*e) >> 1);
7021 Indices.push_back(((i + vi*e) >> 1)+e);
7023 Value *Addr =
Builder.CreateConstInBoundsGEP1_32(Ty, Ops[0], vi);
7024 SV =
Builder.CreateShuffleVector(Ops[1], Ops[2], Indices,
"vzip");
7029 case NEON::BI__builtin_neon_vqtbl1q_v: {
7033 case NEON::BI__builtin_neon_vqtbl2q_v: {
7037 case NEON::BI__builtin_neon_vqtbl3q_v: {
7041 case NEON::BI__builtin_neon_vqtbl4q_v: {
7045 case NEON::BI__builtin_neon_vqtbx1q_v: {
7049 case NEON::BI__builtin_neon_vqtbx2q_v: {
7053 case NEON::BI__builtin_neon_vqtbx3q_v: {
7057 case NEON::BI__builtin_neon_vqtbx4q_v: {
7061 case NEON::BI__builtin_neon_vsqadd_v:
7062 case NEON::BI__builtin_neon_vsqaddq_v: {
7063 Int = Intrinsic::aarch64_neon_usqadd;
7066 case NEON::BI__builtin_neon_vuqadd_v:
7067 case NEON::BI__builtin_neon_vuqaddq_v: {
7068 Int = Intrinsic::aarch64_neon_suqadd;
7076 assert((Ops.size() & (Ops.size() - 1)) == 0 &&
7077 "Not a power-of-two sized vector!");
7078 bool AllConstants =
true;
7079 for (
unsigned i = 0, e = Ops.size(); i != e && AllConstants; ++i)
7080 AllConstants &= isa<Constant>(Ops[i]);
7085 for (
unsigned i = 0, e = Ops.size(); i != e; ++i)
7086 CstOps.push_back(cast<Constant>(Ops[i]));
7087 return llvm::ConstantVector::get(CstOps);
7092 llvm::UndefValue::get(llvm::VectorType::get(Ops[0]->getType(), Ops.size()));
7094 for (
unsigned i = 0, e = Ops.size(); i != e; ++i)
7095 Result =
Builder.CreateInsertElement(Result, Ops[i],
Builder.getInt32(i));
7104 llvm::VectorType *MaskTy = llvm::VectorType::get(CGF.
Builder.getInt1Ty(),
7105 cast<IntegerType>(Mask->getType())->getBitWidth());
7111 uint32_t Indices[4];
7112 for (
unsigned i = 0; i != NumElts; ++i)
7114 MaskVec = CGF.
Builder.CreateShuffleVector(MaskVec, MaskVec,
7115 makeArrayRef(Indices, NumElts),
7126 llvm::PointerType::getUnqual(Ops[1]->getType()));
7129 if (
const auto *C = dyn_cast<Constant>(Ops[2]))
7130 if (C->isAllOnesValue())
7134 Ops[1]->getType()->getVectorNumElements());
7136 return CGF.
Builder.CreateMaskedStore(Ops[1], Ops[0], Align, MaskVec);
7143 llvm::PointerType::getUnqual(Ops[1]->getType()));
7146 if (
const auto *C = dyn_cast<Constant>(Ops[2]))
7147 if (C->isAllOnesValue())
7151 Ops[1]->getType()->getVectorNumElements());
7153 return CGF.
Builder.CreateMaskedLoad(Ops[0], Align, MaskVec, Ops[1]);
7159 unsigned SrcSizeInBits,
7165 unsigned NumDstElts = DstTy->getVectorNumElements();
7166 unsigned NumSrcElts = SrcSizeInBits / DstTy->getScalarSizeInBits();
7169 for (
unsigned i = 0; i != NumDstElts; i += NumSrcElts)
7170 for (
unsigned j = 0; j != NumSrcElts; ++j)
7173 return CGF.
Builder.CreateShuffleVector(Ops[0], Ops[0], Mask,
"subvecbcst");
7177 Value *Mask, Value *Op0, Value *Op1) {
7180 if (
const auto *C = dyn_cast<Constant>(Mask))
7181 if (C->isAllOnesValue())
7184 Mask =
getMaskVecValue(CGF, Mask, Op0->getType()->getVectorNumElements());
7186 return CGF.
Builder.CreateSelect(Mask, Op0, Op1);
7191 unsigned NumElts = Ops[0]->getType()->getVectorNumElements();
7195 Cmp = Constant::getNullValue(
7196 llvm::VectorType::get(CGF.
Builder.getInt1Ty(), NumElts));
7197 }
else if (CC == 7) {
7198 Cmp = Constant::getAllOnesValue(
7199 llvm::VectorType::get(CGF.
Builder.getInt1Ty(), NumElts));
7201 ICmpInst::Predicate Pred;
7203 default: llvm_unreachable(
"Unknown condition code");
7204 case 0: Pred = ICmpInst::ICMP_EQ;
break;
7205 case 1: Pred = Signed ? ICmpInst::ICMP_SLT : ICmpInst::ICMP_ULT;
break;
7206 case 2: Pred = Signed ? ICmpInst::ICMP_SLE : ICmpInst::ICMP_ULE;
break;
7207 case 4: Pred = ICmpInst::ICMP_NE;
break;
7208 case 5: Pred = Signed ? ICmpInst::ICMP_SGE : ICmpInst::ICMP_UGE;
break;
7209 case 6: Pred = Signed ? ICmpInst::ICMP_SGT : ICmpInst::ICMP_UGT;
break;
7211 Cmp = CGF.
Builder.CreateICmp(Pred, Ops[0], Ops[1]);
7214 const auto *C = dyn_cast<Constant>(Ops.back());
7215 if (!C || !C->isAllOnesValue())
7219 uint32_t Indices[8];
7220 for (
unsigned i = 0; i != NumElts; ++i)
7222 for (
unsigned i = NumElts; i != 8; ++i)
7223 Indices[i] = i % NumElts + NumElts;
7224 Cmp = CGF.
Builder.CreateShuffleVector(
7225 Cmp, llvm::Constant::getNullValue(Cmp->getType()), Indices);
7234 Value *Cmp = CGF.
Builder.CreateICmp(Pred, Ops[0], Ops[1]);
7235 Value *Res = CGF.
Builder.CreateSelect(Cmp, Ops[0], Ops[1]);
7237 if (Ops.size() == 2)
7240 assert(Ops.size() == 4);
7246 unsigned NumberOfElements = DstTy->getVectorNumElements();
7248 return CGF.
Builder.CreateSExt(Mask, DstTy,
"vpmovm2");
7256 unsigned ICEArguments = 0;
7261 for (
unsigned i = 0, e = E->
getNumArgs(); i != e; i++) {
7263 if ((ICEArguments & (1 << i)) == 0) {
7270 llvm::APSInt Result;
7272 assert(IsConst &&
"Constant arg isn't actually constant?"); (void)IsConst;
7282 auto getCmpIntrinsicCall = [
this, &Ops](
Intrinsic::ID ID,
unsigned Imm) {
7283 Ops.push_back(llvm::ConstantInt::get(
Int8Ty, Imm));
7285 return Builder.CreateCall(F, Ops);
7293 auto getVectorFCmpIR = [
this, &Ops](CmpInst::Predicate Pred) {
7294 Value *Cmp =
Builder.CreateFCmp(Pred, Ops[0], Ops[1]);
7295 llvm::VectorType *FPVecTy = cast<llvm::VectorType>(Ops[0]->getType());
7296 llvm::VectorType *IntVecTy = llvm::VectorType::getInteger(FPVecTy);
7297 Value *Sext =
Builder.CreateSExt(Cmp, IntVecTy);
7301 switch (BuiltinID) {
7302 default:
return nullptr;
7303 case X86::BI__builtin_cpu_supports: {
7305 StringRef FeatureStr = cast<StringLiteral>(FeatureExpr)->getString();
7345 X86Features Feature =
7346 StringSwitch<X86Features>(FeatureStr)
7347 .Case(
"cmov", X86Features::CMOV)
7348 .Case(
"mmx", X86Features::MMX)
7349 .Case(
"popcnt", X86Features::POPCNT)
7350 .Case(
"sse", X86Features::SSE)
7351 .Case(
"sse2", X86Features::SSE2)
7352 .Case(
"sse3", X86Features::SSE3)
7353 .Case(
"ssse3", X86Features::SSSE3)
7354 .Case(
"sse4.1", X86Features::SSE4_1)
7355 .Case(
"sse4.2", X86Features::SSE4_2)
7356 .Case(
"avx", X86Features::AVX)
7357 .Case(
"avx2", X86Features::AVX2)
7358 .Case(
"sse4a", X86Features::SSE4_A)
7359 .Case(
"fma4", X86Features::FMA4)
7360 .Case(
"xop", X86Features::XOP)
7361 .Case(
"fma", X86Features::FMA)
7362 .Case(
"avx512f", X86Features::AVX512F)
7363 .Case(
"bmi", X86Features::BMI)
7364 .Case(
"bmi2", X86Features::BMI2)
7365 .Case(
"aes", X86Features::AES)
7366 .Case(
"pclmul", X86Features::PCLMUL)
7367 .Case(
"avx512vl", X86Features::AVX512VL)
7368 .Case(
"avx512bw", X86Features::AVX512BW)
7369 .Case(
"avx512dq", X86Features::AVX512DQ)
7370 .Case(
"avx512cd", X86Features::AVX512CD)
7371 .Case(
"avx512er", X86Features::AVX512ER)
7372 .Case(
"avx512pf", X86Features::AVX512PF)
7373 .Case(
"avx512vbmi", X86Features::AVX512VBMI)
7374 .Case(
"avx512ifma", X86Features::AVX512IFMA)
7375 .Case(
"avx512vpopcntdq", X86Features::AVX512VPOPCNTDQ)
7376 .Default(X86Features::MAX);
7377 assert(Feature != X86Features::MAX &&
"Invalid feature!");
7386 llvm::ArrayType::get(
Int32Ty, 1));
7398 Value *CpuFeatures =
Builder.CreateGEP(STy, CpuModel, Idxs);
7403 Value *Bitset =
Builder.CreateAnd(
7404 Features, llvm::ConstantInt::get(
Int32Ty, 1ULL << Feature));
7405 return Builder.CreateICmpNE(Bitset, llvm::ConstantInt::get(
Int32Ty, 0));
7407 case X86::BI_mm_prefetch: {
7408 Value *Address = Ops[0];
7409 Value *RW = ConstantInt::get(
Int32Ty, 0);
7410 Value *Locality = Ops[1];
7411 Value *Data = ConstantInt::get(
Int32Ty, 1);
7413 return Builder.CreateCall(F, {Address, RW, Locality, Data});
7415 case X86::BI_mm_clflush: {
7419 case X86::BI_mm_lfence: {
7422 case X86::BI_mm_mfence: {
7425 case X86::BI_mm_sfence: {
7428 case X86::BI_mm_pause: {
7431 case X86::BI__rdtsc: {
7434 case X86::BI__builtin_ia32_undef128:
7435 case X86::BI__builtin_ia32_undef256:
7436 case X86::BI__builtin_ia32_undef512:
7443 case X86::BI__builtin_ia32_vec_init_v8qi:
7444 case X86::BI__builtin_ia32_vec_init_v4hi:
7445 case X86::BI__builtin_ia32_vec_init_v2si:
7448 case X86::BI__builtin_ia32_vec_ext_v2si:
7449 return Builder.CreateExtractElement(Ops[0],
7450 llvm::ConstantInt::get(Ops[1]->getType(), 0));
7451 case X86::BI_mm_setcsr:
7452 case X86::BI__builtin_ia32_ldmxcsr: {
7458 case X86::BI_mm_getcsr:
7459 case X86::BI__builtin_ia32_stmxcsr: {
7465 case X86::BI__builtin_ia32_xsave:
7466 case X86::BI__builtin_ia32_xsave64:
7467 case X86::BI__builtin_ia32_xrstor:
7468 case X86::BI__builtin_ia32_xrstor64:
7469 case X86::BI__builtin_ia32_xsaveopt:
7470 case X86::BI__builtin_ia32_xsaveopt64:
7471 case X86::BI__builtin_ia32_xrstors:
7472 case X86::BI__builtin_ia32_xrstors64:
7473 case X86::BI__builtin_ia32_xsavec:
7474 case X86::BI__builtin_ia32_xsavec64:
7475 case X86::BI__builtin_ia32_xsaves:
7476 case X86::BI__builtin_ia32_xsaves64: {
7478 #define INTRINSIC_X86_XSAVE_ID(NAME) \
7479 case X86::BI__builtin_ia32_##NAME: \
7480 ID = Intrinsic::x86_##NAME; \
7482 switch (BuiltinID) {
7483 default: llvm_unreachable(
"Unsupported intrinsic!");
7497 #undef INTRINSIC_X86_XSAVE_ID
7498 Value *Mhi =
Builder.CreateTrunc(
7505 case X86::BI__builtin_ia32_storedqudi128_mask:
7506 case X86::BI__builtin_ia32_storedqusi128_mask:
7507 case X86::BI__builtin_ia32_storedquhi128_mask:
7508 case X86::BI__builtin_ia32_storedquqi128_mask:
7509 case X86::BI__builtin_ia32_storeupd128_mask:
7510 case X86::BI__builtin_ia32_storeups128_mask:
7511 case X86::BI__builtin_ia32_storedqudi256_mask:
7512 case X86::BI__builtin_ia32_storedqusi256_mask:
7513 case X86::BI__builtin_ia32_storedquhi256_mask:
7514 case X86::BI__builtin_ia32_storedquqi256_mask:
7515 case X86::BI__builtin_ia32_storeupd256_mask:
7516 case X86::BI__builtin_ia32_storeups256_mask:
7517 case X86::BI__builtin_ia32_storedqudi512_mask:
7518 case X86::BI__builtin_ia32_storedqusi512_mask:
7519 case X86::BI__builtin_ia32_storedquhi512_mask:
7520 case X86::BI__builtin_ia32_storedquqi512_mask:
7521 case X86::BI__builtin_ia32_storeupd512_mask:
7522 case X86::BI__builtin_ia32_storeups512_mask:
7525 case X86::BI__builtin_ia32_storess128_mask:
7526 case X86::BI__builtin_ia32_storesd128_mask: {
7529 case X86::BI__builtin_ia32_vpopcntd_512:
7530 case X86::BI__builtin_ia32_vpopcntq_512: {
7533 return Builder.CreateCall(F, Ops);
7535 case X86::BI__builtin_ia32_cvtmask2b128:
7536 case X86::BI__builtin_ia32_cvtmask2b256:
7537 case X86::BI__builtin_ia32_cvtmask2b512:
7538 case X86::BI__builtin_ia32_cvtmask2w128:
7539 case X86::BI__builtin_ia32_cvtmask2w256:
7540 case X86::BI__builtin_ia32_cvtmask2w512:
7541 case X86::BI__builtin_ia32_cvtmask2d128:
7542 case X86::BI__builtin_ia32_cvtmask2d256:
7543 case X86::BI__builtin_ia32_cvtmask2d512:
7544 case X86::BI__builtin_ia32_cvtmask2q128:
7545 case X86::BI__builtin_ia32_cvtmask2q256:
7546 case X86::BI__builtin_ia32_cvtmask2q512:
7549 case X86::BI__builtin_ia32_movdqa32store128_mask:
7550 case X86::BI__builtin_ia32_movdqa64store128_mask:
7551 case X86::BI__builtin_ia32_storeaps128_mask:
7552 case X86::BI__builtin_ia32_storeapd128_mask:
7553 case X86::BI__builtin_ia32_movdqa32store256_mask:
7554 case X86::BI__builtin_ia32_movdqa64store256_mask:
7555 case X86::BI__builtin_ia32_storeaps256_mask:
7556 case X86::BI__builtin_ia32_storeapd256_mask:
7557 case X86::BI__builtin_ia32_movdqa32store512_mask:
7558 case X86::BI__builtin_ia32_movdqa64store512_mask:
7559 case X86::BI__builtin_ia32_storeaps512_mask:
7560 case X86::BI__builtin_ia32_storeapd512_mask: {
7565 case X86::BI__builtin_ia32_loadups128_mask:
7566 case X86::BI__builtin_ia32_loadups256_mask:
7567 case X86::BI__builtin_ia32_loadups512_mask:
7568 case X86::BI__builtin_ia32_loadupd128_mask:
7569 case X86::BI__builtin_ia32_loadupd256_mask:
7570 case X86::BI__builtin_ia32_loadupd512_mask:
7571 case X86::BI__builtin_ia32_loaddquqi128_mask:
7572 case X86::BI__builtin_ia32_loaddquqi256_mask:
7573 case X86::BI__builtin_ia32_loaddquqi512_mask:
7574 case X86::BI__builtin_ia32_loaddquhi128_mask:
7575 case X86::BI__builtin_ia32_loaddquhi256_mask:
7576 case X86::BI__builtin_ia32_loaddquhi512_mask:
7577 case X86::BI__builtin_ia32_loaddqusi128_mask:
7578 case X86::BI__builtin_ia32_loaddqusi256_mask:
7579 case X86::BI__builtin_ia32_loaddqusi512_mask:
7580 case X86::BI__builtin_ia32_loaddqudi128_mask:
7581 case X86::BI__builtin_ia32_loaddqudi256_mask:
7582 case X86::BI__builtin_ia32_loaddqudi512_mask:
7585 case X86::BI__builtin_ia32_loadss128_mask:
7586 case X86::BI__builtin_ia32_loadsd128_mask:
7589 case X86::BI__builtin_ia32_loadaps128_mask:
7590 case X86::BI__builtin_ia32_loadaps256_mask:
7591 case X86::BI__builtin_ia32_loadaps512_mask:
7592 case X86::BI__builtin_ia32_loadapd128_mask:
7593 case X86::BI__builtin_ia32_loadapd256_mask:
7594 case X86::BI__builtin_ia32_loadapd512_mask:
7595 case X86::BI__builtin_ia32_movdqa32load128_mask:
7596 case X86::BI__builtin_ia32_movdqa32load256_mask:
7597 case X86::BI__builtin_ia32_movdqa32load512_mask:
7598 case X86::BI__builtin_ia32_movdqa64load128_mask:
7599 case X86::BI__builtin_ia32_movdqa64load256_mask:
7600 case X86::BI__builtin_ia32_movdqa64load512_mask: {
7606 case X86::BI__builtin_ia32_vbroadcastf128_pd256:
7607 case X86::BI__builtin_ia32_vbroadcastf128_ps256: {
7612 case X86::BI__builtin_ia32_storehps:
7613 case X86::BI__builtin_ia32_storelps: {
7621 unsigned Index = BuiltinID == X86::BI__builtin_ia32_storelps ? 0 : 1;
7623 Ops[1] =
Builder.CreateExtractElement(Ops[1], Idx,
"extract");
7629 case X86::BI__builtin_ia32_palignr128:
7630 case X86::BI__builtin_ia32_palignr256:
7631 case X86::BI__builtin_ia32_palignr512_mask: {
7632 unsigned ShiftVal = cast<llvm::ConstantInt>(Ops[2])->getZExtValue();
7634 unsigned NumElts = Ops[0]->getType()->getVectorNumElements();
7635 assert(NumElts % 16 == 0);
7644 if (ShiftVal > 16) {
7647 Ops[0] = llvm::Constant::getNullValue(Ops[0]->getType());
7650 uint32_t Indices[64];
7652 for (
unsigned l = 0; l != NumElts; l += 16) {
7653 for (
unsigned i = 0; i != 16; ++i) {
7654 unsigned Idx = ShiftVal + i;
7656 Idx += NumElts - 16;
7657 Indices[l + i] = Idx + l;
7661 Value *Align =
Builder.CreateShuffleVector(Ops[1], Ops[0],
7662 makeArrayRef(Indices, NumElts),
7666 if (Ops.size() == 3)
7672 case X86::BI__builtin_ia32_movnti:
7673 case X86::BI__builtin_ia32_movnti64:
7674 case X86::BI__builtin_ia32_movntsd:
7675 case X86::BI__builtin_ia32_movntss: {
7676 llvm::MDNode *
Node = llvm::MDNode::get(
7679 Value *Ptr = Ops[0];
7680 Value *Src = Ops[1];
7683 if (BuiltinID == X86::BI__builtin_ia32_movntsd ||
7684 BuiltinID == X86::BI__builtin_ia32_movntss)
7685 Src =
Builder.CreateExtractElement(Src, (uint64_t)0,
"extract");
7689 Ptr, llvm::PointerType::getUnqual(Src->getType()),
"cast");
7694 SI->setAlignment(1);
7698 case X86::BI__builtin_ia32_selectb_128:
7699 case X86::BI__builtin_ia32_selectb_256:
7700 case X86::BI__builtin_ia32_selectb_512:
7701 case X86::BI__builtin_ia32_selectw_128:
7702 case X86::BI__builtin_ia32_selectw_256:
7703 case X86::BI__builtin_ia32_selectw_512:
7704 case X86::BI__builtin_ia32_selectd_128:
7705 case X86::BI__builtin_ia32_selectd_256:
7706 case X86::BI__builtin_ia32_selectd_512:
7707 case X86::BI__builtin_ia32_selectq_128:
7708 case X86::BI__builtin_ia32_selectq_256:
7709 case X86::BI__builtin_ia32_selectq_512:
7710 case X86::BI__builtin_ia32_selectps_128:
7711 case X86::BI__builtin_ia32_selectps_256:
7712 case X86::BI__builtin_ia32_selectps_512:
7713 case X86::BI__builtin_ia32_selectpd_128:
7714 case X86::BI__builtin_ia32_selectpd_256:
7715 case X86::BI__builtin_ia32_selectpd_512:
7717 case X86::BI__builtin_ia32_pcmpeqb128_mask:
7718 case X86::BI__builtin_ia32_pcmpeqb256_mask:
7719 case X86::BI__builtin_ia32_pcmpeqb512_mask:
7720 case X86::BI__builtin_ia32_pcmpeqw128_mask:
7721 case X86::BI__builtin_ia32_pcmpeqw256_mask:
7722 case X86::BI__builtin_ia32_pcmpeqw512_mask:
7723 case X86::BI__builtin_ia32_pcmpeqd128_mask:
7724 case X86::BI__builtin_ia32_pcmpeqd256_mask:
7725 case X86::BI__builtin_ia32_pcmpeqd512_mask:
7726 case X86::BI__builtin_ia32_pcmpeqq128_mask:
7727 case X86::BI__builtin_ia32_pcmpeqq256_mask:
7728 case X86::BI__builtin_ia32_pcmpeqq512_mask:
7730 case X86::BI__builtin_ia32_pcmpgtb128_mask:
7731 case X86::BI__builtin_ia32_pcmpgtb256_mask:
7732 case X86::BI__builtin_ia32_pcmpgtb512_mask:
7733 case X86::BI__builtin_ia32_pcmpgtw128_mask:
7734 case X86::BI__builtin_ia32_pcmpgtw256_mask:
7735 case X86::BI__builtin_ia32_pcmpgtw512_mask:
7736 case X86::BI__builtin_ia32_pcmpgtd128_mask:
7737 case X86::BI__builtin_ia32_pcmpgtd256_mask:
7738 case X86::BI__builtin_ia32_pcmpgtd512_mask:
7739 case X86::BI__builtin_ia32_pcmpgtq128_mask:
7740 case X86::BI__builtin_ia32_pcmpgtq256_mask:
7741 case X86::BI__builtin_ia32_pcmpgtq512_mask:
7743 case X86::BI__builtin_ia32_cmpb128_mask:
7744 case X86::BI__builtin_ia32_cmpb256_mask:
7745 case X86::BI__builtin_ia32_cmpb512_mask:
7746 case X86::BI__builtin_ia32_cmpw128_mask:
7747 case X86::BI__builtin_ia32_cmpw256_mask:
7748 case X86::BI__builtin_ia32_cmpw512_mask:
7749 case X86::BI__builtin_ia32_cmpd128_mask:
7750 case X86::BI__builtin_ia32_cmpd256_mask:
7751 case X86::BI__builtin_ia32_cmpd512_mask:
7752 case X86::BI__builtin_ia32_cmpq128_mask:
7753 case X86::BI__builtin_ia32_cmpq256_mask:
7754 case X86::BI__builtin_ia32_cmpq512_mask: {
7755 unsigned CC = cast<llvm::ConstantInt>(Ops[2])->getZExtValue() & 0x7;
7758 case X86::BI__builtin_ia32_ucmpb128_mask:
7759 case X86::BI__builtin_ia32_ucmpb256_mask:
7760 case X86::BI__builtin_ia32_ucmpb512_mask:
7761 case X86::BI__builtin_ia32_ucmpw128_mask:
7762 case X86::BI__builtin_ia32_ucmpw256_mask:
7763 case X86::BI__builtin_ia32_ucmpw512_mask:
7764 case X86::BI__builtin_ia32_ucmpd128_mask:
7765 case X86::BI__builtin_ia32_ucmpd256_mask:
7766 case X86::BI__builtin_ia32_ucmpd512_mask:
7767 case X86::BI__builtin_ia32_ucmpq128_mask:
7768 case X86::BI__builtin_ia32_ucmpq256_mask:
7769 case X86::BI__builtin_ia32_ucmpq512_mask: {
7770 unsigned CC = cast<llvm::ConstantInt>(Ops[2])->getZExtValue() & 0x7;
7774 case X86::BI__builtin_ia32_vplzcntd_128_mask:
7775 case X86::BI__builtin_ia32_vplzcntd_256_mask:
7776 case X86::BI__builtin_ia32_vplzcntd_512_mask:
7777 case X86::BI__builtin_ia32_vplzcntq_128_mask:
7778 case X86::BI__builtin_ia32_vplzcntq_256_mask:
7779 case X86::BI__builtin_ia32_vplzcntq_512_mask: {
7786 case X86::BI__builtin_ia32_pmaxsb128:
7787 case X86::BI__builtin_ia32_pmaxsw128:
7788 case X86::BI__builtin_ia32_pmaxsd128:
7789 case X86::BI__builtin_ia32_pmaxsq128_mask:
7790 case X86::BI__builtin_ia32_pmaxsb256:
7791 case X86::BI__builtin_ia32_pmaxsw256:
7792 case X86::BI__builtin_ia32_pmaxsd256:
7793 case X86::BI__builtin_ia32_pmaxsq256_mask:
7794 case X86::BI__builtin_ia32_pmaxsb512_mask:
7795 case X86::BI__builtin_ia32_pmaxsw512_mask:
7796 case X86::BI__builtin_ia32_pmaxsd512_mask:
7797 case X86::BI__builtin_ia32_pmaxsq512_mask:
7799 case X86::BI__builtin_ia32_pmaxub128:
7800 case X86::BI__builtin_ia32_pmaxuw128:
7801 case X86::BI__builtin_ia32_pmaxud128:
7802 case X86::BI__builtin_ia32_pmaxuq128_mask:
7803 case X86::BI__builtin_ia32_pmaxub256:
7804 case X86::BI__builtin_ia32_pmaxuw256:
7805 case X86::BI__builtin_ia32_pmaxud256:
7806 case X86::BI__builtin_ia32_pmaxuq256_mask:
7807 case X86::BI__builtin_ia32_pmaxub512_mask:
7808 case X86::BI__builtin_ia32_pmaxuw512_mask:
7809 case X86::BI__builtin_ia32_pmaxud512_mask:
7810 case X86::BI__builtin_ia32_pmaxuq512_mask:
7812 case X86::BI__builtin_ia32_pminsb128:
7813 case X86::BI__builtin_ia32_pminsw128:
7814 case X86::BI__builtin_ia32_pminsd128:
7815 case X86::BI__builtin_ia32_pminsq128_mask:
7816 case X86::BI__builtin_ia32_pminsb256:
7817 case X86::BI__builtin_ia32_pminsw256:
7818 case X86::BI__builtin_ia32_pminsd256:
7819 case X86::BI__builtin_ia32_pminsq256_mask:
7820 case X86::BI__builtin_ia32_pminsb512_mask:
7821 case X86::BI__builtin_ia32_pminsw512_mask:
7822 case X86::BI__builtin_ia32_pminsd512_mask:
7823 case X86::BI__builtin_ia32_pminsq512_mask:
7825 case X86::BI__builtin_ia32_pminub128:
7826 case X86::BI__builtin_ia32_pminuw128:
7827 case X86::BI__builtin_ia32_pminud128:
7828 case X86::BI__builtin_ia32_pminuq128_mask:
7829 case X86::BI__builtin_ia32_pminub256:
7830 case X86::BI__builtin_ia32_pminuw256:
7831 case X86::BI__builtin_ia32_pminud256:
7832 case X86::BI__builtin_ia32_pminuq256_mask:
7833 case X86::BI__builtin_ia32_pminub512_mask:
7834 case X86::BI__builtin_ia32_pminuw512_mask:
7835 case X86::BI__builtin_ia32_pminud512_mask:
7836 case X86::BI__builtin_ia32_pminuq512_mask:
7840 case X86::BI__builtin_ia32_pswapdsf:
7841 case X86::BI__builtin_ia32_pswapdsi: {
7845 return Builder.CreateCall(F, Ops,
"pswapd");
7847 case X86::BI__builtin_ia32_rdrand16_step:
7848 case X86::BI__builtin_ia32_rdrand32_step:
7849 case X86::BI__builtin_ia32_rdrand64_step:
7850 case X86::BI__builtin_ia32_rdseed16_step:
7851 case X86::BI__builtin_ia32_rdseed32_step:
7852 case X86::BI__builtin_ia32_rdseed64_step: {
7854 switch (BuiltinID) {
7855 default: llvm_unreachable(
"Unsupported intrinsic!");
7856 case X86::BI__builtin_ia32_rdrand16_step:
7857 ID = Intrinsic::x86_rdrand_16;
7859 case X86::BI__builtin_ia32_rdrand32_step:
7860 ID = Intrinsic::x86_rdrand_32;
7862 case X86::BI__builtin_ia32_rdrand64_step:
7863 ID = Intrinsic::x86_rdrand_64;
7865 case X86::BI__builtin_ia32_rdseed16_step:
7866 ID = Intrinsic::x86_rdseed_16;
7868 case X86::BI__builtin_ia32_rdseed32_step:
7869 ID = Intrinsic::x86_rdseed_32;
7871 case X86::BI__builtin_ia32_rdseed64_step:
7872 ID = Intrinsic::x86_rdseed_64;
7879 return Builder.CreateExtractValue(Call, 1);
7883 case X86::BI__builtin_ia32_cmpeqps:
7884 case X86::BI__builtin_ia32_cmpeqpd:
7885 return getVectorFCmpIR(CmpInst::FCMP_OEQ);
7886 case X86::BI__builtin_ia32_cmpltps:
7887 case X86::BI__builtin_ia32_cmpltpd:
7888 return getVectorFCmpIR(CmpInst::FCMP_OLT);
7889 case X86::BI__builtin_ia32_cmpleps:
7890 case X86::BI__builtin_ia32_cmplepd:
7891 return getVectorFCmpIR(CmpInst::FCMP_OLE);
7892 case X86::BI__builtin_ia32_cmpunordps:
7893 case X86::BI__builtin_ia32_cmpunordpd:
7894 return getVectorFCmpIR(CmpInst::FCMP_UNO);
7895 case X86::BI__builtin_ia32_cmpneqps:
7896 case X86::BI__builtin_ia32_cmpneqpd:
7897 return getVectorFCmpIR(CmpInst::FCMP_UNE);
7898 case X86::BI__builtin_ia32_cmpnltps:
7899 case X86::BI__builtin_ia32_cmpnltpd:
7900 return getVectorFCmpIR(CmpInst::FCMP_UGE);
7901 case X86::BI__builtin_ia32_cmpnleps:
7902 case X86::BI__builtin_ia32_cmpnlepd:
7903 return getVectorFCmpIR(CmpInst::FCMP_UGT);
7904 case X86::BI__builtin_ia32_cmpordps:
7905 case X86::BI__builtin_ia32_cmpordpd:
7906 return getVectorFCmpIR(CmpInst::FCMP_ORD);
7907 case X86::BI__builtin_ia32_cmpps:
7908 case X86::BI__builtin_ia32_cmpps256:
7909 case X86::BI__builtin_ia32_cmppd:
7910 case X86::BI__builtin_ia32_cmppd256: {
7911 unsigned CC = cast<llvm::ConstantInt>(Ops[2])->getZExtValue();
7914 FCmpInst::Predicate Pred;
7916 case 0: Pred = FCmpInst::FCMP_OEQ;
break;
7917 case 1: Pred = FCmpInst::FCMP_OLT;
break;
7918 case 2: Pred = FCmpInst::FCMP_OLE;
break;
7919 case 3: Pred = FCmpInst::FCMP_UNO;
break;
7920 case 4: Pred = FCmpInst::FCMP_UNE;
break;
7921 case 5: Pred = FCmpInst::FCMP_UGE;
break;
7922 case 6: Pred = FCmpInst::FCMP_UGT;
break;
7923 case 7: Pred = FCmpInst::FCMP_ORD;
break;
7925 return getVectorFCmpIR(Pred);
7931 switch (BuiltinID) {
7932 default: llvm_unreachable(
"Unsupported intrinsic!");
7933 case X86::BI__builtin_ia32_cmpps:
7934 ID = Intrinsic::x86_sse_cmp_ps;
7936 case X86::BI__builtin_ia32_cmpps256:
7939 if (CC == 0xf || CC == 0xb || CC == 0x1b || CC == 0x1f) {
7940 Value *Constant = (CC == 0xf || CC == 0x1f) ?
7941 llvm::Constant::getAllOnesValue(
Builder.getInt32Ty()) :
7942 llvm::Constant::getNullValue(
Builder.getInt32Ty());
7943 Value *Vec =
Builder.CreateVectorSplat(
7944 Ops[0]->getType()->getVectorNumElements(), Constant);
7947 ID = Intrinsic::x86_avx_cmp_ps_256;
7949 case X86::BI__builtin_ia32_cmppd:
7950 ID = Intrinsic::x86_sse2_cmp_pd;
7952 case X86::BI__builtin_ia32_cmppd256:
7955 if (CC == 0xf || CC == 0xb || CC == 0x1b || CC == 0x1f) {
7956 Value *Constant = (CC == 0xf || CC == 0x1f) ?
7957 llvm::Constant::getAllOnesValue(
Builder.getInt64Ty()) :
7958 llvm::Constant::getNullValue(
Builder.getInt64Ty());
7959 Value *Vec =
Builder.CreateVectorSplat(
7960 Ops[0]->getType()->getVectorNumElements(), Constant);
7963 ID = Intrinsic::x86_avx_cmp_pd_256;
7971 case X86::BI__builtin_ia32_cmpeqss:
7972 return getCmpIntrinsicCall(Intrinsic::x86_sse_cmp_ss, 0);
7973 case X86::BI__builtin_ia32_cmpltss:
7974 return getCmpIntrinsicCall(Intrinsic::x86_sse_cmp_ss, 1);
7975 case X86::BI__builtin_ia32_cmpless:
7976 return getCmpIntrinsicCall(Intrinsic::x86_sse_cmp_ss, 2);
7977 case X86::BI__builtin_ia32_cmpunordss:
7978 return getCmpIntrinsicCall(Intrinsic::x86_sse_cmp_ss, 3);
7979 case X86::BI__builtin_ia32_cmpneqss:
7980 return getCmpIntrinsicCall(Intrinsic::x86_sse_cmp_ss, 4);
7981 case X86::BI__builtin_ia32_cmpnltss:
7982 return getCmpIntrinsicCall(Intrinsic::x86_sse_cmp_ss, 5);
7983 case X86::BI__builtin_ia32_cmpnless:
7984 return getCmpIntrinsicCall(Intrinsic::x86_sse_cmp_ss, 6);
7985 case X86::BI__builtin_ia32_cmpordss:
7986 return getCmpIntrinsicCall(Intrinsic::x86_sse_cmp_ss, 7);
7987 case X86::BI__builtin_ia32_cmpeqsd:
7988 return getCmpIntrinsicCall(Intrinsic::x86_sse2_cmp_sd, 0);
7989 case X86::BI__builtin_ia32_cmpltsd:
7990 return getCmpIntrinsicCall(Intrinsic::x86_sse2_cmp_sd, 1);
7991 case X86::BI__builtin_ia32_cmplesd:
7992 return getCmpIntrinsicCall(Intrinsic::x86_sse2_cmp_sd, 2);
7993 case X86::BI__builtin_ia32_cmpunordsd:
7994 return getCmpIntrinsicCall(Intrinsic::x86_sse2_cmp_sd, 3);
7995 case X86::BI__builtin_ia32_cmpneqsd:
7996 return getCmpIntrinsicCall(Intrinsic::x86_sse2_cmp_sd, 4);
7997 case X86::BI__builtin_ia32_cmpnltsd:
7998 return getCmpIntrinsicCall(Intrinsic::x86_sse2_cmp_sd, 5);
7999 case X86::BI__builtin_ia32_cmpnlesd:
8000 return getCmpIntrinsicCall(Intrinsic::x86_sse2_cmp_sd, 6);
8001 case X86::BI__builtin_ia32_cmpordsd:
8002 return getCmpIntrinsicCall(Intrinsic::x86_sse2_cmp_sd, 7);
8005 case X86::BI__emulu: {
8007 bool isSigned = (BuiltinID == X86::BI__emul);
8008 Value *LHS =
Builder.CreateIntCast(Ops[0], Int64Ty, isSigned);
8009 Value *RHS =
Builder.CreateIntCast(Ops[1], Int64Ty, isSigned);
8010 return Builder.CreateMul(LHS, RHS,
"", !isSigned, isSigned);
8013 case X86::BI__umulh:
8014 case X86::BI_mul128:
8015 case X86::BI_umul128: {
8019 bool IsSigned = (BuiltinID == X86::BI__mulh || BuiltinID == X86::BI_mul128);
8020 Value *LHS =
Builder.CreateIntCast(Ops[0], Int128Ty, IsSigned);
8021 Value *RHS =
Builder.CreateIntCast(Ops[1], Int128Ty, IsSigned);
8023 Value *MulResult, *HigherBits;
8025 MulResult =
Builder.CreateNSWMul(LHS, RHS);
8026 HigherBits =
Builder.CreateAShr(MulResult, 64);
8028 MulResult =
Builder.CreateNUWMul(LHS, RHS);
8029 HigherBits =
Builder.CreateLShr(MulResult, 64);
8031 HigherBits =
Builder.CreateIntCast(HigherBits, ResType, IsSigned);
8033 if (BuiltinID == X86::BI__mulh || BuiltinID == X86::BI__umulh)
8038 return Builder.CreateIntCast(MulResult, ResType, IsSigned);
8041 case X86::BI__faststorefence: {
8042 return Builder.CreateFence(llvm::AtomicOrdering::SequentiallyConsistent,
8045 case X86::BI_ReadWriteBarrier:
8046 case X86::BI_ReadBarrier:
8047 case X86::BI_WriteBarrier: {
8048 return Builder.CreateFence(llvm::AtomicOrdering::SequentiallyConsistent,
8049 llvm::SyncScope::SingleThread);
8051 case X86::BI_BitScanForward:
8052 case X86::BI_BitScanForward64:
8054 case X86::BI_BitScanReverse:
8055 case X86::BI_BitScanReverse64:
8058 case X86::BI_InterlockedAnd64:
8060 case X86::BI_InterlockedExchange64:
8062 case X86::BI_InterlockedExchangeAdd64:
8064 case X86::BI_InterlockedExchangeSub64:
8066 case X86::BI_InterlockedOr64:
8068 case X86::BI_InterlockedXor64:
8070 case X86::BI_InterlockedDecrement64:
8072 case X86::BI_InterlockedIncrement64:
8075 case X86::BI_AddressOfReturnAddress: {
8079 case X86::BI__stosb: {
8087 case X86::BI__int2c: {
8089 llvm::FunctionType *FTy = llvm::FunctionType::get(
VoidTy,
false);
8090 llvm::InlineAsm *IA =
8091 llvm::InlineAsm::get(FTy,
"int $$0x2c",
"",
true);
8092 llvm::AttributeList NoReturnAttr = llvm::AttributeList::get(
8094 llvm::Attribute::NoReturn);
8095 CallSite CS =
Builder.CreateCall(IA);
8096 CS.setAttributes(NoReturnAttr);
8097 return CS.getInstruction();
8099 case X86::BI__readfsbyte:
8100 case X86::BI__readfsword:
8101 case X86::BI__readfsdword:
8102 case X86::BI__readfsqword: {
8105 llvm::PointerType::get(IntTy, 257));
8108 Load->setVolatile(
true);
8111 case X86::BI__readgsbyte:
8112 case X86::BI__readgsword:
8113 case X86::BI__readgsdword:
8114 case X86::BI__readgsqword: {
8117 llvm::PointerType::get(IntTy, 256));
8120 Load->setVolatile(
true);
8131 for (
unsigned i = 0, e = E->
getNumArgs(); i != e; i++)
8136 switch (BuiltinID) {
8137 default:
return nullptr;
8141 case PPC::BI__builtin_ppc_get_timebase:
8145 case PPC::BI__builtin_altivec_lvx:
8146 case PPC::BI__builtin_altivec_lvxl:
8147 case PPC::BI__builtin_altivec_lvebx:
8148 case PPC::BI__builtin_altivec_lvehx:
8149 case PPC::BI__builtin_altivec_lvewx:
8150 case PPC::BI__builtin_altivec_lvsl:
8151 case PPC::BI__builtin_altivec_lvsr:
8152 case PPC::BI__builtin_vsx_lxvd2x:
8153 case PPC::BI__builtin_vsx_lxvw4x:
8154 case PPC::BI__builtin_vsx_lxvd2x_be:
8155 case PPC::BI__builtin_vsx_lxvw4x_be:
8156 case PPC::BI__builtin_vsx_lxvl:
8157 case PPC::BI__builtin_vsx_lxvll:
8159 if(BuiltinID == PPC::BI__builtin_vsx_lxvl ||
8160 BuiltinID == PPC::BI__builtin_vsx_lxvll){
8164 Ops[0] =
Builder.CreateGEP(Ops[1], Ops[0]);
8168 switch (BuiltinID) {
8169 default: llvm_unreachable(
"Unsupported ld/lvsl/lvsr intrinsic!");
8170 case PPC::BI__builtin_altivec_lvx:
8171 ID = Intrinsic::ppc_altivec_lvx;
8173 case PPC::BI__builtin_altivec_lvxl:
8174 ID = Intrinsic::ppc_altivec_lvxl;
8176 case PPC::BI__builtin_altivec_lvebx:
8177 ID = Intrinsic::ppc_altivec_lvebx;
8179 case PPC::BI__builtin_altivec_lvehx:
8180 ID = Intrinsic::ppc_altivec_lvehx;
8182 case PPC::BI__builtin_altivec_lvewx:
8183 ID = Intrinsic::ppc_altivec_lvewx;
8185 case PPC::BI__builtin_altivec_lvsl:
8186 ID = Intrinsic::ppc_altivec_lvsl;
8188 case PPC::BI__builtin_altivec_lvsr:
8189 ID = Intrinsic::ppc_altivec_lvsr;
8191 case PPC::BI__builtin_vsx_lxvd2x:
8192 ID = Intrinsic::ppc_vsx_lxvd2x;
8194 case PPC::BI__builtin_vsx_lxvw4x:
8195 ID = Intrinsic::ppc_vsx_lxvw4x;
8197 case PPC::BI__builtin_vsx_lxvd2x_be:
8198 ID = Intrinsic::ppc_vsx_lxvd2x_be;
8200 case PPC::BI__builtin_vsx_lxvw4x_be:
8201 ID = Intrinsic::ppc_vsx_lxvw4x_be;
8203 case PPC::BI__builtin_vsx_lxvl:
8204 ID = Intrinsic::ppc_vsx_lxvl;
8206 case PPC::BI__builtin_vsx_lxvll:
8207 ID = Intrinsic::ppc_vsx_lxvll;
8211 return Builder.CreateCall(F, Ops,
"");
8215 case PPC::BI__builtin_altivec_stvx:
8216 case PPC::BI__builtin_altivec_stvxl:
8217 case PPC::BI__builtin_altivec_stvebx:
8218 case PPC::BI__builtin_altivec_stvehx:
8219 case PPC::BI__builtin_altivec_stvewx:
8220 case PPC::BI__builtin_vsx_stxvd2x:
8221 case PPC::BI__builtin_vsx_stxvw4x:
8222 case PPC::BI__builtin_vsx_stxvd2x_be:
8223 case PPC::BI__builtin_vsx_stxvw4x_be:
8224 case PPC::BI__builtin_vsx_stxvl:
8225 case PPC::BI__builtin_vsx_stxvll:
8227 if(BuiltinID == PPC::BI__builtin_vsx_stxvl ||
8228 BuiltinID == PPC::BI__builtin_vsx_stxvll ){
8232 Ops[1] =
Builder.CreateGEP(Ops[2], Ops[1]);
8236 switch (BuiltinID) {
8237 default: llvm_unreachable(
"Unsupported st intrinsic!");
8238 case PPC::BI__builtin_altivec_stvx:
8239 ID = Intrinsic::ppc_altivec_stvx;
8241 case PPC::BI__builtin_altivec_stvxl:
8242 ID = Intrinsic::ppc_altivec_stvxl;
8244 case PPC::BI__builtin_altivec_stvebx:
8245 ID = Intrinsic::ppc_altivec_stvebx;
8247 case PPC::BI__builtin_altivec_stvehx:
8248 ID = Intrinsic::ppc_altivec_stvehx;
8250 case PPC::BI__builtin_altivec_stvewx:
8251 ID = Intrinsic::ppc_altivec_stvewx;
8253 case PPC::BI__builtin_vsx_stxvd2x:
8254 ID = Intrinsic::ppc_vsx_stxvd2x;
8256 case PPC::BI__builtin_vsx_stxvw4x:
8257 ID = Intrinsic::ppc_vsx_stxvw4x;
8259 case PPC::BI__builtin_vsx_stxvd2x_be:
8260 ID = Intrinsic::ppc_vsx_stxvd2x_be;
8262 case PPC::BI__builtin_vsx_stxvw4x_be:
8263 ID = Intrinsic::ppc_vsx_stxvw4x_be;
8265 case PPC::BI__builtin_vsx_stxvl:
8266 ID = Intrinsic::ppc_vsx_stxvl;
8268 case PPC::BI__builtin_vsx_stxvll:
8269 ID = Intrinsic::ppc_vsx_stxvll;
8273 return Builder.CreateCall(F, Ops,
"");
8276 case PPC::BI__builtin_vsx_xvsqrtsp:
8277 case PPC::BI__builtin_vsx_xvsqrtdp: {
8282 return Builder.CreateCall(F, X);
8285 case PPC::BI__builtin_altivec_vclzb:
8286 case PPC::BI__builtin_altivec_vclzh:
8287 case PPC::BI__builtin_altivec_vclzw:
8288 case PPC::BI__builtin_altivec_vclzd: {
8291 Value *Undef = ConstantInt::get(
Builder.getInt1Ty(),
false);
8293 return Builder.CreateCall(F, {
X, Undef});
8295 case PPC::BI__builtin_altivec_vctzb:
8296 case PPC::BI__builtin_altivec_vctzh:
8297 case PPC::BI__builtin_altivec_vctzw:
8298 case PPC::BI__builtin_altivec_vctzd: {
8301 Value *Undef = ConstantInt::get(
Builder.getInt1Ty(),
false);
8303 return Builder.CreateCall(F, {
X, Undef});
8305 case PPC::BI__builtin_altivec_vpopcntb:
8306 case PPC::BI__builtin_altivec_vpopcnth:
8307 case PPC::BI__builtin_altivec_vpopcntw:
8308 case PPC::BI__builtin_altivec_vpopcntd: {
8312 return Builder.CreateCall(F, X);
8315 case PPC::BI__builtin_vsx_xvcpsgnsp:
8316 case PPC::BI__builtin_vsx_xvcpsgndp: {
8322 return Builder.CreateCall(F, {
X, Y});
8325 case PPC::BI__builtin_vsx_xvrspip:
8326 case PPC::BI__builtin_vsx_xvrdpip:
8327 case PPC::BI__builtin_vsx_xvrdpim:
8328 case PPC::BI__builtin_vsx_xvrspim:
8329 case PPC::BI__builtin_vsx_xvrdpi:
8330 case PPC::BI__builtin_vsx_xvrspi:
8331 case PPC::BI__builtin_vsx_xvrdpic:
8332 case PPC::BI__builtin_vsx_xvrspic:
8333 case PPC::BI__builtin_vsx_xvrdpiz:
8334 case PPC::BI__builtin_vsx_xvrspiz: {
8337 if (BuiltinID == PPC::BI__builtin_vsx_xvrdpim ||
8338 BuiltinID == PPC::BI__builtin_vsx_xvrspim)
8340 else if (BuiltinID == PPC::BI__builtin_vsx_xvrdpi ||
8341 BuiltinID == PPC::BI__builtin_vsx_xvrspi)
8343 else if (BuiltinID == PPC::BI__builtin_vsx_xvrdpic ||
8344 BuiltinID == PPC::BI__builtin_vsx_xvrspic)
8346 else if (BuiltinID == PPC::BI__builtin_vsx_xvrdpip ||
8347 BuiltinID == PPC::BI__builtin_vsx_xvrspip)
8349 else if (BuiltinID == PPC::BI__builtin_vsx_xvrdpiz ||
8350 BuiltinID == PPC::BI__builtin_vsx_xvrspiz)
8353 return Builder.CreateCall(F, X);
8357 case PPC::BI__builtin_vsx_xvabsdp:
8358 case PPC::BI__builtin_vsx_xvabssp: {
8362 return Builder.CreateCall(F, X);
8366 case PPC::BI__builtin_vsx_xvmaddadp:
8367 case PPC::BI__builtin_vsx_xvmaddasp:
8368 case PPC::BI__builtin_vsx_xvnmaddadp:
8369 case PPC::BI__builtin_vsx_xvnmaddasp:
8370 case PPC::BI__builtin_vsx_xvmsubadp:
8371 case PPC::BI__builtin_vsx_xvmsubasp:
8372 case PPC::BI__builtin_vsx_xvnmsubadp:
8373 case PPC::BI__builtin_vsx_xvnmsubasp: {
8378 Value *Zero = llvm::ConstantFP::getZeroValueForNegation(ResultType);
8380 switch (BuiltinID) {
8381 case PPC::BI__builtin_vsx_xvmaddadp:
8382 case PPC::BI__builtin_vsx_xvmaddasp:
8383 return Builder.CreateCall(F, {
X, Y, Z});
8384 case PPC::BI__builtin_vsx_xvnmaddadp:
8385 case PPC::BI__builtin_vsx_xvnmaddasp:
8386 return Builder.CreateFSub(Zero,
8387 Builder.CreateCall(F, {X, Y, Z}),
"sub");
8388 case PPC::BI__builtin_vsx_xvmsubadp:
8389 case PPC::BI__builtin_vsx_xvmsubasp:
8391 {
X, Y,
Builder.CreateFSub(Zero, Z,
"sub")});
8392 case PPC::BI__builtin_vsx_xvnmsubadp:
8393 case PPC::BI__builtin_vsx_xvnmsubasp:
8396 return Builder.CreateFSub(Zero, FsubRes,
"sub");
8398 llvm_unreachable(
"Unknown FMA operation");
8402 case PPC::BI__builtin_vsx_insertword: {
8407 ConstantInt *ArgCI = dyn_cast<ConstantInt>(Ops[2]);
8409 "Third arg to xxinsertw intrinsic must be constant integer");
8410 const int64_t MaxIndex = 12;
8411 int64_t Index =
clamp(ArgCI->getSExtValue(), 0, MaxIndex);
8418 std::swap(Ops[0], Ops[1]);
8426 Constant *ShuffleElts[2] = { ConstantInt::get(
Int32Ty, 1),
8429 Constant *ShuffleMask = llvm::ConstantVector::get(ShuffleElts);
8433 Ops[0] =
Builder.CreateShuffleVector(Ops[0], Ops[0], ShuffleMask);
8436 Index = MaxIndex - Index;
8441 Ops[2] = ConstantInt::getSigned(
Int32Ty, Index);
8442 return Builder.CreateCall(F, Ops);
8445 case PPC::BI__builtin_vsx_extractuword: {
8453 ConstantInt *ArgCI = dyn_cast<ConstantInt>(Ops[1]);
8455 "Second Arg to xxextractuw intrinsic must be a constant integer!");
8456 const int64_t MaxIndex = 12;
8457 int64_t Index =
clamp(ArgCI->getSExtValue(), 0, MaxIndex);
8461 Index = MaxIndex - Index;
8462 Ops[1] = ConstantInt::getSigned(
Int32Ty, Index);
8465 Value *Call =
Builder.CreateCall(F, Ops);
8468 Constant *ShuffleElts[2] = { ConstantInt::get(
Int32Ty, 1),
8471 Constant *ShuffleMask = llvm::ConstantVector::get(ShuffleElts);
8473 Value *ShuffleCall =
Builder.CreateShuffleVector(Call, Call, ShuffleMask);
8476 Ops[1] = ConstantInt::getSigned(
Int32Ty, Index);
8477 return Builder.CreateCall(F, Ops);
8481 case PPC::BI__builtin_vsx_xxpermdi: {
8482 ConstantInt *ArgCI = dyn_cast<ConstantInt>(Ops[2]);
8483 assert(ArgCI &&
"Third arg must be constant integer!");
8485 unsigned Index = ArgCI->getZExtValue();
8496 ElemIdx0 = (~Index & 1) + 2;
8497 ElemIdx1 = (~Index & 2) >> 1;
8499 ElemIdx0 = (Index & 2) >> 1;
8500 ElemIdx1 = 2 + (Index & 1);
8503 Constant *ShuffleElts[2] = {ConstantInt::get(
Int32Ty, ElemIdx0),
8504 ConstantInt::get(
Int32Ty, ElemIdx1)};
8505 Constant *ShuffleMask = llvm::ConstantVector::get(ShuffleElts);
8507 Value *ShuffleCall =
8508 Builder.CreateShuffleVector(Ops[0], Ops[1], ShuffleMask);
8514 case PPC::BI__builtin_vsx_xxsldwi: {
8515 ConstantInt *ArgCI = dyn_cast<ConstantInt>(Ops[2]);
8516 assert(ArgCI &&
"Third argument must be a compile time constant");
8517 unsigned Index = ArgCI->getZExtValue() & 0x3;
8530 ElemIdx0 = (8 - Index) % 8;
8531 ElemIdx1 = (9 - Index) % 8;
8532 ElemIdx2 = (10 - Index) % 8;
8533 ElemIdx3 = (11 - Index) % 8;
8537 ElemIdx1 = Index + 1;
8538 ElemIdx2 = Index + 2;
8539 ElemIdx3 = Index + 3;
8542 Constant *ShuffleElts[4] = {ConstantInt::get(
Int32Ty, ElemIdx0),
8543 ConstantInt::get(
Int32Ty, ElemIdx1),
8544 ConstantInt::get(
Int32Ty, ElemIdx2),
8545 ConstantInt::get(
Int32Ty, ElemIdx3)};
8547 Constant *ShuffleMask = llvm::ConstantVector::get(ShuffleElts);
8548 Value *ShuffleCall =
8549 Builder.CreateShuffleVector(Ops[0], Ops[1], ShuffleMask);
8559 switch (BuiltinID) {
8560 case AMDGPU::BI__builtin_amdgcn_div_scale:
8561 case AMDGPU::BI__builtin_amdgcn_div_scalef: {
8580 = FlagOutPtr.
getPointer()->getType()->getPointerElementType();
8586 case AMDGPU::BI__builtin_amdgcn_div_fmas:
8587 case AMDGPU::BI__builtin_amdgcn_div_fmasf: {
8596 return Builder.CreateCall(F, {Src0, Src1, Src2, Src3ToBool});
8599 case AMDGPU::BI__builtin_amdgcn_ds_swizzle:
8601 case AMDGPU::BI__builtin_amdgcn_mov_dpp: {
8603 for (
unsigned I = 0;
I != 5; ++
I)
8606 Args[0]->getType());
8607 return Builder.CreateCall(F, Args);
8609 case AMDGPU::BI__builtin_amdgcn_div_fixup:
8610 case AMDGPU::BI__builtin_amdgcn_div_fixupf:
8611 case AMDGPU::BI__builtin_amdgcn_div_fixuph:
8613 case AMDGPU::BI__builtin_amdgcn_trig_preop:
8614 case AMDGPU::BI__builtin_amdgcn_trig_preopf:
8616 case AMDGPU::BI__builtin_amdgcn_rcp:
8617 case AMDGPU::BI__builtin_amdgcn_rcpf:
8618 case AMDGPU::BI__builtin_amdgcn_rcph:
8620 case AMDGPU::BI__builtin_amdgcn_rsq:
8621 case AMDGPU::BI__builtin_amdgcn_rsqf:
8622 case AMDGPU::BI__builtin_amdgcn_rsqh:
8624 case AMDGPU::BI__builtin_amdgcn_rsq_clamp:
8625 case AMDGPU::BI__builtin_amdgcn_rsq_clampf:
8627 case AMDGPU::BI__builtin_amdgcn_sinf:
8628 case AMDGPU::BI__builtin_amdgcn_sinh:
8630 case AMDGPU::BI__builtin_amdgcn_cosf:
8631 case AMDGPU::BI__builtin_amdgcn_cosh:
8633 case AMDGPU::BI__builtin_amdgcn_log_clampf:
8635 case AMDGPU::BI__builtin_amdgcn_ldexp:
8636 case AMDGPU::BI__builtin_amdgcn_ldexpf:
8637 case AMDGPU::BI__builtin_amdgcn_ldexph:
8639 case AMDGPU::BI__builtin_amdgcn_frexp_mant:
8640 case AMDGPU::BI__builtin_amdgcn_frexp_mantf:
8641 case AMDGPU::BI__builtin_amdgcn_frexp_manth:
8643 case AMDGPU::BI__builtin_amdgcn_frexp_exp:
8644 case AMDGPU::BI__builtin_amdgcn_frexp_expf: {
8647 {
Builder.getInt32Ty(), Src0->getType() });
8648 return Builder.CreateCall(F, Src0);
8650 case AMDGPU::BI__builtin_amdgcn_frexp_exph: {
8653 {
Builder.getInt16Ty(), Src0->getType() });
8654 return Builder.CreateCall(F, Src0);
8656 case AMDGPU::BI__builtin_amdgcn_fract:
8657 case AMDGPU::BI__builtin_amdgcn_fractf:
8658 case AMDGPU::BI__builtin_amdgcn_fracth:
8660 case AMDGPU::BI__builtin_amdgcn_lerp:
8662 case AMDGPU::BI__builtin_amdgcn_uicmp:
8663 case AMDGPU::BI__builtin_amdgcn_uicmpl:
8664 case AMDGPU::BI__builtin_amdgcn_sicmp:
8665 case AMDGPU::BI__builtin_amdgcn_sicmpl:
8667 case AMDGPU::BI__builtin_amdgcn_fcmp:
8668 case AMDGPU::BI__builtin_amdgcn_fcmpf:
8670 case AMDGPU::BI__builtin_amdgcn_class:
8671 case AMDGPU::BI__builtin_amdgcn_classf:
8672 case AMDGPU::BI__builtin_amdgcn_classh:
8674 case AMDGPU::BI__builtin_amdgcn_fmed3f:
8675 case AMDGPU::BI__builtin_amdgcn_fmed3h:
8677 case AMDGPU::BI__builtin_amdgcn_read_exec: {
8678 CallInst *CI = cast<CallInst>(
8680 CI->setConvergent();
8685 case AMDGPU::BI__builtin_amdgcn_workitem_id_x:
8687 case AMDGPU::BI__builtin_amdgcn_workitem_id_y:
8689 case AMDGPU::BI__builtin_amdgcn_workitem_id_z:
8693 case AMDGPU::BI__builtin_r600_recipsqrt_ieee:
8694 case AMDGPU::BI__builtin_r600_recipsqrt_ieeef:
8696 case AMDGPU::BI__builtin_r600_read_tidig_x:
8698 case AMDGPU::BI__builtin_r600_read_tidig_y:
8700 case AMDGPU::BI__builtin_r600_read_tidig_z:
8711 unsigned IntrinsicID,
8715 for (
unsigned I = 0;
I < NumArgs; ++
I)
8722 return CGF.
Builder.CreateExtractValue(Call, 0);
8727 switch (BuiltinID) {
8728 case SystemZ::BI__builtin_tbegin: {
8730 Value *Control = llvm::ConstantInt::get(
Int32Ty, 0xff0c);
8732 return Builder.CreateCall(F, {TDB, Control});
8734 case SystemZ::BI__builtin_tbegin_nofloat: {
8736 Value *Control = llvm::ConstantInt::get(
Int32Ty, 0xff0c);
8738 return Builder.CreateCall(F, {TDB, Control});
8740 case SystemZ::BI__builtin_tbeginc: {
8742 Value *Control = llvm::ConstantInt::get(
Int32Ty, 0xff08);
8744 return Builder.CreateCall(F, {TDB, Control});
8746 case SystemZ::BI__builtin_tabort: {
8751 case SystemZ::BI__builtin_non_tx_store: {
8755 return Builder.CreateCall(F, {Data, Address});
8763 case SystemZ::BI__builtin_s390_vpopctb:
8764 case SystemZ::BI__builtin_s390_vpopcth:
8765 case SystemZ::BI__builtin_s390_vpopctf:
8766 case SystemZ::BI__builtin_s390_vpopctg: {
8770 return Builder.CreateCall(F, X);
8773 case SystemZ::BI__builtin_s390_vclzb:
8774 case SystemZ::BI__builtin_s390_vclzh:
8775 case SystemZ::BI__builtin_s390_vclzf:
8776 case SystemZ::BI__builtin_s390_vclzg: {
8779 Value *Undef = ConstantInt::get(
Builder.getInt1Ty(),
false);
8781 return Builder.CreateCall(F, {
X, Undef});
8784 case SystemZ::BI__builtin_s390_vctzb:
8785 case SystemZ::BI__builtin_s390_vctzh:
8786 case SystemZ::BI__builtin_s390_vctzf:
8787 case SystemZ::BI__builtin_s390_vctzg: {
8790 Value *Undef = ConstantInt::get(
Builder.getInt1Ty(),
false);
8792 return Builder.CreateCall(F, {
X, Undef});
8795 case SystemZ::BI__builtin_s390_vfsqsb:
8796 case SystemZ::BI__builtin_s390_vfsqdb: {
8800 return Builder.CreateCall(F, X);
8802 case SystemZ::BI__builtin_s390_vfmasb:
8803 case SystemZ::BI__builtin_s390_vfmadb: {
8809 return Builder.CreateCall(F, {
X, Y, Z});
8811 case SystemZ::BI__builtin_s390_vfmssb:
8812 case SystemZ::BI__builtin_s390_vfmsdb: {
8817 Value *Zero = llvm::ConstantFP::getZeroValueForNegation(ResultType);
8819 return Builder.CreateCall(F, {
X, Y,
Builder.CreateFSub(Zero, Z,
"sub")});
8821 case SystemZ::BI__builtin_s390_vfnmasb:
8822 case SystemZ::BI__builtin_s390_vfnmadb: {
8827 Value *Zero = llvm::ConstantFP::getZeroValueForNegation(ResultType);
8829 return Builder.CreateFSub(Zero,
Builder.CreateCall(F, {X, Y, Z}),
"sub");
8831 case SystemZ::BI__builtin_s390_vfnmssb:
8832 case SystemZ::BI__builtin_s390_vfnmsdb: {
8837 Value *Zero = llvm::ConstantFP::getZeroValueForNegation(ResultType);
8840 return Builder.CreateFSub(Zero,
Builder.CreateCall(F, {X, Y, NegZ}));
8842 case SystemZ::BI__builtin_s390_vflpsb:
8843 case SystemZ::BI__builtin_s390_vflpdb: {
8845 Value *X = EmitScalarExpr(E->getArg(0));
8847 return Builder.CreateCall(F, X);
8849 case SystemZ::BI__builtin_s390_vflnsb:
8850 case SystemZ::BI__builtin_s390_vflndb: {
8852 Value *X = EmitScalarExpr(E->getArg(0));
8853 Value *Zero = llvm::ConstantFP::getZeroValueForNegation(ResultType);
8855 return Builder.CreateFSub(Zero,
Builder.CreateCall(F, X),
"sub");
8857 case SystemZ::BI__builtin_s390_vfisb:
8858 case SystemZ::BI__builtin_s390_vfidb: {
8860 Value *X = EmitScalarExpr(E->getArg(0));
8862 llvm::APSInt M4, M5;
8865 assert(IsConstM4 && IsConstM5 &&
"Constant arg isn't actually constant?");
8866 (void)IsConstM4; (void)IsConstM5;
8870 switch (M4.getZExtValue()) {
8873 switch (M5.getZExtValue()) {
8879 switch (M5.getZExtValue()) {
8889 if (ID != Intrinsic::not_intrinsic) {
8890 Function *F = CGM.getIntrinsic(ID, ResultType);
8891 return Builder.CreateCall(F, X);
8893 switch (BuiltinID) {
8894 case SystemZ::BI__builtin_s390_vfisb: ID = Intrinsic::s390_vfisb;
break;
8895 case SystemZ::BI__builtin_s390_vfidb: ID = Intrinsic::s390_vfidb;
break;
8896 default: llvm_unreachable(
"Unknown BuiltinID");
8898 Function *F = CGM.getIntrinsic(ID);
8899 Value *M4Value = llvm::ConstantInt::get(getLLVMContext(), M4);
8900 Value *M5Value = llvm::ConstantInt::get(getLLVMContext(), M5);
8901 return Builder.CreateCall(F, {
X, M4Value, M5Value});
8903 case SystemZ::BI__builtin_s390_vfmaxsb:
8904 case SystemZ::BI__builtin_s390_vfmaxdb: {
8906 Value *X = EmitScalarExpr(E->getArg(0));
8907 Value *Y = EmitScalarExpr(E->getArg(1));
8911 assert(IsConstM4 &&
"Constant arg isn't actually constant?");
8916 switch (M4.getZExtValue()) {
8918 case 4: ID = Intrinsic::maxnum;
break;
8920 if (ID != Intrinsic::not_intrinsic) {
8921 Function *F = CGM.getIntrinsic(ID, ResultType);
8922 return Builder.CreateCall(F, {
X, Y});
8924 switch (BuiltinID) {
8925 case SystemZ::BI__builtin_s390_vfmaxsb: ID = Intrinsic::s390_vfmaxsb;
break;
8926 case SystemZ::BI__builtin_s390_vfmaxdb: ID = Intrinsic::s390_vfmaxdb;
break;
8927 default: llvm_unreachable(
"Unknown BuiltinID");
8929 Function *F = CGM.getIntrinsic(ID);
8930 Value *M4Value = llvm::ConstantInt::get(getLLVMContext(), M4);
8931 return Builder.CreateCall(F, {
X, Y, M4Value});
8933 case SystemZ::BI__builtin_s390_vfminsb:
8934 case SystemZ::BI__builtin_s390_vfmindb: {
8936 Value *X = EmitScalarExpr(E->getArg(0));
8937 Value *Y = EmitScalarExpr(E->getArg(1));
8941 assert(IsConstM4 &&
"Constant arg isn't actually constant?");
8946 switch (M4.getZExtValue()) {
8948 case 4: ID = Intrinsic::minnum;
break;
8950 if (ID != Intrinsic::not_intrinsic) {
8951 Function *F = CGM.getIntrinsic(ID, ResultType);
8952 return Builder.CreateCall(F, {
X, Y});
8954 switch (BuiltinID) {
8955 case SystemZ::BI__builtin_s390_vfminsb: ID = Intrinsic::s390_vfminsb;
break;
8956 case SystemZ::BI__builtin_s390_vfmindb: ID = Intrinsic::s390_vfmindb;
break;
8957 default: llvm_unreachable(
"Unknown BuiltinID");
8959 Function *F = CGM.getIntrinsic(ID);
8960 Value *M4Value = llvm::ConstantInt::get(getLLVMContext(), M4);
8961 return Builder.CreateCall(F, {
X, Y, M4Value});
8966 #define INTRINSIC_WITH_CC(NAME) \
8967 case SystemZ::BI__builtin_##NAME: \
8968 return EmitSystemZIntrinsicWithCC(*this, Intrinsic::NAME, E)
9039 #undef INTRINSIC_WITH_CC
9048 auto MakeLdg = [&](
unsigned IntrinsicID) {
9053 CGM.
getIntrinsic(IntrinsicID, {Ptr->getType()->getPointerElementType(),
9055 {Ptr, ConstantInt::get(
Builder.getInt32Ty(), Align.getQuantity())});
9057 auto MakeScopedAtomic = [&](
unsigned IntrinsicID) {
9060 CGM.
getIntrinsic(IntrinsicID, {Ptr->getType()->getPointerElementType(),
9064 switch (BuiltinID) {
9065 case NVPTX::BI__nvvm_atom_add_gen_i:
9066 case NVPTX::BI__nvvm_atom_add_gen_l:
9067 case NVPTX::BI__nvvm_atom_add_gen_ll:
9070 case NVPTX::BI__nvvm_atom_sub_gen_i:
9071 case NVPTX::BI__nvvm_atom_sub_gen_l:
9072 case NVPTX::BI__nvvm_atom_sub_gen_ll:
9075 case NVPTX::BI__nvvm_atom_and_gen_i:
9076 case NVPTX::BI__nvvm_atom_and_gen_l:
9077 case NVPTX::BI__nvvm_atom_and_gen_ll:
9080 case NVPTX::BI__nvvm_atom_or_gen_i:
9081 case NVPTX::BI__nvvm_atom_or_gen_l:
9082 case NVPTX::BI__nvvm_atom_or_gen_ll:
9085 case NVPTX::BI__nvvm_atom_xor_gen_i:
9086 case NVPTX::BI__nvvm_atom_xor_gen_l:
9087 case NVPTX::BI__nvvm_atom_xor_gen_ll:
9090 case NVPTX::BI__nvvm_atom_xchg_gen_i:
9091 case NVPTX::BI__nvvm_atom_xchg_gen_l:
9092 case NVPTX::BI__nvvm_atom_xchg_gen_ll:
9095 case NVPTX::BI__nvvm_atom_max_gen_i:
9096 case NVPTX::BI__nvvm_atom_max_gen_l:
9097 case NVPTX::BI__nvvm_atom_max_gen_ll:
9100 case NVPTX::BI__nvvm_atom_max_gen_ui:
9101 case NVPTX::BI__nvvm_atom_max_gen_ul:
9102 case NVPTX::BI__nvvm_atom_max_gen_ull:
9105 case NVPTX::BI__nvvm_atom_min_gen_i:
9106 case NVPTX::BI__nvvm_atom_min_gen_l:
9107 case NVPTX::BI__nvvm_atom_min_gen_ll:
9110 case NVPTX::BI__nvvm_atom_min_gen_ui:
9111 case NVPTX::BI__nvvm_atom_min_gen_ul:
9112 case NVPTX::BI__nvvm_atom_min_gen_ull:
9115 case NVPTX::BI__nvvm_atom_cas_gen_i:
9116 case NVPTX::BI__nvvm_atom_cas_gen_l:
9117 case NVPTX::BI__nvvm_atom_cas_gen_ll:
9122 case NVPTX::BI__nvvm_atom_add_gen_f: {
9128 CGM.
getIntrinsic(Intrinsic::nvvm_atomic_load_add_f32, Ptr->getType());
9129 return Builder.CreateCall(FnALAF32, {Ptr, Val});
9132 case NVPTX::BI__nvvm_atom_inc_gen_ui: {
9137 return Builder.CreateCall(FnALI32, {Ptr, Val});
9140 case NVPTX::BI__nvvm_atom_dec_gen_ui: {
9145 return Builder.CreateCall(FnALD32, {Ptr, Val});
9148 case NVPTX::BI__nvvm_ldg_c:
9149 case NVPTX::BI__nvvm_ldg_c2:
9150 case NVPTX::BI__nvvm_ldg_c4:
9151 case NVPTX::BI__nvvm_ldg_s:
9152 case NVPTX::BI__nvvm_ldg_s2:
9153 case NVPTX::BI__nvvm_ldg_s4:
9154 case NVPTX::BI__nvvm_ldg_i:
9155 case NVPTX::BI__nvvm_ldg_i2:
9156 case NVPTX::BI__nvvm_ldg_i4:
9157 case NVPTX::BI__nvvm_ldg_l:
9158 case NVPTX::BI__nvvm_ldg_ll:
9159 case NVPTX::BI__nvvm_ldg_ll2:
9160 case NVPTX::BI__nvvm_ldg_uc:
9161 case NVPTX::BI__nvvm_ldg_uc2:
9162 case NVPTX::BI__nvvm_ldg_uc4:
9163 case NVPTX::BI__nvvm_ldg_us:
9164 case NVPTX::BI__nvvm_ldg_us2:
9165 case NVPTX::BI__nvvm_ldg_us4:
9166 case NVPTX::BI__nvvm_ldg_ui:
9167 case NVPTX::BI__nvvm_ldg_ui2:
9168 case NVPTX::BI__nvvm_ldg_ui4:
9169 case NVPTX::BI__nvvm_ldg_ul:
9170 case NVPTX::BI__nvvm_ldg_ull:
9171 case NVPTX::BI__nvvm_ldg_ull2:
9175 return MakeLdg(Intrinsic::nvvm_ldg_global_i);
9176 case NVPTX::BI__nvvm_ldg_f:
9177 case NVPTX::BI__nvvm_ldg_f2:
9178 case NVPTX::BI__nvvm_ldg_f4:
9179 case NVPTX::BI__nvvm_ldg_d:
9180 case NVPTX::BI__nvvm_ldg_d2:
9181 return MakeLdg(Intrinsic::nvvm_ldg_global_f);
9183 case NVPTX::BI__nvvm_atom_cta_add_gen_i:
9184 case NVPTX::BI__nvvm_atom_cta_add_gen_l:
9185 case NVPTX::BI__nvvm_atom_cta_add_gen_ll:
9186 return MakeScopedAtomic(Intrinsic::nvvm_atomic_add_gen_i_cta);
9187 case NVPTX::BI__nvvm_atom_sys_add_gen_i:
9188 case NVPTX::BI__nvvm_atom_sys_add_gen_l:
9189 case NVPTX::BI__nvvm_atom_sys_add_gen_ll:
9190 return MakeScopedAtomic(Intrinsic::nvvm_atomic_add_gen_i_sys);
9191 case NVPTX::BI__nvvm_atom_cta_add_gen_f:
9192 case NVPTX::BI__nvvm_atom_cta_add_gen_d:
9193 return MakeScopedAtomic(Intrinsic::nvvm_atomic_add_gen_f_cta);
9194 case NVPTX::BI__nvvm_atom_sys_add_gen_f:
9195 case NVPTX::BI__nvvm_atom_sys_add_gen_d:
9196 return MakeScopedAtomic(Intrinsic::nvvm_atomic_add_gen_f_sys);
9197 case NVPTX::BI__nvvm_atom_cta_xchg_gen_i:
9198 case NVPTX::BI__nvvm_atom_cta_xchg_gen_l:
9199 case NVPTX::BI__nvvm_atom_cta_xchg_gen_ll:
9200 return MakeScopedAtomic(Intrinsic::nvvm_atomic_exch_gen_i_cta);
9201 case NVPTX::BI__nvvm_atom_sys_xchg_gen_i:
9202 case NVPTX::BI__nvvm_atom_sys_xchg_gen_l:
9203 case NVPTX::BI__nvvm_atom_sys_xchg_gen_ll:
9204 return MakeScopedAtomic(Intrinsic::nvvm_atomic_exch_gen_i_sys);
9205 case NVPTX::BI__nvvm_atom_cta_max_gen_i:
9206 case NVPTX::BI__nvvm_atom_cta_max_gen_ui:
9207 case NVPTX::BI__nvvm_atom_cta_max_gen_l:
9208 case NVPTX::BI__nvvm_atom_cta_max_gen_ul:
9209 case NVPTX::BI__nvvm_atom_cta_max_gen_ll:
9210 case NVPTX::BI__nvvm_atom_cta_max_gen_ull:
9211 return MakeScopedAtomic(Intrinsic::nvvm_atomic_max_gen_i_cta);
9212 case NVPTX::BI__nvvm_atom_sys_max_gen_i:
9213 case NVPTX::BI__nvvm_atom_sys_max_gen_ui:
9214 case NVPTX::BI__nvvm_atom_sys_max_gen_l:
9215 case NVPTX::BI__nvvm_atom_sys_max_gen_ul:
9216 case NVPTX::BI__nvvm_atom_sys_max_gen_ll:
9217 case NVPTX::BI__nvvm_atom_sys_max_gen_ull:
9218 return MakeScopedAtomic(Intrinsic::nvvm_atomic_max_gen_i_sys);
9219 case NVPTX::BI__nvvm_atom_cta_min_gen_i:
9220 case NVPTX::BI__nvvm_atom_cta_min_gen_ui:
9221 case NVPTX::BI__nvvm_atom_cta_min_gen_l:
9222 case NVPTX::BI__nvvm_atom_cta_min_gen_ul:
9223 case NVPTX::BI__nvvm_atom_cta_min_gen_ll:
9224 case NVPTX::BI__nvvm_atom_cta_min_gen_ull:
9225 return MakeScopedAtomic(Intrinsic::nvvm_atomic_min_gen_i_cta);
9226 case NVPTX::BI__nvvm_atom_sys_min_gen_i:
9227 case NVPTX::BI__nvvm_atom_sys_min_gen_ui:
9228 case NVPTX::BI__nvvm_atom_sys_min_gen_l:
9229 case NVPTX::BI__nvvm_atom_sys_min_gen_ul:
9230 case NVPTX::BI__nvvm_atom_sys_min_gen_ll:
9231 case NVPTX::BI__nvvm_atom_sys_min_gen_ull:
9232 return MakeScopedAtomic(Intrinsic::nvvm_atomic_min_gen_i_sys);
9233 case NVPTX::BI__nvvm_atom_cta_inc_gen_ui:
9234 return MakeScopedAtomic(Intrinsic::nvvm_atomic_inc_gen_i_cta);
9235 case NVPTX::BI__nvvm_atom_cta_dec_gen_ui:
9236 return MakeScopedAtomic(Intrinsic::nvvm_atomic_dec_gen_i_cta);
9237 case NVPTX::BI__nvvm_atom_sys_inc_gen_ui:
9238 return MakeScopedAtomic(Intrinsic::nvvm_atomic_inc_gen_i_sys);
9239 case NVPTX::BI__nvvm_atom_sys_dec_gen_ui:
9240 return MakeScopedAtomic(Intrinsic::nvvm_atomic_dec_gen_i_sys);
9241 case NVPTX::BI__nvvm_atom_cta_and_gen_i:
9242 case NVPTX::BI__nvvm_atom_cta_and_gen_l:
9243 case NVPTX::BI__nvvm_atom_cta_and_gen_ll:
9244 return MakeScopedAtomic(Intrinsic::nvvm_atomic_and_gen_i_cta);
9245 case NVPTX::BI__nvvm_atom_sys_and_gen_i:
9246 case NVPTX::BI__nvvm_atom_sys_and_gen_l:
9247 case NVPTX::BI__nvvm_atom_sys_and_gen_ll:
9248 return MakeScopedAtomic(Intrinsic::nvvm_atomic_and_gen_i_sys);
9249 case NVPTX::BI__nvvm_atom_cta_or_gen_i:
9250 case NVPTX::BI__nvvm_atom_cta_or_gen_l:
9251 case NVPTX::BI__nvvm_atom_cta_or_gen_ll:
9252 return MakeScopedAtomic(Intrinsic::nvvm_atomic_or_gen_i_cta);
9253 case NVPTX::BI__nvvm_atom_sys_or_gen_i:
9254 case NVPTX::BI__nvvm_atom_sys_or_gen_l:
9255 case NVPTX::BI__nvvm_atom_sys_or_gen_ll:
9256 return MakeScopedAtomic(Intrinsic::nvvm_atomic_or_gen_i_sys);
9257 case NVPTX::BI__nvvm_atom_cta_xor_gen_i:
9258 case NVPTX::BI__nvvm_atom_cta_xor_gen_l:
9259 case NVPTX::BI__nvvm_atom_cta_xor_gen_ll:
9260 return MakeScopedAtomic(Intrinsic::nvvm_atomic_xor_gen_i_cta);
9261 case NVPTX::BI__nvvm_atom_sys_xor_gen_i:
9262 case NVPTX::BI__nvvm_atom_sys_xor_gen_l:
9263 case NVPTX::BI__nvvm_atom_sys_xor_gen_ll:
9264 return MakeScopedAtomic(Intrinsic::nvvm_atomic_xor_gen_i_sys);
9265 case NVPTX::BI__nvvm_atom_cta_cas_gen_i:
9266 case NVPTX::BI__nvvm_atom_cta_cas_gen_l:
9267 case NVPTX::BI__nvvm_atom_cta_cas_gen_ll: {
9271 Intrinsic::nvvm_atomic_cas_gen_i_cta,
9272 {Ptr->getType()->getPointerElementType(), Ptr->getType()}),
9275 case NVPTX::BI__nvvm_atom_sys_cas_gen_i:
9276 case NVPTX::BI__nvvm_atom_sys_cas_gen_l:
9277 case NVPTX::BI__nvvm_atom_sys_cas_gen_ll: {
9281 Intrinsic::nvvm_atomic_cas_gen_i_sys,
9282 {Ptr->getType()->getPointerElementType(), Ptr->getType()}),
9292 switch (BuiltinID) {
9293 case WebAssembly::BI__builtin_wasm_current_memory: {
9296 return Builder.CreateCall(Callee);
9298 case WebAssembly::BI__builtin_wasm_grow_memory: {
9301 return Builder.CreateCall(Callee, X);
9303 case WebAssembly::BI__builtin_wasm_throw: {
9307 return Builder.CreateCall(Callee, {Tag, Obj});
9309 case WebAssembly::BI__builtin_wasm_rethrow: {
9311 return Builder.CreateCall(Callee);
unsigned getAddressSpace() const
Return the address space of this type.
ReturnValueSlot - Contains the address where the return value of a function can be stored...
Defines the clang::ASTContext interface.
llvm::StoreInst * CreateDefaultAlignedStore(llvm::Value *Val, llvm::Value *Addr, bool IsVolatile=false)
static Value * emitFPIntBuiltin(CodeGenFunction &CGF, const CallExpr *E, unsigned IntrinsicID)
FunctionDecl - An instance of this class is created to represent a function declaration or definition...
llvm::Value * EmitAArch64CompareBuiltinExpr(llvm::Value *Op, llvm::Type *Ty, const llvm::CmpInst::Predicate Fp, const llvm::CmpInst::Predicate Ip, const llvm::Twine &Name="")
PointerType - C99 6.7.5.1 - Pointer Declarators.
A (possibly-)qualified type.
#define fma(__x, __y, __z)
CodeGenTypes & getTypes()
static WidthAndSignedness getIntegerWidthAndSignedness(const clang::ASTContext &context, const clang::QualType Type)
Expr * getArg(unsigned Arg)
getArg - Return the specified argument.
static Value * emitBinaryBuiltin(CodeGenFunction &CGF, const CallExpr *E, unsigned IntrinsicID)
llvm::Module & getModule() const
static struct WidthAndSignedness EncompassingIntegerType(ArrayRef< struct WidthAndSignedness > Types)
llvm::Value * EmitNVPTXBuiltinExpr(unsigned BuiltinID, const CallExpr *E)
llvm::LLVMContext & getLLVMContext()
const TargetInfo & getTarget() const
CanQualType getSizeType() const
Return the unique type for "size_t" (C99 7.17), defined in <stddef.h>.
static const Builtin::Info BuiltinInfo[]
QuantityType getQuantity() const
getQuantity - Get the raw integer representation of this quantity.
static Value * EmitX86MaskedCompare(CodeGenFunction &CGF, unsigned CC, bool Signed, SmallVectorImpl< Value * > &Ops)
static Value * getMaskVecValue(CodeGenFunction &CGF, Value *Mask, unsigned NumElts)
llvm::Value * getTypeSize(QualType Ty)
Returns calculated size of the specified type.
llvm::Type * FloatTy
float, double
llvm::Value * EmitMSVCBuiltinExpr(MSVCIntrin BuiltinID, const CallExpr *E)
static Value * EmitToInt(CodeGenFunction &CGF, llvm::Value *V, QualType T, llvm::IntegerType *IntType)
Emit the conversions required to turn the given value into an integer of the given size...
const llvm::DataLayout & getDataLayout() const
const void * Store
Store - This opaque type encapsulates an immutable mapping from locations to values.
static Value * EmitSpecialRegisterBuiltin(CodeGenFunction &CGF, const CallExpr *E, llvm::Type *RegisterType, llvm::Type *ValueType, bool IsRead, StringRef SysReg="")
The base class of the type hierarchy.
bool isLittleEndian() const
bool isBooleanType() const
#define NEONMAP1(NameBase, LLVMIntrinsic, TypeModifier)
bool isBlockPointerType() const
llvm::IntegerType * Int8Ty
i8, i16, i32, and i64
bool HasSideEffects(const ASTContext &Ctx, bool IncludePossibleEffects=true) const
HasSideEffects - This routine returns true for all those expressions which have any effect other than...
llvm::Type * getElementType() const
Return the type of the values stored in this address.
const Expr * getCallee() const
static Value * EmitSystemZIntrinsicWithCC(CodeGenFunction &CGF, unsigned IntrinsicID, const CallExpr *E)
Handle a SystemZ function in which the final argument is a pointer to an int that receives the post-i...
uint64_t getTypeSize(QualType T) const
Return the size of the specified (complete) type T, in bits.
static int64_t clamp(int64_t Value, int64_t Low, int64_t High)
llvm::CallInst * EmitTrapCall(llvm::Intrinsic::ID IntrID)
Emit a call to trap or debugtrap and attach function attribute "trap-func-name" if specified...
llvm::Value * EmitARMBuiltinExpr(unsigned BuiltinID, const CallExpr *E)
ParmVarDecl - Represents a parameter to a function.
static bool HasExtraNeonArgument(unsigned BuiltinID)
Return true if BuiltinID is an overloaded Neon intrinsic with an extra argument that specifies the ve...
The collection of all-type qualifiers we support.
Expr * IgnoreImpCasts() LLVM_READONLY
IgnoreImpCasts - Skip past any implicit casts which might surround this expression.
llvm::Value * EmitCommonNeonBuiltinExpr(unsigned BuiltinID, unsigned LLVMIntrinsic, unsigned AltLLVMIntrinsic, const char *NameHint, unsigned Modifier, const CallExpr *E, SmallVectorImpl< llvm::Value * > &Ops, Address PtrOp0, Address PtrOp1)
void __ovld prefetch(const __global char *p, size_t num_elements)
Prefetch num_elements * sizeof(gentype) bytes into the global cache.
CodeGenFunction - This class organizes the per-function state that is used while generating LLVM code...
llvm::Type * ConvertType(QualType T)
ConvertType - Convert type T into a llvm::Type.
static llvm::VectorType * GetNeonType(CodeGenFunction *CGF, NeonTypeFlags TypeFlags, bool V1Ty=false)
Holds long-lived AST nodes (such as types and decls) that can be referred to throughout the semantic ...
llvm::IntegerType * Int64Ty
llvm::IntegerType * SizeTy
static bool NEONSIMDIntrinsicsProvenSorted
static RValue EmitBinaryAtomicPost(CodeGenFunction &CGF, llvm::AtomicRMWInst::BinOp Kind, const CallExpr *E, Instruction::BinaryOps Op, bool Invert=false)
Utility to insert an atomic instruction based Instrinsic::ID and the expression node, where the return value is the result of the operation.
bool hasSameUnqualifiedType(QualType T1, QualType T2) const
Determine whether the given types are equivalent after cvr-qualifiers have been removed.
static const NeonIntrinsicInfo AArch64SIMDIntrinsicMap[]
static Value * MakeBinaryAtomicValue(CodeGenFunction &CGF, llvm::AtomicRMWInst::BinOp Kind, const CallExpr *E)
Utility to insert an atomic instruction based on Instrinsic::ID and the expression node...
llvm::Value * EmitLoadOfScalar(Address Addr, bool Volatile, QualType Ty, SourceLocation Loc, LValueBaseInfo BaseInfo=LValueBaseInfo(AlignmentSource::Type), llvm::MDNode *TBAAInfo=nullptr, QualType TBAABaseTy=QualType(), uint64_t TBAAOffset=0, bool isNontemporal=false)
EmitLoadOfScalar - Load a scalar value from an address, taking care to appropriately convert from the...
llvm::Value * BuildVector(ArrayRef< llvm::Value * > Ops)
CharUnits getNaturalPointeeTypeAlignment(QualType T, LValueBaseInfo *BaseInfo=nullptr)
Address CreateElementBitCast(Address Addr, llvm::Type *Ty, const llvm::Twine &Name="")
Cast the element type of the given address to a different type, preserving information like the align...
CharUnits - This is an opaque type for sizes expressed in character units.
static const NeonIntrinsicInfo AArch64SISDIntrinsicMap[]
APValue Val
Val - This is the value the expression can be folded to.
#define INTRINSIC_WITH_CC(NAME)
QualType GetBuiltinType(unsigned ID, GetBuiltinTypeError &Error, unsigned *IntegerConstantArgs=nullptr) const
Return the type for the specified builtin.
llvm::PointerType * VoidPtrTy
static bool AArch64SISDIntrinsicsProvenSorted
Expr * IgnoreParenCasts() LLVM_READONLY
IgnoreParenCasts - Ignore parentheses and casts.
llvm::Constant * CreateRuntimeVariable(llvm::Type *Ty, StringRef Name)
Create a new runtime global variable with the specified type and name.
CharUnits getTypeSizeInChars(QualType T) const
Return the size of the specified (complete) type T, in characters.
RValue EmitBuiltinExpr(const FunctionDecl *FD, unsigned BuiltinID, const CallExpr *E, ReturnValueSlot ReturnValue)
bool computeOSLogBufferLayout(clang::ASTContext &Ctx, const clang::CallExpr *E, OSLogBufferLayout &layout)
detail::InMemoryDirectory::const_iterator I
llvm::Constant * CreateRuntimeFunction(llvm::FunctionType *Ty, StringRef Name, llvm::AttributeList ExtraAttrs=llvm::AttributeList(), bool Local=false)
Create a new runtime function with the specified type and name.
std::pair< llvm::Value *, llvm::Value * > ComplexPairTy
Represents a prototype with parameter type info, e.g.
llvm::CallInst * EmitNounwindRuntimeCall(llvm::Value *callee, const Twine &name="")
#define NEONMAP0(NameBase)
void EmitAnyExprToMem(const Expr *E, Address Location, Qualifiers Quals, bool IsInitializer)
EmitAnyExprToMem - Emits the code necessary to evaluate an arbitrary expression into the given memory...
bool EvaluateAsRValue(EvalResult &Result, const ASTContext &Ctx) const
EvaluateAsRValue - Return true if this is a constant which we can fold to an rvalue using any crazy t...
RValue - This trivial value class is used to represent the result of an expression that is evaluated...
QualType getPointeeType() const
If this is a pointer, ObjC object pointer, or block pointer, this returns the respective pointee...
llvm::Value * EmitVAStartEnd(llvm::Value *ArgValue, bool IsStart)
Emits a call to an LLVM variable-argument intrinsic, either llvm.va_start or llvm.va_end.
Exposes information about the current target.
llvm::Value * getPointer() const
#define copysign(__x, __y)
Expr - This represents one expression.
const char * getName(unsigned ID) const
Return the identifier name for the specified builtin, e.g.
static RValue emitLibraryCall(CodeGenFunction &CGF, const FunctionDecl *FD, const CallExpr *E, llvm::Constant *calleeValue)
static CGCallee forDirect(llvm::Constant *functionPtr, const CGCalleeInfo &abstractInfo=CGCalleeInfo())
#define INTRINSIC_X86_XSAVE_ID(NAME)
static Value * EmitX86Select(CodeGenFunction &CGF, Value *Mask, Value *Op0, Value *Op1)
ASTContext & getContext() const
llvm::Value * EmitToMemory(llvm::Value *Value, QualType Ty)
EmitToMemory - Change a scalar value from its value representation to its in-memory representation...
static Value * emitUnaryBuiltin(CodeGenFunction &CGF, const CallExpr *E, unsigned IntrinsicID)
static CharUnits fromQuantity(QuantityType Quantity)
fromQuantity - Construct a CharUnits quantity from a raw integer type.
void add(RValue rvalue, QualType type, bool needscopy=false)
static Value * EmitFromInt(CodeGenFunction &CGF, llvm::Value *V, QualType T, llvm::Type *ResultType)
void EmitStoreOfScalar(llvm::Value *Value, Address Addr, bool Volatile, QualType Ty, LValueBaseInfo BaseInfo=LValueBaseInfo(AlignmentSource::Type), llvm::MDNode *TBAAInfo=nullptr, bool isInit=false, QualType TBAABaseTy=QualType(), uint64_t TBAAOffset=0, bool isNontemporal=false)
EmitStoreOfScalar - Store a scalar value to an address, taking care to appropriately convert from the...
char __ovld __cnfn min(char x, char y)
Returns y if y < x, otherwise it returns x.
static SVal getValue(SVal val, SValBuilder &svalBuilder)
llvm::LLVMContext & getLLVMContext()
static Value * EmitX86MaskedStore(CodeGenFunction &CGF, SmallVectorImpl< Value * > &Ops, unsigned Align)
llvm::IntegerType * Int32Ty
static Value * emitRangedBuiltin(CodeGenFunction &CGF, unsigned IntrinsicID, int low, int high)
static const NeonIntrinsicInfo ARMSIMDIntrinsicMap[]
bool EvaluateAsInt(llvm::APSInt &Result, const ASTContext &Ctx, SideEffectsKind AllowSideEffects=SE_NoSideEffects) const
EvaluateAsInt - Return true if this is a constant which we can fold and convert to an integer...
LValue MakeNaturalAlignAddrLValue(llvm::Value *V, QualType T)
Address EmitPointerWithAlignment(const Expr *Addr, LValueBaseInfo *BaseInfo=nullptr)
EmitPointerWithAlignment - Given an expression with a pointer type, emit the value and compute our be...
static bool areBOSTypesCompatible(int From, int To)
Checks if using the result of __builtin_object_size(p, From) in place of __builtin_object_size(p, To) is correct.
llvm::CallInst * CreateMemSet(Address Dest, llvm::Value *Value, llvm::Value *Size, bool IsVolatile=false)
llvm::Function * getIntrinsic(unsigned IID, ArrayRef< llvm::Type * > Tys=None)
static const NeonIntrinsicInfo * findNeonIntrinsicInMap(ArrayRef< NeonIntrinsicInfo > IntrinsicMap, unsigned BuiltinID, bool &MapProvenSorted)
static Value * EmitNontemporalLoad(CodeGenFunction &CGF, const CallExpr *E)
llvm::Value * EmitX86BuiltinExpr(unsigned BuiltinID, const CallExpr *E)
GlobalDecl - represents a global declaration.
static Value * EmitX86MaskedLoad(CodeGenFunction &CGF, SmallVectorImpl< Value * > &Ops, unsigned Align)
llvm::Value * EmitAMDGPUBuiltinExpr(unsigned BuiltinID, const CallExpr *E)
The l-value was considered opaque, so the alignment was determined from a type.
llvm::Value * EmitPPCBuiltinExpr(unsigned BuiltinID, const CallExpr *E)
static llvm::Value * EmitOverflowIntrinsic(CodeGenFunction &CGF, const llvm::Intrinsic::ID IntrinsicID, llvm::Value *X, llvm::Value *Y, llvm::Value *&Carry)
Emit a call to llvm.
Enumerates target-specific builtins in their own namespaces within namespace clang.
Address CreateBitCast(Address Addr, llvm::Type *Ty, const llvm::Twine &Name="")
TypeInfo getTypeInfo(const Type *T) const
Get the size and alignment of the specified complete type in bits.
CharUnits getTypeAlignInChars(QualType T) const
Return the ABI-specified alignment of a (complete) type T, in characters.
static Value * packTBLDVectorList(CodeGenFunction &CGF, ArrayRef< Value * > Ops, Value *ExtOp, Value *IndexOp, llvm::Type *ResTy, unsigned IntID, const char *Name)
ASTContext & getContext() const
void EmitARCIntrinsicUse(ArrayRef< llvm::Value * > values)
Given a number of pointers, inform the optimizer that they're being intrinsically used up until this ...
llvm::Value * EmitFromMemory(llvm::Value *Value, QualType Ty)
EmitFromMemory - Change a scalar value from its memory representation to its value representation...
bool hasSideEffects() const
static llvm::VectorType * GetFloatNeonType(CodeGenFunction *CGF, NeonTypeFlags IntTypeFlags)
static Value * EmitNontemporalStore(CodeGenFunction &CGF, const CallExpr *E)
llvm::IntegerType * Int16Ty
llvm::APSInt EvaluateKnownConstInt(const ASTContext &Ctx, SmallVectorImpl< PartialDiagnosticAt > *Diag=nullptr) const
EvaluateKnownConstInt - Call EvaluateAsRValue and return the folded integer.
llvm::Value * EmitAArch64BuiltinExpr(unsigned BuiltinID, const CallExpr *E)
static Value * MakeAtomicCmpXchgValue(CodeGenFunction &CGF, const CallExpr *E, bool ReturnBool)
Utility to insert an atomic cmpxchg instruction.
OpenMPLinearClauseKind Modifier
Modifier of 'linear' clause.
bool isIntegerConstantExpr(llvm::APSInt &Result, const ASTContext &Ctx, SourceLocation *Loc=nullptr, bool isEvaluated=true) const
isIntegerConstantExpr - Return true if this expression is a valid integer constant expression...
llvm::Value * EmitNeonCall(llvm::Function *F, SmallVectorImpl< llvm::Value * > &O, const char *name, unsigned shift=0, bool rightshift=false)
EltType getEltType() const
const T * castAs() const
Member-template castAs<specific type>.
All available information about a concrete callee.
bool operator<(DeclarationName LHS, DeclarationName RHS)
Ordering on two declaration names.
static Value * EmitX86SExtMask(CodeGenFunction &CGF, Value *Op, llvm::Type *DstTy)
bool isVolatileQualified() const
Determine whether this type is volatile-qualified.
char __ovld __cnfn rotate(char v, char i)
For each element in v, the bits are shifted left by the number of bits given by the corresponding ele...
SourceLocation getExprLoc() const LLVM_READONLY
getExprLoc - Return the preferred location for the arrow when diagnosing a problem with a generic exp...
static Value * EmitFAbs(CodeGenFunction &CGF, Value *V)
EmitFAbs - Emit a call to .fabs().
llvm::Value * EmitScalarExpr(const Expr *E, bool IgnoreResultAssign=false)
EmitScalarExpr - Emit the computation of the specified expression of LLVM scalar type, returning the result.
QualType getCallReturnType(const ASTContext &Ctx) const
getCallReturnType - Get the return type of the call expr.
ast_type_traits::DynTypedNode Node
CGFunctionInfo - Class to encapsulate the information about a function definition.
llvm::Value * EmitNeonRShiftImm(llvm::Value *Vec, llvm::Value *Amt, llvm::Type *Ty, bool usgn, const char *name)
CharUnits getAlignment() const
Return the alignment of this pointer.
This class organizes the cross-function state that is used while generating LLVM code.
#define NEONMAP2(NameBase, LLVMIntrinsic, AltLLVMIntrinsic, TypeModifier)
bool tryEvaluateObjectSize(uint64_t &Result, ASTContext &Ctx, unsigned Type) const
If the current Expr is a pointer, this will try to statically determine the number of bytes available...
static Value * EmitX86SubVectorBroadcast(CodeGenFunction &CGF, SmallVectorImpl< Value * > &Ops, llvm::Type *DstTy, unsigned SrcSizeInBits, unsigned Align)
static RValue getComplex(llvm::Value *V1, llvm::Value *V2)
EvalResult is a struct with detailed info about an evaluated expression.
FunctionDecl * getDirectCallee()
If the callee is a FunctionDecl, return it. Otherwise return 0.
llvm::Value * EmitWebAssemblyBuiltinExpr(unsigned BuiltinID, const CallExpr *E)
llvm::LoadInst * CreateAlignedLoad(llvm::Value *Addr, CharUnits Align, const llvm::Twine &Name="")
const TargetInfo * getAuxTargetInfo() const
llvm::LoadInst * CreateLoad(Address Addr, const llvm::Twine &Name="")
static Value * EmitTargetArchBuiltinExpr(CodeGenFunction *CGF, unsigned BuiltinID, const CallExpr *E, llvm::Triple::ArchType Arch)
static Value * EmitX86MinMax(CodeGenFunction &CGF, ICmpInst::Predicate Pred, ArrayRef< Value * > Ops)
const llvm::Triple & getTriple() const
Returns the target triple of the primary target.
detail::InMemoryDirectory::const_iterator E
llvm::StoreInst * CreateStore(llvm::Value *Val, Address Addr, bool IsVolatile=false)
unsigned getNumArgs() const
getNumArgs - Return the number of actual arguments to this call.
Flags to identify the types for overloaded Neon builtins.
Expr * IgnoreParenImpCasts() LLVM_READONLY
IgnoreParenImpCasts - Ignore parentheses and implicit casts.
llvm::Value * EmitNeonSplat(llvm::Value *V, llvm::Constant *Idx)
static Value * EmitCommonNeonSISDBuiltinExpr(CodeGenFunction &CGF, const NeonIntrinsicInfo &SISDInfo, SmallVectorImpl< Value * > &Ops, const CallExpr *E)
llvm::PointerType * getType() const
Return the type of the pointer value.
static Value * EmitAArch64TblBuiltinExpr(CodeGenFunction &CGF, unsigned BuiltinID, const CallExpr *E, SmallVectorImpl< Value * > &Ops)
bool isLibFunction(unsigned ID) const
Return true if this is a builtin for a libc/libm function, with a "__builtin_" prefix (e...
llvm::Value * vectorWrapScalar16(llvm::Value *Op)
static llvm::Value * getDefaultBuiltinObjectSizeResult(unsigned Type, llvm::IntegerType *ResType)
llvm::Value * EmitNeonShiftVector(llvm::Value *V, llvm::Type *Ty, bool negateForRightShift)
llvm::PointerType * Int8PtrTy
void setNontemporal(bool Value)
char __ovld __cnfn max(char x, char y)
Returns y if x < y, otherwise it returns x.
X
Add a minimal nested name specifier fixit hint to allow lookup of a tag name from an outer enclosing ...
BoundNodesTreeBuilder *const Builder
llvm::Function * LookupNeonLLVMIntrinsic(unsigned IntrinsicID, unsigned Modifier, llvm::Type *ArgTy, const CallExpr *E)
llvm::Type * ConvertType(QualType T)
Builtin::Context & BuiltinInfo
llvm::Constant * getBuiltinLibFunction(const FunctionDecl *FD, unsigned BuiltinID)
Given a builtin id for a function like "__builtin_fabsf", return a Function* for "fabsf".
Defines the clang::TargetInfo interface.
CallExpr - Represents a function call (C99 6.5.2.2, C++ [expr.call]).
static Value * emitTernaryBuiltin(CodeGenFunction &CGF, const CallExpr *E, unsigned IntrinsicID)
static Value * EmitSignBit(CodeGenFunction &CGF, Value *V)
Emit the computation of the sign bit for a floating point value.
static RValue get(llvm::Value *V)
llvm::StoreInst * CreateAlignedStore(llvm::Value *Val, llvm::Value *Addr, CharUnits Align, bool IsVolatile=false)
LValue - This represents an lvalue references.
Information for lazily generating a cleanup.
bool isSignedIntegerType() const
Return true if this is an integer type that is signed, according to C99 6.2.5p4 [char, signed char, short, int, long..], or an enum decl which has a signed representation.
SourceLocation getLocStart() const LLVM_READONLY
CallArgList - Type for representing both the value and type of arguments in a call.
Address CreateMemTemp(QualType T, const Twine &Name="tmp", bool CastToDefaultAddrSpace=true)
CreateMemTemp - Create a temporary memory object of the given type, with appropriate alignment...
static RValue EmitBinaryAtomic(CodeGenFunction &CGF, llvm::AtomicRMWInst::BinOp Kind, const CallExpr *E)
RValue EmitCall(const CGFunctionInfo &CallInfo, const CGCallee &Callee, ReturnValueSlot ReturnValue, const CallArgList &Args, llvm::Instruction **callOrInvoke=nullptr)
EmitCall - Generate a call of the given function, expecting the given result type, and using the given argument list which specifies both the LLVM arguments and the types they were derived from.
bool isIntegerType() const
isIntegerType() does not include complex integers (a GCC extension).
static bool AArch64SIMDIntrinsicsProvenSorted
llvm::Value * EmitSystemZBuiltinExpr(unsigned BuiltinID, const CallExpr *E)
bool isPointerType() const