66 assert(F &&
"Illegal to upgrade a non-existent Function.");
121 Intrinsic::objectsize, Tys);
133 Name ==
"x86.avx.vinsertf128.pd.256" ||
134 Name ==
"x86.avx.vinsertf128.ps.256" ||
135 Name ==
"x86.avx.vinsertf128.si.256" ||
136 Name ==
"x86.avx2.vinserti128" ||
137 Name ==
"x86.avx.vextractf128.pd.256" ||
138 Name ==
"x86.avx.vextractf128.ps.256" ||
139 Name ==
"x86.avx.vextractf128.si.256" ||
140 Name ==
"x86.avx2.vextracti128" ||
141 Name ==
"x86.avx.movnt.dq.256" ||
142 Name ==
"x86.avx.movnt.pd.256" ||
143 Name ==
"x86.avx.movnt.ps.256" ||
144 Name ==
"x86.sse42.crc32.64.8" ||
145 Name ==
"x86.avx.vbroadcast.ss" ||
146 Name ==
"x86.avx.vbroadcast.ss.256" ||
147 Name ==
"x86.avx.vbroadcast.sd.256" ||
148 Name ==
"x86.sse2.psll.dq" ||
149 Name ==
"x86.sse2.psrl.dq" ||
150 Name ==
"x86.avx2.psll.dq" ||
151 Name ==
"x86.avx2.psrl.dq" ||
152 Name ==
"x86.sse2.psll.dq.bs" ||
153 Name ==
"x86.sse2.psrl.dq.bs" ||
154 Name ==
"x86.avx2.psll.dq.bs" ||
155 Name ==
"x86.avx2.psrl.dq.bs" ||
156 Name ==
"x86.sse41.pblendw" ||
157 Name ==
"x86.sse41.blendpd" ||
158 Name ==
"x86.sse41.blendps" ||
159 Name ==
"x86.avx.blend.pd.256" ||
160 Name ==
"x86.avx.blend.ps.256" ||
161 Name ==
"x86.avx2.pblendw" ||
162 Name ==
"x86.avx2.pblendd.128" ||
163 Name ==
"x86.avx2.pblendd.256" ||
164 Name ==
"x86.avx2.vbroadcasti128" ||
171 if (Name ==
"x86.sse41.ptestc")
173 if (Name ==
"x86.sse41.ptestz")
175 if (Name ==
"x86.sse41.ptestnzc")
180 if (Name ==
"x86.sse41.insertps")
183 if (Name ==
"x86.sse41.dppd")
186 if (Name ==
"x86.sse41.dpps")
189 if (Name ==
"x86.sse41.mpsadbw")
192 if (Name ==
"x86.avx.dp.ps.256")
195 if (Name ==
"x86.avx2.mpsadbw")
203 Intrinsic::x86_xop_vfrcz_ss);
209 Intrinsic::x86_xop_vfrcz_sd);
232 assert(F != NewFn &&
"Intrinsic function upgraded to the same function");
250 Value *Op,
unsigned NumLanes,
253 unsigned NumElts = NumLanes * 16;
267 for (
unsigned l = 0; l != NumElts; l += 16)
268 for (
unsigned i = 0; i != 16; ++i) {
269 unsigned Idx = NumElts + i - Shift;
287 Value *Op,
unsigned NumLanes,
290 unsigned NumElts = NumLanes * 16;
304 for (
unsigned l = 0; l != NumElts; l += 16)
305 for (
unsigned i = 0; i != 16; ++i) {
306 unsigned Idx = i + Shift;
330 assert(F &&
"Intrinsic call is not direct?");
338 if (Name.
startswith(
"llvm.x86.sse2.pcmpeq.") ||
344 }
else if (Name.
startswith(
"llvm.x86.sse2.pcmpgt.") ||
350 }
else if (Name ==
"llvm.x86.avx.movnt.dq.256" ||
351 Name ==
"llvm.x86.avx.movnt.ps.256" ||
352 Name ==
"llvm.x86.avx.movnt.pd.256") {
376 }
else if (Name.
startswith(
"llvm.x86.xop.vpcom")) {
379 intID = Intrinsic::x86_xop_vpcomub;
381 intID = Intrinsic::x86_xop_vpcomuw;
383 intID = Intrinsic::x86_xop_vpcomud;
385 intID = Intrinsic::x86_xop_vpcomuq;
387 intID = Intrinsic::x86_xop_vpcomb;
389 intID = Intrinsic::x86_xop_vpcomw;
391 intID = Intrinsic::x86_xop_vpcomd;
393 intID = Intrinsic::x86_xop_vpcomq;
422 }
else if (Name ==
"llvm.x86.sse42.crc32.64.8") {
424 Intrinsic::x86_sse42_crc32_32_8);
428 }
else if (Name.
startswith(
"llvm.x86.avx.vbroadcast")) {
438 for (
unsigned I = 0;
I < EltNum; ++
I)
441 }
else if (Name ==
"llvm.x86.avx2.vbroadcasti128") {
447 const int Idxs[4] = { 0, 1, 0, 1 };
450 }
else if (Name ==
"llvm.x86.sse2.psll.dq") {
452 unsigned Shift = cast<ConstantInt>(CI->
getArgOperand(1))->getZExtValue();
455 }
else if (Name ==
"llvm.x86.sse2.psrl.dq") {
457 unsigned Shift = cast<ConstantInt>(CI->
getArgOperand(1))->getZExtValue();
460 }
else if (Name ==
"llvm.x86.avx2.psll.dq") {
462 unsigned Shift = cast<ConstantInt>(CI->
getArgOperand(1))->getZExtValue();
465 }
else if (Name ==
"llvm.x86.avx2.psrl.dq") {
467 unsigned Shift = cast<ConstantInt>(CI->
getArgOperand(1))->getZExtValue();
470 }
else if (Name ==
"llvm.x86.sse2.psll.dq.bs") {
472 unsigned Shift = cast<ConstantInt>(CI->
getArgOperand(1))->getZExtValue();
475 }
else if (Name ==
"llvm.x86.sse2.psrl.dq.bs") {
477 unsigned Shift = cast<ConstantInt>(CI->
getArgOperand(1))->getZExtValue();
480 }
else if (Name ==
"llvm.x86.avx2.psll.dq.bs") {
482 unsigned Shift = cast<ConstantInt>(CI->
getArgOperand(1))->getZExtValue();
485 }
else if (Name ==
"llvm.x86.avx2.psrl.dq.bs") {
487 unsigned Shift = cast<ConstantInt>(CI->
getArgOperand(1))->getZExtValue();
490 }
else if (Name ==
"llvm.x86.sse41.pblendw" ||
491 Name ==
"llvm.x86.sse41.blendpd" ||
492 Name ==
"llvm.x86.sse41.blendps" ||
493 Name ==
"llvm.x86.avx.blend.pd.256" ||
494 Name ==
"llvm.x86.avx.blend.ps.256" ||
495 Name ==
"llvm.x86.avx2.pblendw" ||
496 Name ==
"llvm.x86.avx2.pblendd.128" ||
497 Name ==
"llvm.x86.avx2.pblendd.256") {
500 unsigned Imm = cast <ConstantInt>(CI->
getArgOperand(2))->getZExtValue();
505 for (
unsigned i = 0; i != NumElts; ++i) {
506 unsigned Idx = ((Imm >> (i%8)) & 1) ? i + NumElts : i;
511 }
else if (Name ==
"llvm.x86.avx.vinsertf128.pd.256" ||
512 Name ==
"llvm.x86.avx.vinsertf128.ps.256" ||
513 Name ==
"llvm.x86.avx.vinsertf128.si.256" ||
514 Name ==
"llvm.x86.avx2.vinserti128") {
517 unsigned Imm = cast<ConstantInt>(CI->
getArgOperand(2))->getZExtValue();
527 for (
unsigned i = 0; i != NumElts; ++i) {
546 for (
unsigned i = 0; i != NumElts / 2; ++i) {
547 unsigned Idx = Imm ? i : (i + NumElts);
552 for (
unsigned i = NumElts / 2; i != NumElts; ++i) {
553 unsigned Idx = Imm ? (i + NumElts / 2) : i;
557 }
else if (Name ==
"llvm.x86.avx.vextractf128.pd.256" ||
558 Name ==
"llvm.x86.avx.vextractf128.ps.256" ||
559 Name ==
"llvm.x86.avx.vextractf128.si.256" ||
560 Name ==
"llvm.x86.avx2.vextracti128") {
562 unsigned Imm = cast<ConstantInt>(CI->
getArgOperand(1))->getZExtValue();
571 for (
unsigned i = 0; i != NumElts; ++i) {
572 unsigned Idx = Imm ? (i + NumElts) : i;
579 bool PD128 =
false, PD256 =
false, PS128 =
false, PS256 =
false;
580 if (Name ==
"llvm.x86.avx.vpermil.pd.256")
582 else if (Name ==
"llvm.x86.avx.vpermil.pd")
584 else if (Name ==
"llvm.x86.avx.vpermil.ps.256")
586 else if (Name ==
"llvm.x86.avx.vpermil.ps")
589 if (PD256 || PD128 || PS256 || PS128) {
591 unsigned Imm = cast<ConstantInt>(CI->
getArgOperand(1))->getZExtValue();
595 for (
unsigned i = 0; i != 2; ++i)
598 for (
unsigned l = 0; l != 4; l+=2)
599 for (
unsigned i = 0; i != 2; ++i)
602 for (
unsigned i = 0; i != 4; ++i)
605 for (
unsigned l = 0; l != 8; l+=4)
606 for (
unsigned i = 0; i != 4; ++i)
622 std::string Name = CI->
getName();
630 case Intrinsic::ctlz:
631 case Intrinsic::cttz:
633 "Mismatch between function args and call args");
639 case Intrinsic::objectsize:
642 CI->eraseFromParent();
645 case Intrinsic::ctpop: {
646 CI->replaceAllUsesWith(Builder.CreateCall(NewFn, {CI->getArgOperand(0)}));
647 CI->eraseFromParent();
651 case Intrinsic::x86_xop_vfrcz_ss:
652 case Intrinsic::x86_xop_vfrcz_sd:
653 CI->replaceAllUsesWith(
654 Builder.CreateCall(NewFn, {CI->getArgOperand(1)},
Name));
655 CI->eraseFromParent();
658 case Intrinsic::x86_sse41_ptestc:
659 case Intrinsic::x86_sse41_ptestz:
660 case Intrinsic::x86_sse41_ptestnzc: {
665 Value *Arg0 = CI->getArgOperand(0);
666 if (Arg0->
getType() != VectorType::get(Type::getFloatTy(
C), 4))
670 Value *Arg1 = CI->getArgOperand(1);
672 Type *NewVecTy = VectorType::get(Type::getInt64Ty(
C), 2);
674 Value *BC0 = Builder.CreateBitCast(Arg0, NewVecTy,
"cast");
675 Value *BC1 = Builder.CreateBitCast(Arg1, NewVecTy,
"cast");
677 CallInst *NewCall = Builder.CreateCall(NewFn, {BC0, BC1},
Name);
679 CI->eraseFromParent();
683 case Intrinsic::x86_sse41_insertps:
684 case Intrinsic::x86_sse41_dppd:
685 case Intrinsic::x86_sse41_dpps:
686 case Intrinsic::x86_sse41_mpsadbw:
687 case Intrinsic::x86_avx_dp_ps_256:
688 case Intrinsic::x86_avx2_mpsadbw: {
692 CI->arg_operands().end());
695 Args.
back() = Builder.CreateTrunc(Args.back(), Type::getInt8Ty(
C),
"trunc");
697 CallInst *NewCall = Builder.CreateCall(NewFn, Args);
699 CI->eraseFromParent();
709 assert(F &&
"Illegal attempt to upgrade a non-existent intrinsic.");
717 if (
CallInst *CI = dyn_cast<CallInst>(*UI++))
727 assert(MD &&
"UpgradeInstWithTBAATag should have a TBAA tag");
736 Metadata *Elts2[] = {ScalarType, ScalarType,
737 ConstantAsMetadata::get(Constant::getNullValue(
743 Metadata *Elts[] = {MD, MD, ConstantAsMetadata::get(Constant::getNullValue(
751 if (Opc != Instruction::BitCast)
762 Type *MidTy = Type::getInt64Ty(Context);
763 Temp = CastInst::Create(Instruction::PtrToInt, V, MidTy);
765 return CastInst::Create(Instruction::IntToPtr, Temp, DestTy);
772 if (Opc != Instruction::BitCast)
782 Type *MidTy = Type::getInt64Ty(Context);
784 return ConstantExpr::getIntToPtr(ConstantExpr::getPtrToInt(C, MidTy),
807 const std::string OldPrefix =
"llvm.vectorizer.";
808 if (String ==
"llvm.vectorizer.unroll") {
809 String =
"llvm.loop.interleave.count";
810 }
else if (String.find(OldPrefix) == 0) {
811 String.replace(0, OldPrefix.size(),
"llvm.loop.vectorize.");
static bool UpgradeIntrinsicFunction1(Function *F, Function *&NewFn)
iplist< Instruction >::iterator eraseFromParent()
eraseFromParent - This method unlinks 'this' from the containing basic block and deletes it...
void push_back(const T &Elt)
LinkageTypes getLinkage() const
static IntegerType * getInt1Ty(LLVMContext &C)
LoadInst * CreateLoad(Value *Ptr, const char *Name)
LLVMContext & getContext() const
getContext - Return a reference to the LLVMContext associated with this function. ...
size_t size() const
size - Get the string size.
A Module instance is used to store all the information related to an LLVM module. ...
unsigned getNumParams() const
getNumParams - Return the number of fixed parameters this function type requires. ...
unsigned getNumOperands() const
Return number of MDNode operands.
bool endswith(StringRef Suffix) const
Check if this string ends with the given Suffix.
CallInst - This class represents a function call, abstracting a target machine's calling convention...
void UpgradeMDStringConstant(std::string &String)
Upgrade a metadata string constant in place.
Value * CreatePointerCast(Value *V, Type *DestTy, const Twine &Name="")
StringRef substr(size_t Start, size_t N=npos) const
Return a reference to the substring from [Start, Start + N).
unsigned getDebugMetadataVersionFromModule(const Module &M)
Return Debug Info Metadata Version by checking module flags.
bool isPtrOrPtrVectorTy() const
isPtrOrPtrVectorTy - Return true if this is a pointer type or a vector of pointer types...
Type * getReturnType() const
static IntegerType * getInt64Ty(LLVMContext &C)
Value * CreateICmpEQ(Value *LHS, Value *RHS, const Twine &Name="")
unsigned getPointerAddressSpace() const
Get the address space of this pointer or pointer vector type.
void UpgradeIntrinsicCall(CallInst *CI, Function *NewFn)
This is the complement to the above, replacing a specific call to an intrinsic function with a call t...
Value * CreateShuffleVector(Value *V1, Value *V2, Value *Mask, const Twine &Name="")
Value * CreateInsertElement(Value *Vec, Value *NewElt, Value *Idx, const Twine &Name="")
StringRef getName() const
Return a constant reference to the value's name.
void UpgradeCallsToIntrinsic(Function *F)
This is an auto-upgrade hook for any old intrinsic function syntaxes which need to have both the func...
static Type * getFloatTy(LLVMContext &C)
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
unsigned getNumArgOperands() const
getNumArgOperands - Return the number of call arguments.
static Constant * get(ArrayRef< Constant * > V)
This provides a uniform API for creating instructions and inserting them into a basic block: either a...
void setName(const Twine &Name)
Change the name of the value.
Type * getVectorElementType() const
Value * CreateICmpSGT(Value *LHS, Value *RHS, const Twine &Name="")
user_iterator_impl< User > user_iterator
FunctionType - Class to represent function types.
static Value * UpgradeX86PSRLDQIntrinsics(IRBuilder<> &Builder, LLVMContext &C, Value *Op, unsigned NumLanes, unsigned Shift)
bool StripDebugInfo(Module &M)
Strip debug info in the module if it exists.
static FunctionType * get(Type *Result, ArrayRef< Type * > Params, bool isVarArg)
FunctionType::get - This static method is the primary way of constructing a FunctionType.
bool UpgradeIntrinsicFunction(Function *F, Function *&NewFn)
This is a more granular function that simply checks an intrinsic function for upgrading, and returns true if it requires upgrading.
Function * getDeclaration(Module *M, ID id, ArrayRef< Type * > Tys=None)
Create or insert an LLVM Function declaration for an intrinsic, and return it.
StoreInst - an instruction for storing to memory.
void replaceAllUsesWith(Value *V)
Change all uses of this to point to a new Value.
unsigned getNumElements() const
Return the number of elements in the Vector type.
StoreInst * CreateStore(Value *Val, Value *Ptr, bool isVolatile=false)
void SetInsertPoint(BasicBlock *TheBB)
This specifies that created instructions should be appended to the end of the specified block...
Type * getParamType(unsigned i) const
Parameter type accessors.
The instances of the Type class are immutable: once they are created, they are never changed...
This is an important class for using LLVM in a threaded context.
This is an important base class in LLVM.
This file contains the declarations for the subclasses of Constant, which represent the different fla...
bool UpgradeDebugInfo(Module &M)
Check the debug info version number, if it is out-dated, drop the debug info.
static bool UpgradeX86IntrinsicsWith8BitMask(Function *F, Intrinsic::ID IID, Function *&NewFn)
static UndefValue * get(Type *T)
get() - Static factory methods - Return an 'undef' object of the specified type.
LLVMContext & getContext() const
All values hold a context through their type.
CallInst * CreateCall(Value *Callee, ArrayRef< Value * > Args=None, const Twine &Name="")
PointerType * getPointerTo(unsigned AddrSpace=0)
getPointerTo - Return a pointer to the current type.
void setMetadata(unsigned KindID, MDNode *Node)
setMetadata - Set the metadata of the specified kind to the specified node.
static bool UpgradeSSE41Function(Function *F, Intrinsic::ID IID, Function *&NewFn)
static Constant * getSplat(unsigned NumElts, Constant *Elt)
getSplat - Return a ConstantVector with the specified constant in each element.
const MDOperand & getOperand(unsigned I) const
Instruction * UpgradeBitCastInst(unsigned Opc, Value *V, Type *DestTy, Instruction *&Temp)
This is an auto-upgrade for bitcast between pointers with different address spaces: the instruction i...
bool startswith(StringRef Prefix) const
Check if this string starts with the given Prefix.
static PointerType * getUnqual(Type *ElementType)
PointerType::getUnqual - This constructs a pointer to an object of the specified type in the generic ...
Value * CreateZExt(Value *V, Type *DestTy, const Twine &Name="")
unsigned getVectorNumElements() const
Value * CreateBitCast(Value *V, Type *DestTy, const Twine &Name="")
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small...
Module.h This file contains the declarations for the Module class.
Type * getType() const
All values are typed, get the type of this value.
bool UpgradeGlobalVariable(GlobalVariable *GV)
This checks for global variables which should be upgraded.
MDNode * getMetadata(unsigned KindID) const
getMetadata - Get the metadata of given kind attached to this Instruction.
ConstantInt * getInt32(uint32_t C)
Get a constant 32-bit value.
static Constant * get(Type *Ty, uint64_t V, bool isSigned=false)
If Ty is a vector type, return a Constant with a splat of the given value.
Function * getCalledFunction() const
getCalledFunction - Return the function called, or null if this is an indirect function invocation...
Intrinsic::ID getIntrinsicID() const LLVM_READONLY
getIntrinsicID - This method returns the ID number of the specified function, or Intrinsic::not_intri...
Value * getArgOperand(unsigned i) const
getArgOperand/setArgOperand - Return/set the i-th call argument.
VectorType - Class to represent vector types.
static Value * UpgradeX86PSLLDQIntrinsics(IRBuilder<> &Builder, LLVMContext &C, Value *Op, unsigned NumLanes, unsigned Shift)
bool isIntegerTy() const
isIntegerTy - True if this is an instance of IntegerType.
void eraseFromParent() override
eraseFromParent - This method unlinks 'this' from the containing module and deletes it...
ConstantInt * getFalse()
Get the constant value for i1 false.
static MDTuple * get(LLVMContext &Context, ArrayRef< Metadata * > MDs)
std::string getName(ID id, ArrayRef< Type * > Tys=None)
Return the LLVM name for an intrinsic, such as "llvm.ppc.altivec.lvx".
static IntegerType * getInt32Ty(LLVMContext &C)
Value * CreateSExt(Value *V, Type *DestTy, const Twine &Name="")
FunctionType * getFunctionType() const
void diagnose(const DiagnosticInfo &DI)
Report a message to the currently installed diagnostic handler.
AttributeSet getAttributes(LLVMContext &C, ID id)
Return the attributes for an intrinsic.
void setAttributes(AttributeSet attrs)
Set the attribute list for this Function.
user_iterator user_begin()
Module * getParent()
Get the module that this global value is contained inside of...
Value * CreateTrunc(Value *V, Type *DestTy, const Twine &Name="")
LLVM Value Representation.
void setAlignment(unsigned Align)
static VectorType * get(Type *ElementType, unsigned NumElements)
VectorType::get - This static method is the primary way to construct an VectorType.
void UpgradeInstWithTBAATag(Instruction *I)
If the TBAA tag for the given instruction uses the scalar TBAA format, we upgrade it to the struct-pa...
ConstantInt * getInt8(uint8_t C)
Get a constant 8-bit value.
C - The default llvm calling convention, compatible with C.
StringRef - Represent a constant reference to a string, i.e.
static Function * Create(FunctionType *Ty, LinkageTypes Linkage, const Twine &N="", Module *M=nullptr)
Value * UpgradeBitCastExpr(unsigned Opc, Constant *C, Type *DestTy)
This is an auto-upgrade for bitcast constant expression between pointers with different address space...
unsigned getMDKindID(StringRef Name) const
Return a unique non-zero ID for the specified metadata kind.
static IntegerType * getInt8Ty(LLVMContext &C)
const BasicBlock * getParent() const
LLVMContext & getContext() const
Get the global data context.