LLVM API Documentation

AutoUpgrade.cpp
Go to the documentation of this file.
00001 //===-- AutoUpgrade.cpp - Implement auto-upgrade helper functions ---------===//
00002 //
00003 //                     The LLVM Compiler Infrastructure
00004 //
00005 // This file is distributed under the University of Illinois Open Source
00006 // License. See LICENSE.TXT for details.
00007 //
00008 //===----------------------------------------------------------------------===//
00009 //
00010 // This file implements the auto-upgrade helper functions
00011 //
00012 //===----------------------------------------------------------------------===//
00013 
00014 #include "llvm/IR/AutoUpgrade.h"
00015 #include "llvm/IR/CFG.h"
00016 #include "llvm/IR/CallSite.h"
00017 #include "llvm/IR/Constants.h"
00018 #include "llvm/IR/DIBuilder.h"
00019 #include "llvm/IR/DebugInfo.h"
00020 #include "llvm/IR/DiagnosticInfo.h"
00021 #include "llvm/IR/Function.h"
00022 #include "llvm/IR/IRBuilder.h"
00023 #include "llvm/IR/Instruction.h"
00024 #include "llvm/IR/IntrinsicInst.h"
00025 #include "llvm/IR/LLVMContext.h"
00026 #include "llvm/IR/Module.h"
00027 #include "llvm/Support/ErrorHandling.h"
00028 #include <cstring>
00029 using namespace llvm;
00030 
00031 // Upgrade the declarations of the SSE4.1 functions whose arguments have
00032 // changed their type from v4f32 to v2i64.
00033 static bool UpgradeSSE41Function(Function* F, Intrinsic::ID IID,
00034                                  Function *&NewFn) {
00035   // Check whether this is an old version of the function, which received
00036   // v4f32 arguments.
00037   Type *Arg0Type = F->getFunctionType()->getParamType(0);
00038   if (Arg0Type != VectorType::get(Type::getFloatTy(F->getContext()), 4))
00039     return false;
00040 
00041   // Yes, it's old, replace it with new version.
00042   F->setName(F->getName() + ".old");
00043   NewFn = Intrinsic::getDeclaration(F->getParent(), IID);
00044   return true;
00045 }
00046 
00047 // Upgrade the declarations of intrinsic functions whose 8-bit immediate mask
00048 // arguments have changed their type from i32 to i8.
00049 static bool UpgradeX86IntrinsicsWith8BitMask(Function *F, Intrinsic::ID IID,
00050                                              Function *&NewFn) {
00051   // Check that the last argument is an i32.
00052   Type *LastArgType = F->getFunctionType()->getParamType(
00053      F->getFunctionType()->getNumParams() - 1);
00054   if (!LastArgType->isIntegerTy(32))
00055     return false;
00056 
00057   // Move this function aside and map down.
00058   F->setName(F->getName() + ".old");
00059   NewFn = Intrinsic::getDeclaration(F->getParent(), IID);
00060   return true;
00061 }
00062 
00063 // Upgrade the declarations of AVX-512 cmp intrinsic functions whose 8-bit
00064 // immediates have changed their type from i32 to i8.
00065 static bool UpgradeAVX512CmpIntrinsic(Function *F, Intrinsic::ID IID,
00066                                       Function *&NewFn) {
00067   // Check that the last argument is an i32.
00068   Type *LastArgType = F->getFunctionType()->getParamType(2);
00069   if (!LastArgType->isIntegerTy(32))
00070     return false;
00071 
00072   // Move this function aside and map down.
00073   F->setName(F->getName() + ".old");
00074   NewFn = Intrinsic::getDeclaration(F->getParent(), IID);
00075   return true;
00076 }
00077 
00078 static bool UpgradeIntrinsicFunction1(Function *F, Function *&NewFn) {
00079   assert(F && "Illegal to upgrade a non-existent Function.");
00080 
00081   // Quickly eliminate it, if it's not a candidate.
00082   StringRef Name = F->getName();
00083   if (Name.size() <= 8 || !Name.startswith("llvm."))
00084     return false;
00085   Name = Name.substr(5); // Strip off "llvm."
00086 
00087   switch (Name[0]) {
00088   default: break;
00089   case 'a': {
00090     if (Name.startswith("arm.neon.vclz")) {
00091       Type* args[2] = {
00092         F->arg_begin()->getType(),
00093         Type::getInt1Ty(F->getContext())
00094       };
00095       // Can't use Intrinsic::getDeclaration here as it adds a ".i1" to
00096       // the end of the name. Change name from llvm.arm.neon.vclz.* to
00097       //  llvm.ctlz.*
00098       FunctionType* fType = FunctionType::get(F->getReturnType(), args, false);
00099       NewFn = Function::Create(fType, F->getLinkage(),
00100                                "llvm.ctlz." + Name.substr(14), F->getParent());
00101       return true;
00102     }
00103     if (Name.startswith("arm.neon.vcnt")) {
00104       NewFn = Intrinsic::getDeclaration(F->getParent(), Intrinsic::ctpop,
00105                                         F->arg_begin()->getType());
00106       return true;
00107     }
00108     break;
00109   }
00110   case 'c': {
00111     if (Name.startswith("ctlz.") && F->arg_size() == 1) {
00112       F->setName(Name + ".old");
00113       NewFn = Intrinsic::getDeclaration(F->getParent(), Intrinsic::ctlz,
00114                                         F->arg_begin()->getType());
00115       return true;
00116     }
00117     if (Name.startswith("cttz.") && F->arg_size() == 1) {
00118       F->setName(Name + ".old");
00119       NewFn = Intrinsic::getDeclaration(F->getParent(), Intrinsic::cttz,
00120                                         F->arg_begin()->getType());
00121       return true;
00122     }
00123     break;
00124   }
00125   case 'd': {
00126     if (Name.startswith("dbg.declare") && F->arg_size() == 2) {
00127       F->setName(Name + ".old");
00128       NewFn = Intrinsic::getDeclaration(F->getParent(), Intrinsic::dbg_declare);
00129       return true;
00130     }
00131     if (Name.startswith("dbg.value") && F->arg_size() == 3) {
00132       F->setName(Name + ".old");
00133       NewFn = Intrinsic::getDeclaration(F->getParent(), Intrinsic::dbg_value);
00134       return true;
00135     }
00136     break;
00137   }
00138 
00139   case 'o':
00140     // We only need to change the name to match the mangling including the
00141     // address space.
00142     if (F->arg_size() == 2 && Name.startswith("objectsize.")) {
00143       Type *Tys[2] = { F->getReturnType(), F->arg_begin()->getType() };
00144       if (F->getName() != Intrinsic::getName(Intrinsic::objectsize, Tys)) {
00145         F->setName(Name + ".old");
00146         NewFn = Intrinsic::getDeclaration(F->getParent(),
00147                                           Intrinsic::objectsize, Tys);
00148         return true;
00149       }
00150     }
00151     break;
00152 
00153   case 'x': {
00154     if (Name.startswith("x86.sse2.pcmpeq.") ||
00155         Name.startswith("x86.sse2.pcmpgt.") ||
00156         Name.startswith("x86.avx2.pcmpeq.") ||
00157         Name.startswith("x86.avx2.pcmpgt.") ||
00158         Name.startswith("x86.avx.vpermil.") ||
00159         Name == "x86.avx.movnt.dq.256" ||
00160         Name == "x86.avx.movnt.pd.256" ||
00161         Name == "x86.avx.movnt.ps.256" ||
00162         Name == "x86.sse42.crc32.64.8" ||
00163         Name == "x86.avx.vbroadcast.ss" ||
00164         Name == "x86.avx.vbroadcast.ss.256" ||
00165         Name == "x86.avx.vbroadcast.sd.256" ||
00166         Name == "x86.sse2.psll.dq" ||
00167         Name == "x86.sse2.psrl.dq" ||
00168         Name == "x86.avx2.psll.dq" ||
00169         Name == "x86.avx2.psrl.dq" ||
00170         Name == "x86.sse2.psll.dq.bs" ||
00171         Name == "x86.sse2.psrl.dq.bs" ||
00172         Name == "x86.avx2.psll.dq.bs" ||
00173         Name == "x86.avx2.psrl.dq.bs" ||
00174         (Name.startswith("x86.xop.vpcom") && F->arg_size() == 2)) {
00175       NewFn = nullptr;
00176       return true;
00177     }
00178     // SSE4.1 ptest functions may have an old signature.
00179     if (Name.startswith("x86.sse41.ptest")) {
00180       if (Name == "x86.sse41.ptestc")
00181         return UpgradeSSE41Function(F, Intrinsic::x86_sse41_ptestc, NewFn);
00182       if (Name == "x86.sse41.ptestz")
00183         return UpgradeSSE41Function(F, Intrinsic::x86_sse41_ptestz, NewFn);
00184       if (Name == "x86.sse41.ptestnzc")
00185         return UpgradeSSE41Function(F, Intrinsic::x86_sse41_ptestnzc, NewFn);
00186     }
00187     // Several blend and other instructions with maskes used the wrong number of
00188     // bits.
00189     if (Name == "x86.sse41.pblendw")
00190       return UpgradeX86IntrinsicsWith8BitMask(F, Intrinsic::x86_sse41_pblendw,
00191                                               NewFn);
00192     if (Name == "x86.sse41.blendpd")
00193       return UpgradeX86IntrinsicsWith8BitMask(F, Intrinsic::x86_sse41_blendpd,
00194                                               NewFn);
00195     if (Name == "x86.sse41.blendps")
00196       return UpgradeX86IntrinsicsWith8BitMask(F, Intrinsic::x86_sse41_blendps,
00197                                               NewFn);
00198     if (Name == "x86.sse41.insertps")
00199       return UpgradeX86IntrinsicsWith8BitMask(F, Intrinsic::x86_sse41_insertps,
00200                                               NewFn);
00201     if (Name == "x86.sse41.dppd")
00202       return UpgradeX86IntrinsicsWith8BitMask(F, Intrinsic::x86_sse41_dppd,
00203                                               NewFn);
00204     if (Name == "x86.sse41.dpps")
00205       return UpgradeX86IntrinsicsWith8BitMask(F, Intrinsic::x86_sse41_dpps,
00206                                               NewFn);
00207     if (Name == "x86.sse41.mpsadbw")
00208       return UpgradeX86IntrinsicsWith8BitMask(F, Intrinsic::x86_sse41_mpsadbw,
00209                                               NewFn);
00210     if (Name == "x86.avx.blend.pd.256")
00211       return UpgradeX86IntrinsicsWith8BitMask(
00212           F, Intrinsic::x86_avx_blend_pd_256, NewFn);
00213     if (Name == "x86.avx.blend.ps.256")
00214       return UpgradeX86IntrinsicsWith8BitMask(
00215           F, Intrinsic::x86_avx_blend_ps_256, NewFn);
00216     if (Name == "x86.avx.dp.ps.256")
00217       return UpgradeX86IntrinsicsWith8BitMask(F, Intrinsic::x86_avx_dp_ps_256,
00218                                               NewFn);
00219     if (Name == "x86.avx2.pblendw")
00220       return UpgradeX86IntrinsicsWith8BitMask(F, Intrinsic::x86_avx2_pblendw,
00221                                               NewFn);
00222     if (Name == "x86.avx2.pblendd.128")
00223       return UpgradeX86IntrinsicsWith8BitMask(
00224           F, Intrinsic::x86_avx2_pblendd_128, NewFn);
00225     if (Name == "x86.avx2.pblendd.256")
00226       return UpgradeX86IntrinsicsWith8BitMask(
00227           F, Intrinsic::x86_avx2_pblendd_256, NewFn);
00228     if (Name == "x86.avx2.mpsadbw")
00229       return UpgradeX86IntrinsicsWith8BitMask(F, Intrinsic::x86_avx2_mpsadbw,
00230                                               NewFn);
00231 
00232     if (Name == "x86.avx512.mask.cmp.ps.512")
00233       return UpgradeAVX512CmpIntrinsic(F, Intrinsic::x86_avx512_mask_cmp_ps_512,
00234                                        NewFn);
00235     if (Name == "x86.avx512.mask.cmp.pd.512")
00236       return UpgradeAVX512CmpIntrinsic(F, Intrinsic::x86_avx512_mask_cmp_pd_512,
00237                                        NewFn);
00238 
00239     if (Name == "x86.avx512.mask.cmp.b.512")
00240       return UpgradeAVX512CmpIntrinsic(F, Intrinsic::x86_avx512_mask_cmp_b_512,
00241                                        NewFn);
00242     if (Name == "x86.avx512.mask.cmp.w.512")
00243       return UpgradeAVX512CmpIntrinsic(F, Intrinsic::x86_avx512_mask_cmp_w_512,
00244                                        NewFn);
00245     if (Name == "x86.avx512.mask.cmp.d.512")
00246       return UpgradeAVX512CmpIntrinsic(F, Intrinsic::x86_avx512_mask_cmp_d_512,
00247                                        NewFn);
00248     if (Name == "x86.avx512.mask.cmp.q.512")
00249       return UpgradeAVX512CmpIntrinsic(F, Intrinsic::x86_avx512_mask_cmp_q_512,
00250                                        NewFn);
00251     if (Name == "x86.avx512.mask.ucmp.b.512")
00252       return UpgradeAVX512CmpIntrinsic(F, Intrinsic::x86_avx512_mask_ucmp_b_512,
00253                                        NewFn);
00254     if (Name == "x86.avx512.mask.ucmp.w.512")
00255       return UpgradeAVX512CmpIntrinsic(F, Intrinsic::x86_avx512_mask_ucmp_w_512,
00256                                        NewFn);
00257     if (Name == "x86.avx512.mask.ucmp.d.512")
00258       return UpgradeAVX512CmpIntrinsic(F, Intrinsic::x86_avx512_mask_ucmp_d_512,
00259                                        NewFn);
00260     if (Name == "x86.avx512.mask.ucmp.q.512")
00261       return UpgradeAVX512CmpIntrinsic(F, Intrinsic::x86_avx512_mask_ucmp_q_512,
00262                                        NewFn);
00263 
00264     if (Name == "x86.avx512.mask.cmp.b.256")
00265       return UpgradeAVX512CmpIntrinsic(F, Intrinsic::x86_avx512_mask_cmp_b_256,
00266                                        NewFn);
00267     if (Name == "x86.avx512.mask.cmp.w.256")
00268       return UpgradeAVX512CmpIntrinsic(F, Intrinsic::x86_avx512_mask_cmp_w_256,
00269                                        NewFn);
00270     if (Name == "x86.avx512.mask.cmp.d.256")
00271       return UpgradeAVX512CmpIntrinsic(F, Intrinsic::x86_avx512_mask_cmp_d_256,
00272                                        NewFn);
00273     if (Name == "x86.avx512.mask.cmp.q.256")
00274       return UpgradeAVX512CmpIntrinsic(F, Intrinsic::x86_avx512_mask_cmp_q_256,
00275                                        NewFn);
00276     if (Name == "x86.avx512.mask.ucmp.b.256")
00277       return UpgradeAVX512CmpIntrinsic(F, Intrinsic::x86_avx512_mask_ucmp_b_256,
00278                                        NewFn);
00279     if (Name == "x86.avx512.mask.ucmp.w.256")
00280       return UpgradeAVX512CmpIntrinsic(F, Intrinsic::x86_avx512_mask_ucmp_w_256,
00281                                        NewFn);
00282     if (Name == "x86.avx512.mask.ucmp.d.256")
00283       return UpgradeAVX512CmpIntrinsic(F, Intrinsic::x86_avx512_mask_ucmp_d_256,
00284                                        NewFn);
00285     if (Name == "x86.avx512.mask.ucmp.q.256")
00286       return UpgradeAVX512CmpIntrinsic(F, Intrinsic::x86_avx512_mask_ucmp_q_256,
00287                                        NewFn);
00288 
00289     if (Name == "x86.avx512.mask.cmp.b.128")
00290       return UpgradeAVX512CmpIntrinsic(F, Intrinsic::x86_avx512_mask_cmp_b_128,
00291                                        NewFn);
00292     if (Name == "x86.avx512.mask.cmp.w.128")
00293       return UpgradeAVX512CmpIntrinsic(F, Intrinsic::x86_avx512_mask_cmp_w_128,
00294                                        NewFn);
00295     if (Name == "x86.avx512.mask.cmp.d.128")
00296       return UpgradeAVX512CmpIntrinsic(F, Intrinsic::x86_avx512_mask_cmp_d_128,
00297                                        NewFn);
00298     if (Name == "x86.avx512.mask.cmp.q.128")
00299       return UpgradeAVX512CmpIntrinsic(F, Intrinsic::x86_avx512_mask_cmp_q_128,
00300                                        NewFn);
00301     if (Name == "x86.avx512.mask.ucmp.b.128")
00302       return UpgradeAVX512CmpIntrinsic(F, Intrinsic::x86_avx512_mask_ucmp_b_128,
00303                                        NewFn);
00304     if (Name == "x86.avx512.mask.ucmp.w.128")
00305       return UpgradeAVX512CmpIntrinsic(F, Intrinsic::x86_avx512_mask_ucmp_w_128,
00306                                        NewFn);
00307     if (Name == "x86.avx512.mask.ucmp.d.128")
00308       return UpgradeAVX512CmpIntrinsic(F, Intrinsic::x86_avx512_mask_ucmp_d_128,
00309                                        NewFn);
00310     if (Name == "x86.avx512.mask.ucmp.q.128")
00311       return UpgradeAVX512CmpIntrinsic(F, Intrinsic::x86_avx512_mask_ucmp_q_128,
00312                                        NewFn);
00313 
00314     // frcz.ss/sd may need to have an argument dropped
00315     if (Name.startswith("x86.xop.vfrcz.ss") && F->arg_size() == 2) {
00316       F->setName(Name + ".old");
00317       NewFn = Intrinsic::getDeclaration(F->getParent(),
00318                                         Intrinsic::x86_xop_vfrcz_ss);
00319       return true;
00320     }
00321     if (Name.startswith("x86.xop.vfrcz.sd") && F->arg_size() == 2) {
00322       F->setName(Name + ".old");
00323       NewFn = Intrinsic::getDeclaration(F->getParent(),
00324                                         Intrinsic::x86_xop_vfrcz_sd);
00325       return true;
00326     }
00327     // Fix the FMA4 intrinsics to remove the 4
00328     if (Name.startswith("x86.fma4.")) {
00329       F->setName("llvm.x86.fma" + Name.substr(8));
00330       NewFn = F;
00331       return true;
00332     }
00333     break;
00334   }
00335   }
00336 
00337   //  This may not belong here. This function is effectively being overloaded
00338   //  to both detect an intrinsic which needs upgrading, and to provide the
00339   //  upgraded form of the intrinsic. We should perhaps have two separate
00340   //  functions for this.
00341   return false;
00342 }
00343 
00344 bool llvm::UpgradeIntrinsicFunction(Function *F, Function *&NewFn) {
00345   NewFn = nullptr;
00346   bool Upgraded = UpgradeIntrinsicFunction1(F, NewFn);
00347 
00348   // Upgrade intrinsic attributes.  This does not change the function.
00349   if (NewFn)
00350     F = NewFn;
00351   if (unsigned id = F->getIntrinsicID())
00352     F->setAttributes(Intrinsic::getAttributes(F->getContext(),
00353                                               (Intrinsic::ID)id));
00354   return Upgraded;
00355 }
00356 
00357 bool llvm::UpgradeGlobalVariable(GlobalVariable *GV) {
00358   // Nothing to do yet.
00359   return false;
00360 }
00361 
00362 static MDNode *getNodeField(const MDNode *DbgNode, unsigned Elt) {
00363   if (!DbgNode || Elt >= DbgNode->getNumOperands())
00364     return nullptr;
00365   return dyn_cast_or_null<MDNode>(DbgNode->getOperand(Elt));
00366 }
00367 
00368 static MetadataAsValue *getExpression(Value *VarOperand, Function *F) {
00369   // Old-style DIVariables have an optional expression as the 8th element.
00370   DIExpression Expr(getNodeField(
00371       cast<MDNode>(cast<MetadataAsValue>(VarOperand)->getMetadata()), 8));
00372   if (!Expr) {
00373     DIBuilder DIB(*F->getParent(), /*AllowUnresolved*/ false);
00374     Expr = DIB.createExpression();
00375   }
00376   return MetadataAsValue::get(F->getContext(), Expr);
00377 }
00378 
00379 // Handles upgrading SSE2 and AVX2 PSLLDQ intrinsics by converting them
00380 // to byte shuffles.
00381 static Value *UpgradeX86PSLLDQIntrinsics(IRBuilder<> &Builder, LLVMContext &C,
00382                                          Value *Op, unsigned NumLanes,
00383                                          unsigned Shift) {
00384   // Each lane is 16 bytes.
00385   unsigned NumElts = NumLanes * 16;
00386 
00387   // Bitcast from a 64-bit element type to a byte element type.
00388   Op = Builder.CreateBitCast(Op,
00389                              VectorType::get(Type::getInt8Ty(C), NumElts),
00390                              "cast");
00391   // We'll be shuffling in zeroes.
00392   Value *Res = ConstantVector::getSplat(NumElts, Builder.getInt8(0));
00393 
00394   // If shift is less than 16, emit a shuffle to move the bytes. Otherwise,
00395   // we'll just return the zero vector.
00396   if (Shift < 16) {
00397     SmallVector<Constant*, 32> Idxs;
00398     // 256-bit version is split into two 16-byte lanes.
00399     for (unsigned l = 0; l != NumElts; l += 16)
00400       for (unsigned i = 0; i != 16; ++i) {
00401         unsigned Idx = NumElts + i - Shift;
00402         if (Idx < NumElts)
00403           Idx -= NumElts - 16; // end of lane, switch operand.
00404         Idxs.push_back(Builder.getInt32(Idx + l));
00405       }
00406 
00407     Res = Builder.CreateShuffleVector(Res, Op, ConstantVector::get(Idxs));
00408   }
00409 
00410   // Bitcast back to a 64-bit element type.
00411   return Builder.CreateBitCast(Res,
00412                                VectorType::get(Type::getInt64Ty(C), 2*NumLanes),
00413                                "cast");
00414 }
00415 
00416 // Handles upgrading SSE2 and AVX2 PSRLDQ intrinsics by converting them
00417 // to byte shuffles.
00418 static Value *UpgradeX86PSRLDQIntrinsics(IRBuilder<> &Builder, LLVMContext &C,
00419                                          Value *Op, unsigned NumLanes,
00420                                          unsigned Shift) {
00421   // Each lane is 16 bytes.
00422   unsigned NumElts = NumLanes * 16;
00423 
00424   // Bitcast from a 64-bit element type to a byte element type.
00425   Op = Builder.CreateBitCast(Op,
00426                              VectorType::get(Type::getInt8Ty(C), NumElts),
00427                              "cast");
00428   // We'll be shuffling in zeroes.
00429   Value *Res = ConstantVector::getSplat(NumElts, Builder.getInt8(0));
00430 
00431   // If shift is less than 16, emit a shuffle to move the bytes. Otherwise,
00432   // we'll just return the zero vector.
00433   if (Shift < 16) {
00434     SmallVector<Constant*, 32> Idxs;
00435     // 256-bit version is split into two 16-byte lanes.
00436     for (unsigned l = 0; l != NumElts; l += 16)
00437       for (unsigned i = 0; i != 16; ++i) {
00438         unsigned Idx = i + Shift;
00439         if (Idx >= 16)
00440           Idx += NumElts - 16; // end of lane, switch operand.
00441         Idxs.push_back(Builder.getInt32(Idx + l));
00442       }
00443 
00444     Res = Builder.CreateShuffleVector(Op, Res, ConstantVector::get(Idxs));
00445   }
00446 
00447   // Bitcast back to a 64-bit element type.
00448   return Builder.CreateBitCast(Res,
00449                                VectorType::get(Type::getInt64Ty(C), 2*NumLanes),
00450                                "cast");
00451 }
00452 
00453 // UpgradeIntrinsicCall - Upgrade a call to an old intrinsic to be a call the
00454 // upgraded intrinsic. All argument and return casting must be provided in
00455 // order to seamlessly integrate with existing context.
00456 void llvm::UpgradeIntrinsicCall(CallInst *CI, Function *NewFn) {
00457   Function *F = CI->getCalledFunction();
00458   LLVMContext &C = CI->getContext();
00459   IRBuilder<> Builder(C);
00460   Builder.SetInsertPoint(CI->getParent(), CI);
00461 
00462   assert(F && "Intrinsic call is not direct?");
00463 
00464   if (!NewFn) {
00465     // Get the Function's name.
00466     StringRef Name = F->getName();
00467 
00468     Value *Rep;
00469     // Upgrade packed integer vector compares intrinsics to compare instructions
00470     if (Name.startswith("llvm.x86.sse2.pcmpeq.") ||
00471         Name.startswith("llvm.x86.avx2.pcmpeq.")) {
00472       Rep = Builder.CreateICmpEQ(CI->getArgOperand(0), CI->getArgOperand(1),
00473                                  "pcmpeq");
00474       // need to sign extend since icmp returns vector of i1
00475       Rep = Builder.CreateSExt(Rep, CI->getType(), "");
00476     } else if (Name.startswith("llvm.x86.sse2.pcmpgt.") ||
00477                Name.startswith("llvm.x86.avx2.pcmpgt.")) {
00478       Rep = Builder.CreateICmpSGT(CI->getArgOperand(0), CI->getArgOperand(1),
00479                                   "pcmpgt");
00480       // need to sign extend since icmp returns vector of i1
00481       Rep = Builder.CreateSExt(Rep, CI->getType(), "");
00482     } else if (Name == "llvm.x86.avx.movnt.dq.256" ||
00483                Name == "llvm.x86.avx.movnt.ps.256" ||
00484                Name == "llvm.x86.avx.movnt.pd.256") {
00485       IRBuilder<> Builder(C);
00486       Builder.SetInsertPoint(CI->getParent(), CI);
00487 
00488       Module *M = F->getParent();
00489       SmallVector<Metadata *, 1> Elts;
00490       Elts.push_back(
00491           ConstantAsMetadata::get(ConstantInt::get(Type::getInt32Ty(C), 1)));
00492       MDNode *Node = MDNode::get(C, Elts);
00493 
00494       Value *Arg0 = CI->getArgOperand(0);
00495       Value *Arg1 = CI->getArgOperand(1);
00496 
00497       // Convert the type of the pointer to a pointer to the stored type.
00498       Value *BC = Builder.CreateBitCast(Arg0,
00499                                         PointerType::getUnqual(Arg1->getType()),
00500                                         "cast");
00501       StoreInst *SI = Builder.CreateStore(Arg1, BC);
00502       SI->setMetadata(M->getMDKindID("nontemporal"), Node);
00503       SI->setAlignment(16);
00504 
00505       // Remove intrinsic.
00506       CI->eraseFromParent();
00507       return;
00508     } else if (Name.startswith("llvm.x86.xop.vpcom")) {
00509       Intrinsic::ID intID;
00510       if (Name.endswith("ub"))
00511         intID = Intrinsic::x86_xop_vpcomub;
00512       else if (Name.endswith("uw"))
00513         intID = Intrinsic::x86_xop_vpcomuw;
00514       else if (Name.endswith("ud"))
00515         intID = Intrinsic::x86_xop_vpcomud;
00516       else if (Name.endswith("uq"))
00517         intID = Intrinsic::x86_xop_vpcomuq;
00518       else if (Name.endswith("b"))
00519         intID = Intrinsic::x86_xop_vpcomb;
00520       else if (Name.endswith("w"))
00521         intID = Intrinsic::x86_xop_vpcomw;
00522       else if (Name.endswith("d"))
00523         intID = Intrinsic::x86_xop_vpcomd;
00524       else if (Name.endswith("q"))
00525         intID = Intrinsic::x86_xop_vpcomq;
00526       else
00527         llvm_unreachable("Unknown suffix");
00528 
00529       Name = Name.substr(18); // strip off "llvm.x86.xop.vpcom"
00530       unsigned Imm;
00531       if (Name.startswith("lt"))
00532         Imm = 0;
00533       else if (Name.startswith("le"))
00534         Imm = 1;
00535       else if (Name.startswith("gt"))
00536         Imm = 2;
00537       else if (Name.startswith("ge"))
00538         Imm = 3;
00539       else if (Name.startswith("eq"))
00540         Imm = 4;
00541       else if (Name.startswith("ne"))
00542         Imm = 5;
00543       else if (Name.startswith("false"))
00544         Imm = 6;
00545       else if (Name.startswith("true"))
00546         Imm = 7;
00547       else
00548         llvm_unreachable("Unknown condition");
00549 
00550       Function *VPCOM = Intrinsic::getDeclaration(F->getParent(), intID);
00551       Rep = Builder.CreateCall3(VPCOM, CI->getArgOperand(0),
00552                                 CI->getArgOperand(1), Builder.getInt8(Imm));
00553     } else if (Name == "llvm.x86.sse42.crc32.64.8") {
00554       Function *CRC32 = Intrinsic::getDeclaration(F->getParent(),
00555                                                Intrinsic::x86_sse42_crc32_32_8);
00556       Value *Trunc0 = Builder.CreateTrunc(CI->getArgOperand(0), Type::getInt32Ty(C));
00557       Rep = Builder.CreateCall2(CRC32, Trunc0, CI->getArgOperand(1));
00558       Rep = Builder.CreateZExt(Rep, CI->getType(), "");
00559     } else if (Name.startswith("llvm.x86.avx.vbroadcast")) {
00560       // Replace broadcasts with a series of insertelements.
00561       Type *VecTy = CI->getType();
00562       Type *EltTy = VecTy->getVectorElementType();
00563       unsigned EltNum = VecTy->getVectorNumElements();
00564       Value *Cast = Builder.CreateBitCast(CI->getArgOperand(0),
00565                                           EltTy->getPointerTo());
00566       Value *Load = Builder.CreateLoad(Cast);
00567       Type *I32Ty = Type::getInt32Ty(C);
00568       Rep = UndefValue::get(VecTy);
00569       for (unsigned I = 0; I < EltNum; ++I)
00570         Rep = Builder.CreateInsertElement(Rep, Load,
00571                                           ConstantInt::get(I32Ty, I));
00572     } else if (Name == "llvm.x86.sse2.psll.dq") {
00573       // 128-bit shift left specified in bits.
00574       unsigned Shift = cast<ConstantInt>(CI->getArgOperand(1))->getZExtValue();
00575       Rep = UpgradeX86PSLLDQIntrinsics(Builder, C, CI->getArgOperand(0), 1,
00576                                        Shift / 8); // Shift is in bits.
00577     } else if (Name == "llvm.x86.sse2.psrl.dq") {
00578       // 128-bit shift right specified in bits.
00579       unsigned Shift = cast<ConstantInt>(CI->getArgOperand(1))->getZExtValue();
00580       Rep = UpgradeX86PSRLDQIntrinsics(Builder, C, CI->getArgOperand(0), 1,
00581                                        Shift / 8); // Shift is in bits.
00582     } else if (Name == "llvm.x86.avx2.psll.dq") {
00583       // 256-bit shift left specified in bits.
00584       unsigned Shift = cast<ConstantInt>(CI->getArgOperand(1))->getZExtValue();
00585       Rep = UpgradeX86PSLLDQIntrinsics(Builder, C, CI->getArgOperand(0), 2,
00586                                        Shift / 8); // Shift is in bits.
00587     } else if (Name == "llvm.x86.avx2.psrl.dq") {
00588       // 256-bit shift right specified in bits.
00589       unsigned Shift = cast<ConstantInt>(CI->getArgOperand(1))->getZExtValue();
00590       Rep = UpgradeX86PSRLDQIntrinsics(Builder, C, CI->getArgOperand(0), 2,
00591                                        Shift / 8); // Shift is in bits.
00592     } else if (Name == "llvm.x86.sse2.psll.dq.bs") {
00593       // 128-bit shift left specified in bytes.
00594       unsigned Shift = cast<ConstantInt>(CI->getArgOperand(1))->getZExtValue();
00595       Rep = UpgradeX86PSLLDQIntrinsics(Builder, C, CI->getArgOperand(0), 1,
00596                                        Shift);
00597     } else if (Name == "llvm.x86.sse2.psrl.dq.bs") {
00598       // 128-bit shift right specified in bytes.
00599       unsigned Shift = cast<ConstantInt>(CI->getArgOperand(1))->getZExtValue();
00600       Rep = UpgradeX86PSRLDQIntrinsics(Builder, C, CI->getArgOperand(0), 1,
00601                                        Shift);
00602     } else if (Name == "llvm.x86.avx2.psll.dq.bs") {
00603       // 256-bit shift left specified in bytes.
00604       unsigned Shift = cast<ConstantInt>(CI->getArgOperand(1))->getZExtValue();
00605       Rep = UpgradeX86PSLLDQIntrinsics(Builder, C, CI->getArgOperand(0), 2,
00606                                        Shift);
00607     } else if (Name == "llvm.x86.avx2.psrl.dq.bs") {
00608       // 256-bit shift right specified in bytes.
00609       unsigned Shift = cast<ConstantInt>(CI->getArgOperand(1))->getZExtValue();
00610       Rep = UpgradeX86PSRLDQIntrinsics(Builder, C, CI->getArgOperand(0), 2,
00611                                        Shift);
00612     } else {
00613       bool PD128 = false, PD256 = false, PS128 = false, PS256 = false;
00614       if (Name == "llvm.x86.avx.vpermil.pd.256")
00615         PD256 = true;
00616       else if (Name == "llvm.x86.avx.vpermil.pd")
00617         PD128 = true;
00618       else if (Name == "llvm.x86.avx.vpermil.ps.256")
00619         PS256 = true;
00620       else if (Name == "llvm.x86.avx.vpermil.ps")
00621         PS128 = true;
00622 
00623       if (PD256 || PD128 || PS256 || PS128) {
00624         Value *Op0 = CI->getArgOperand(0);
00625         unsigned Imm = cast<ConstantInt>(CI->getArgOperand(1))->getZExtValue();
00626         SmallVector<Constant*, 8> Idxs;
00627 
00628         if (PD128)
00629           for (unsigned i = 0; i != 2; ++i)
00630             Idxs.push_back(Builder.getInt32((Imm >> i) & 0x1));
00631         else if (PD256)
00632           for (unsigned l = 0; l != 4; l+=2)
00633             for (unsigned i = 0; i != 2; ++i)
00634               Idxs.push_back(Builder.getInt32(((Imm >> (l+i)) & 0x1) + l));
00635         else if (PS128)
00636           for (unsigned i = 0; i != 4; ++i)
00637             Idxs.push_back(Builder.getInt32((Imm >> (2 * i)) & 0x3));
00638         else if (PS256)
00639           for (unsigned l = 0; l != 8; l+=4)
00640             for (unsigned i = 0; i != 4; ++i)
00641               Idxs.push_back(Builder.getInt32(((Imm >> (2 * i)) & 0x3) + l));
00642         else
00643           llvm_unreachable("Unexpected function");
00644 
00645         Rep = Builder.CreateShuffleVector(Op0, Op0, ConstantVector::get(Idxs));
00646       } else {
00647         llvm_unreachable("Unknown function for CallInst upgrade.");
00648       }
00649     }
00650 
00651     CI->replaceAllUsesWith(Rep);
00652     CI->eraseFromParent();
00653     return;
00654   }
00655 
00656   std::string Name = CI->getName().str();
00657   if (!Name.empty())
00658     CI->setName(Name + ".old");
00659 
00660   switch (NewFn->getIntrinsicID()) {
00661   default:
00662     llvm_unreachable("Unknown function for CallInst upgrade.");
00663 
00664   // Upgrade debug intrinsics to use an additional DIExpression argument.
00665   case Intrinsic::dbg_declare: {
00666     auto NewCI =
00667         Builder.CreateCall3(NewFn, CI->getArgOperand(0), CI->getArgOperand(1),
00668                             getExpression(CI->getArgOperand(1), F), Name);
00669     NewCI->setDebugLoc(CI->getDebugLoc());
00670     CI->replaceAllUsesWith(NewCI);
00671     CI->eraseFromParent();
00672     return;
00673   }
00674   case Intrinsic::dbg_value: {
00675     auto NewCI = Builder.CreateCall4(
00676         NewFn, CI->getArgOperand(0), CI->getArgOperand(1), CI->getArgOperand(2),
00677         getExpression(CI->getArgOperand(2), F), Name);
00678     NewCI->setDebugLoc(CI->getDebugLoc());
00679     CI->replaceAllUsesWith(NewCI);
00680     CI->eraseFromParent();
00681     return;
00682   }
00683   case Intrinsic::ctlz:
00684   case Intrinsic::cttz:
00685     assert(CI->getNumArgOperands() == 1 &&
00686            "Mismatch between function args and call args");
00687     CI->replaceAllUsesWith(Builder.CreateCall2(NewFn, CI->getArgOperand(0),
00688                                                Builder.getFalse(), Name));
00689     CI->eraseFromParent();
00690     return;
00691 
00692   case Intrinsic::objectsize:
00693     CI->replaceAllUsesWith(Builder.CreateCall2(NewFn,
00694                                                CI->getArgOperand(0),
00695                                                CI->getArgOperand(1),
00696                                                Name));
00697     CI->eraseFromParent();
00698     return;
00699 
00700   case Intrinsic::ctpop: {
00701     CI->replaceAllUsesWith(Builder.CreateCall(NewFn, CI->getArgOperand(0)));
00702     CI->eraseFromParent();
00703     return;
00704   }
00705 
00706   case Intrinsic::x86_xop_vfrcz_ss:
00707   case Intrinsic::x86_xop_vfrcz_sd:
00708     CI->replaceAllUsesWith(Builder.CreateCall(NewFn, CI->getArgOperand(1),
00709                                               Name));
00710     CI->eraseFromParent();
00711     return;
00712 
00713   case Intrinsic::x86_sse41_ptestc:
00714   case Intrinsic::x86_sse41_ptestz:
00715   case Intrinsic::x86_sse41_ptestnzc: {
00716     // The arguments for these intrinsics used to be v4f32, and changed
00717     // to v2i64. This is purely a nop, since those are bitwise intrinsics.
00718     // So, the only thing required is a bitcast for both arguments.
00719     // First, check the arguments have the old type.
00720     Value *Arg0 = CI->getArgOperand(0);
00721     if (Arg0->getType() != VectorType::get(Type::getFloatTy(C), 4))
00722       return;
00723 
00724     // Old intrinsic, add bitcasts
00725     Value *Arg1 = CI->getArgOperand(1);
00726 
00727     Value *BC0 =
00728       Builder.CreateBitCast(Arg0,
00729                             VectorType::get(Type::getInt64Ty(C), 2),
00730                             "cast");
00731     Value *BC1 =
00732       Builder.CreateBitCast(Arg1,
00733                             VectorType::get(Type::getInt64Ty(C), 2),
00734                             "cast");
00735 
00736     CallInst* NewCall = Builder.CreateCall2(NewFn, BC0, BC1, Name);
00737     CI->replaceAllUsesWith(NewCall);
00738     CI->eraseFromParent();
00739     return;
00740   }
00741 
00742   case Intrinsic::x86_sse41_pblendw:
00743   case Intrinsic::x86_sse41_blendpd:
00744   case Intrinsic::x86_sse41_blendps:
00745   case Intrinsic::x86_sse41_insertps:
00746   case Intrinsic::x86_sse41_dppd:
00747   case Intrinsic::x86_sse41_dpps:
00748   case Intrinsic::x86_sse41_mpsadbw:
00749   case Intrinsic::x86_avx_blend_pd_256:
00750   case Intrinsic::x86_avx_blend_ps_256:
00751   case Intrinsic::x86_avx_dp_ps_256:
00752   case Intrinsic::x86_avx2_pblendw:
00753   case Intrinsic::x86_avx2_pblendd_128:
00754   case Intrinsic::x86_avx2_pblendd_256:
00755   case Intrinsic::x86_avx2_mpsadbw: {
00756     // Need to truncate the last argument from i32 to i8 -- this argument models
00757     // an inherently 8-bit immediate operand to these x86 instructions.
00758     SmallVector<Value *, 4> Args(CI->arg_operands().begin(),
00759                                  CI->arg_operands().end());
00760 
00761     // Replace the last argument with a trunc.
00762     Args.back() = Builder.CreateTrunc(Args.back(), Type::getInt8Ty(C), "trunc");
00763 
00764     CallInst *NewCall = Builder.CreateCall(NewFn, Args);
00765     CI->replaceAllUsesWith(NewCall);
00766     CI->eraseFromParent();
00767     return;
00768   }
00769   case Intrinsic::x86_avx512_mask_cmp_ps_512:
00770   case Intrinsic::x86_avx512_mask_cmp_pd_512: {
00771     // Need to truncate the last argument from i32 to i8 -- this argument models
00772     // an inherently 8-bit immediate operand to these x86 instructions.
00773     SmallVector<Value *, 5> Args(CI->arg_operands().begin(),
00774                                  CI->arg_operands().end());
00775 
00776     // Replace the last argument with a trunc.
00777     Args[2] = Builder.CreateTrunc(Args[2], Type::getInt8Ty(C), "trunc");
00778 
00779     CallInst *NewCall = Builder.CreateCall(NewFn, Args);
00780     CI->replaceAllUsesWith(NewCall);
00781     CI->eraseFromParent();
00782     return;
00783   }
00784   }
00785 }
00786 
00787 // This tests each Function to determine if it needs upgrading. When we find
00788 // one we are interested in, we then upgrade all calls to reflect the new
00789 // function.
00790 void llvm::UpgradeCallsToIntrinsic(Function* F) {
00791   assert(F && "Illegal attempt to upgrade a non-existent intrinsic.");
00792 
00793   // Upgrade the function and check if it is a totaly new function.
00794   Function *NewFn;
00795   if (UpgradeIntrinsicFunction(F, NewFn)) {
00796     if (NewFn != F) {
00797       // Replace all uses to the old function with the new one if necessary.
00798       for (Value::user_iterator UI = F->user_begin(), UE = F->user_end();
00799            UI != UE; ) {
00800         if (CallInst *CI = dyn_cast<CallInst>(*UI++))
00801           UpgradeIntrinsicCall(CI, NewFn);
00802       }
00803       // Remove old function, no longer used, from the module.
00804       F->eraseFromParent();
00805     }
00806   }
00807 }
00808 
00809 void llvm::UpgradeInstWithTBAATag(Instruction *I) {
00810   MDNode *MD = I->getMetadata(LLVMContext::MD_tbaa);
00811   assert(MD && "UpgradeInstWithTBAATag should have a TBAA tag");
00812   // Check if the tag uses struct-path aware TBAA format.
00813   if (isa<MDNode>(MD->getOperand(0)) && MD->getNumOperands() >= 3)
00814     return;
00815 
00816   if (MD->getNumOperands() == 3) {
00817     Metadata *Elts[] = {MD->getOperand(0), MD->getOperand(1)};
00818     MDNode *ScalarType = MDNode::get(I->getContext(), Elts);
00819     // Create a MDNode <ScalarType, ScalarType, offset 0, const>
00820     Metadata *Elts2[] = {ScalarType, ScalarType,
00821                          ConstantAsMetadata::get(Constant::getNullValue(
00822                              Type::getInt64Ty(I->getContext()))),
00823                          MD->getOperand(2)};
00824     I->setMetadata(LLVMContext::MD_tbaa, MDNode::get(I->getContext(), Elts2));
00825   } else {
00826     // Create a MDNode <MD, MD, offset 0>
00827     Metadata *Elts[] = {MD, MD, ConstantAsMetadata::get(Constant::getNullValue(
00828                                     Type::getInt64Ty(I->getContext())))};
00829     I->setMetadata(LLVMContext::MD_tbaa, MDNode::get(I->getContext(), Elts));
00830   }
00831 }
00832 
00833 Instruction *llvm::UpgradeBitCastInst(unsigned Opc, Value *V, Type *DestTy,
00834                                       Instruction *&Temp) {
00835   if (Opc != Instruction::BitCast)
00836     return nullptr;
00837 
00838   Temp = nullptr;
00839   Type *SrcTy = V->getType();
00840   if (SrcTy->isPtrOrPtrVectorTy() && DestTy->isPtrOrPtrVectorTy() &&
00841       SrcTy->getPointerAddressSpace() != DestTy->getPointerAddressSpace()) {
00842     LLVMContext &Context = V->getContext();
00843 
00844     // We have no information about target data layout, so we assume that
00845     // the maximum pointer size is 64bit.
00846     Type *MidTy = Type::getInt64Ty(Context);
00847     Temp = CastInst::Create(Instruction::PtrToInt, V, MidTy);
00848 
00849     return CastInst::Create(Instruction::IntToPtr, Temp, DestTy);
00850   }
00851 
00852   return nullptr;
00853 }
00854 
00855 Value *llvm::UpgradeBitCastExpr(unsigned Opc, Constant *C, Type *DestTy) {
00856   if (Opc != Instruction::BitCast)
00857     return nullptr;
00858 
00859   Type *SrcTy = C->getType();
00860   if (SrcTy->isPtrOrPtrVectorTy() && DestTy->isPtrOrPtrVectorTy() &&
00861       SrcTy->getPointerAddressSpace() != DestTy->getPointerAddressSpace()) {
00862     LLVMContext &Context = C->getContext();
00863 
00864     // We have no information about target data layout, so we assume that
00865     // the maximum pointer size is 64bit.
00866     Type *MidTy = Type::getInt64Ty(Context);
00867 
00868     return ConstantExpr::getIntToPtr(ConstantExpr::getPtrToInt(C, MidTy),
00869                                      DestTy);
00870   }
00871 
00872   return nullptr;
00873 }
00874 
00875 /// Check the debug info version number, if it is out-dated, drop the debug
00876 /// info. Return true if module is modified.
00877 bool llvm::UpgradeDebugInfo(Module &M) {
00878   unsigned Version = getDebugMetadataVersionFromModule(M);
00879   if (Version == DEBUG_METADATA_VERSION)
00880     return false;
00881 
00882   bool RetCode = StripDebugInfo(M);
00883   if (RetCode) {
00884     DiagnosticInfoDebugMetadataVersion DiagVersion(M, Version);
00885     M.getContext().diagnose(DiagVersion);
00886   }
00887   return RetCode;
00888 }
00889 
00890 void llvm::UpgradeMDStringConstant(std::string &String) {
00891   const std::string OldPrefix = "llvm.vectorizer.";
00892   if (String == "llvm.vectorizer.unroll") {
00893     String = "llvm.loop.interleave.count";
00894   } else if (String.find(OldPrefix) == 0) {
00895     String.replace(0, OldPrefix.size(), "llvm.loop.vectorize.");
00896   }
00897 }