LLVM  10.0.0svn
AMDGPULibCalls.cpp
Go to the documentation of this file.
1 //===- AMDGPULibCalls.cpp -------------------------------------------------===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 //
9 /// \file
10 /// This file does AMD library function optimizations.
11 //
12 //===----------------------------------------------------------------------===//
13 
14 #define DEBUG_TYPE "amdgpu-simplifylib"
15 
16 #include "AMDGPU.h"
17 #include "AMDGPULibFunc.h"
18 #include "AMDGPUSubtarget.h"
20 #include "llvm/Analysis/Loads.h"
21 #include "llvm/ADT/StringSet.h"
22 #include "llvm/ADT/StringRef.h"
23 #include "llvm/IR/Constants.h"
24 #include "llvm/IR/DerivedTypes.h"
25 #include "llvm/IR/Instructions.h"
26 #include "llvm/IR/Intrinsics.h"
27 #include "llvm/IR/IRBuilder.h"
28 #include "llvm/IR/Function.h"
29 #include "llvm/IR/LLVMContext.h"
30 #include "llvm/IR/Module.h"
32 #include "llvm/Support/Debug.h"
36 #include <vector>
37 #include <cmath>
38 
39 using namespace llvm;
40 
41 static cl::opt<bool> EnablePreLink("amdgpu-prelink",
42  cl::desc("Enable pre-link mode optimizations"),
43  cl::init(false),
44  cl::Hidden);
45 
46 static cl::list<std::string> UseNative("amdgpu-use-native",
47  cl::desc("Comma separated list of functions to replace with native, or all"),
49  cl::Hidden);
50 
51 #define MATH_PI 3.14159265358979323846264338327950288419716939937511
52 #define MATH_E 2.71828182845904523536028747135266249775724709369996
53 #define MATH_SQRT2 1.41421356237309504880168872420969807856967187537695
54 
55 #define MATH_LOG2E 1.4426950408889634073599246810018921374266459541529859
56 #define MATH_LOG10E 0.4342944819032518276511289189166050822943970058036665
57 // Value of log2(10)
58 #define MATH_LOG2_10 3.3219280948873623478703194294893901758648313930245806
59 // Value of 1 / log2(10)
60 #define MATH_RLOG2_10 0.3010299956639811952137388947244930267681898814621085
61 // Value of 1 / M_LOG2E_F = 1 / log2(e)
62 #define MATH_RLOG2_E 0.6931471805599453094172321214581765680755001343602552
63 
64 namespace llvm {
65 
67 private:
68 
70 
71  const TargetMachine *TM;
72 
73  // -fuse-native.
74  bool AllNative = false;
75 
76  bool useNativeFunc(const StringRef F) const;
77 
78  // Return a pointer (pointer expr) to the function if function defintion with
79  // "FuncName" exists. It may create a new function prototype in pre-link mode.
80  FunctionCallee getFunction(Module *M, const FuncInfo &fInfo);
81 
82  // Replace a normal function with its native version.
83  bool replaceWithNative(CallInst *CI, const FuncInfo &FInfo);
84 
85  bool parseFunctionName(const StringRef& FMangledName,
86  FuncInfo *FInfo=nullptr /*out*/);
87 
88  bool TDOFold(CallInst *CI, const FuncInfo &FInfo);
89 
90  /* Specialized optimizations */
91 
92  // recip (half or native)
93  bool fold_recip(CallInst *CI, IRBuilder<> &B, const FuncInfo &FInfo);
94 
95  // divide (half or native)
96  bool fold_divide(CallInst *CI, IRBuilder<> &B, const FuncInfo &FInfo);
97 
98  // pow/powr/pown
99  bool fold_pow(CallInst *CI, IRBuilder<> &B, const FuncInfo &FInfo);
100 
101  // rootn
102  bool fold_rootn(CallInst *CI, IRBuilder<> &B, const FuncInfo &FInfo);
103 
104  // fma/mad
105  bool fold_fma_mad(CallInst *CI, IRBuilder<> &B, const FuncInfo &FInfo);
106 
107  // -fuse-native for sincos
108  bool sincosUseNative(CallInst *aCI, const FuncInfo &FInfo);
109 
110  // evaluate calls if calls' arguments are constants.
111  bool evaluateScalarMathFunc(FuncInfo &FInfo, double& Res0,
112  double& Res1, Constant *copr0, Constant *copr1, Constant *copr2);
113  bool evaluateCall(CallInst *aCI, FuncInfo &FInfo);
114 
115  // exp
116  bool fold_exp(CallInst *CI, IRBuilder<> &B, const FuncInfo &FInfo);
117 
118  // exp2
119  bool fold_exp2(CallInst *CI, IRBuilder<> &B, const FuncInfo &FInfo);
120 
121  // exp10
122  bool fold_exp10(CallInst *CI, IRBuilder<> &B, const FuncInfo &FInfo);
123 
124  // log
125  bool fold_log(CallInst *CI, IRBuilder<> &B, const FuncInfo &FInfo);
126 
127  // log2
128  bool fold_log2(CallInst *CI, IRBuilder<> &B, const FuncInfo &FInfo);
129 
130  // log10
131  bool fold_log10(CallInst *CI, IRBuilder<> &B, const FuncInfo &FInfo);
132 
133  // sqrt
134  bool fold_sqrt(CallInst *CI, IRBuilder<> &B, const FuncInfo &FInfo);
135 
136  // sin/cos
137  bool fold_sincos(CallInst * CI, IRBuilder<> &B, AliasAnalysis * AA);
138 
139  // __read_pipe/__write_pipe
140  bool fold_read_write_pipe(CallInst *CI, IRBuilder<> &B, FuncInfo &FInfo);
141 
142  // llvm.amdgcn.wavefrontsize
143  bool fold_wavefrontsize(CallInst *CI, IRBuilder<> &B);
144 
145  // Get insertion point at entry.
146  BasicBlock::iterator getEntryIns(CallInst * UI);
147  // Insert an Alloc instruction.
148  AllocaInst* insertAlloca(CallInst * UI, IRBuilder<> &B, const char *prefix);
149  // Get a scalar native builtin signle argument FP function
150  FunctionCallee getNativeFunction(Module *M, const FuncInfo &FInfo);
151 
152 protected:
154 
155  bool isUnsafeMath(const CallInst *CI) const;
156 
157  void replaceCall(Value *With) {
158  CI->replaceAllUsesWith(With);
159  CI->eraseFromParent();
160  }
161 
162 public:
163  AMDGPULibCalls(const TargetMachine *TM_ = nullptr) : TM(TM_) {}
164 
165  bool fold(CallInst *CI, AliasAnalysis *AA = nullptr);
166 
167  void initNativeFuncs();
168 
169  // Replace a normal math function call with that native version
170  bool useNative(CallInst *CI);
171 };
172 
173 } // end llvm namespace
174 
175 namespace {
176 
177  class AMDGPUSimplifyLibCalls : public FunctionPass {
178 
179  const TargetOptions Options;
180 
181  AMDGPULibCalls Simplifier;
182 
183  public:
184  static char ID; // Pass identification
185 
186  AMDGPUSimplifyLibCalls(const TargetOptions &Opt = TargetOptions(),
187  const TargetMachine *TM = nullptr)
188  : FunctionPass(ID), Options(Opt), Simplifier(TM) {
190  }
191 
192  void getAnalysisUsage(AnalysisUsage &AU) const override {
194  }
195 
196  bool runOnFunction(Function &M) override;
197  };
198 
199  class AMDGPUUseNativeCalls : public FunctionPass {
200 
201  AMDGPULibCalls Simplifier;
202 
203  public:
204  static char ID; // Pass identification
205 
206  AMDGPUUseNativeCalls() : FunctionPass(ID) {
208  Simplifier.initNativeFuncs();
209  }
210 
211  bool runOnFunction(Function &F) override;
212  };
213 
214 } // end anonymous namespace.
215 
217 char AMDGPUUseNativeCalls::ID = 0;
218 
219 INITIALIZE_PASS_BEGIN(AMDGPUSimplifyLibCalls, "amdgpu-simplifylib",
220  "Simplify well-known AMD library calls", false, false)
222 INITIALIZE_PASS_END(AMDGPUSimplifyLibCalls, "amdgpu-simplifylib",
223  "Simplify well-known AMD library calls", false, false)
224 
225 INITIALIZE_PASS(AMDGPUUseNativeCalls, "amdgpu-usenative",
226  "Replace builtin math calls with that native versions.",
227  false, false)
228 
229 template <typename IRB>
230 static CallInst *CreateCallEx(IRB &B, FunctionCallee Callee, Value *Arg,
231  const Twine &Name = "") {
232  CallInst *R = B.CreateCall(Callee, Arg, Name);
233  if (Function *F = dyn_cast<Function>(Callee.getCallee()))
234  R->setCallingConv(F->getCallingConv());
235  return R;
236 }
237 
238 template <typename IRB>
239 static CallInst *CreateCallEx2(IRB &B, FunctionCallee Callee, Value *Arg1,
240  Value *Arg2, const Twine &Name = "") {
241  CallInst *R = B.CreateCall(Callee, {Arg1, Arg2}, Name);
242  if (Function *F = dyn_cast<Function>(Callee.getCallee()))
243  R->setCallingConv(F->getCallingConv());
244  return R;
245 }
246 
247 // Data structures for table-driven optimizations.
248 // FuncTbl works for both f32 and f64 functions with 1 input argument
249 
250 struct TableEntry {
251  double result;
252  double input;
253 };
254 
255 /* a list of {result, input} */
256 static const TableEntry tbl_acos[] = {
257  {MATH_PI/2.0, 0.0},
258  {MATH_PI/2.0, -0.0},
259  {0.0, 1.0},
260  {MATH_PI, -1.0}
261 };
262 static const TableEntry tbl_acosh[] = {
263  {0.0, 1.0}
264 };
265 static const TableEntry tbl_acospi[] = {
266  {0.5, 0.0},
267  {0.5, -0.0},
268  {0.0, 1.0},
269  {1.0, -1.0}
270 };
271 static const TableEntry tbl_asin[] = {
272  {0.0, 0.0},
273  {-0.0, -0.0},
274  {MATH_PI/2.0, 1.0},
275  {-MATH_PI/2.0, -1.0}
276 };
277 static const TableEntry tbl_asinh[] = {
278  {0.0, 0.0},
279  {-0.0, -0.0}
280 };
281 static const TableEntry tbl_asinpi[] = {
282  {0.0, 0.0},
283  {-0.0, -0.0},
284  {0.5, 1.0},
285  {-0.5, -1.0}
286 };
287 static const TableEntry tbl_atan[] = {
288  {0.0, 0.0},
289  {-0.0, -0.0},
290  {MATH_PI/4.0, 1.0},
291  {-MATH_PI/4.0, -1.0}
292 };
293 static const TableEntry tbl_atanh[] = {
294  {0.0, 0.0},
295  {-0.0, -0.0}
296 };
297 static const TableEntry tbl_atanpi[] = {
298  {0.0, 0.0},
299  {-0.0, -0.0},
300  {0.25, 1.0},
301  {-0.25, -1.0}
302 };
303 static const TableEntry tbl_cbrt[] = {
304  {0.0, 0.0},
305  {-0.0, -0.0},
306  {1.0, 1.0},
307  {-1.0, -1.0},
308 };
309 static const TableEntry tbl_cos[] = {
310  {1.0, 0.0},
311  {1.0, -0.0}
312 };
313 static const TableEntry tbl_cosh[] = {
314  {1.0, 0.0},
315  {1.0, -0.0}
316 };
317 static const TableEntry tbl_cospi[] = {
318  {1.0, 0.0},
319  {1.0, -0.0}
320 };
321 static const TableEntry tbl_erfc[] = {
322  {1.0, 0.0},
323  {1.0, -0.0}
324 };
325 static const TableEntry tbl_erf[] = {
326  {0.0, 0.0},
327  {-0.0, -0.0}
328 };
329 static const TableEntry tbl_exp[] = {
330  {1.0, 0.0},
331  {1.0, -0.0},
332  {MATH_E, 1.0}
333 };
334 static const TableEntry tbl_exp2[] = {
335  {1.0, 0.0},
336  {1.0, -0.0},
337  {2.0, 1.0}
338 };
339 static const TableEntry tbl_exp10[] = {
340  {1.0, 0.0},
341  {1.0, -0.0},
342  {10.0, 1.0}
343 };
344 static const TableEntry tbl_expm1[] = {
345  {0.0, 0.0},
346  {-0.0, -0.0}
347 };
348 static const TableEntry tbl_log[] = {
349  {0.0, 1.0},
350  {1.0, MATH_E}
351 };
352 static const TableEntry tbl_log2[] = {
353  {0.0, 1.0},
354  {1.0, 2.0}
355 };
356 static const TableEntry tbl_log10[] = {
357  {0.0, 1.0},
358  {1.0, 10.0}
359 };
360 static const TableEntry tbl_rsqrt[] = {
361  {1.0, 1.0},
362  {1.0/MATH_SQRT2, 2.0}
363 };
364 static const TableEntry tbl_sin[] = {
365  {0.0, 0.0},
366  {-0.0, -0.0}
367 };
368 static const TableEntry tbl_sinh[] = {
369  {0.0, 0.0},
370  {-0.0, -0.0}
371 };
372 static const TableEntry tbl_sinpi[] = {
373  {0.0, 0.0},
374  {-0.0, -0.0}
375 };
376 static const TableEntry tbl_sqrt[] = {
377  {0.0, 0.0},
378  {1.0, 1.0},
379  {MATH_SQRT2, 2.0}
380 };
381 static const TableEntry tbl_tan[] = {
382  {0.0, 0.0},
383  {-0.0, -0.0}
384 };
385 static const TableEntry tbl_tanh[] = {
386  {0.0, 0.0},
387  {-0.0, -0.0}
388 };
389 static const TableEntry tbl_tanpi[] = {
390  {0.0, 0.0},
391  {-0.0, -0.0}
392 };
393 static const TableEntry tbl_tgamma[] = {
394  {1.0, 1.0},
395  {1.0, 2.0},
396  {2.0, 3.0},
397  {6.0, 4.0}
398 };
399 
401  switch(id) {
417  return true;
418  default:;
419  }
420  return false;
421 }
422 
423 struct TableRef {
424  size_t size;
425  const TableEntry *table; // variable size: from 0 to (size - 1)
426 
427  TableRef() : size(0), table(nullptr) {}
428 
429  template <size_t N>
430  TableRef(const TableEntry (&tbl)[N]) : size(N), table(&tbl[0]) {}
431 };
432 
434  switch(id) {
435  case AMDGPULibFunc::EI_ACOS: return TableRef(tbl_acos);
436  case AMDGPULibFunc::EI_ACOSH: return TableRef(tbl_acosh);
437  case AMDGPULibFunc::EI_ACOSPI: return TableRef(tbl_acospi);
438  case AMDGPULibFunc::EI_ASIN: return TableRef(tbl_asin);
439  case AMDGPULibFunc::EI_ASINH: return TableRef(tbl_asinh);
440  case AMDGPULibFunc::EI_ASINPI: return TableRef(tbl_asinpi);
441  case AMDGPULibFunc::EI_ATAN: return TableRef(tbl_atan);
442  case AMDGPULibFunc::EI_ATANH: return TableRef(tbl_atanh);
443  case AMDGPULibFunc::EI_ATANPI: return TableRef(tbl_atanpi);
444  case AMDGPULibFunc::EI_CBRT: return TableRef(tbl_cbrt);
446  case AMDGPULibFunc::EI_COS: return TableRef(tbl_cos);
447  case AMDGPULibFunc::EI_COSH: return TableRef(tbl_cosh);
448  case AMDGPULibFunc::EI_COSPI: return TableRef(tbl_cospi);
449  case AMDGPULibFunc::EI_ERFC: return TableRef(tbl_erfc);
450  case AMDGPULibFunc::EI_ERF: return TableRef(tbl_erf);
451  case AMDGPULibFunc::EI_EXP: return TableRef(tbl_exp);
453  case AMDGPULibFunc::EI_EXP2: return TableRef(tbl_exp2);
454  case AMDGPULibFunc::EI_EXP10: return TableRef(tbl_exp10);
455  case AMDGPULibFunc::EI_EXPM1: return TableRef(tbl_expm1);
456  case AMDGPULibFunc::EI_LOG: return TableRef(tbl_log);
458  case AMDGPULibFunc::EI_LOG2: return TableRef(tbl_log2);
459  case AMDGPULibFunc::EI_LOG10: return TableRef(tbl_log10);
461  case AMDGPULibFunc::EI_RSQRT: return TableRef(tbl_rsqrt);
463  case AMDGPULibFunc::EI_SIN: return TableRef(tbl_sin);
464  case AMDGPULibFunc::EI_SINH: return TableRef(tbl_sinh);
465  case AMDGPULibFunc::EI_SINPI: return TableRef(tbl_sinpi);
467  case AMDGPULibFunc::EI_SQRT: return TableRef(tbl_sqrt);
468  case AMDGPULibFunc::EI_TAN: return TableRef(tbl_tan);
469  case AMDGPULibFunc::EI_TANH: return TableRef(tbl_tanh);
470  case AMDGPULibFunc::EI_TANPI: return TableRef(tbl_tanpi);
471  case AMDGPULibFunc::EI_TGAMMA: return TableRef(tbl_tgamma);
472  default:;
473  }
474  return TableRef();
475 }
476 
477 static inline int getVecSize(const AMDGPULibFunc& FInfo) {
478  return FInfo.getLeads()[0].VectorSize;
479 }
480 
481 static inline AMDGPULibFunc::EType getArgType(const AMDGPULibFunc& FInfo) {
482  return (AMDGPULibFunc::EType)FInfo.getLeads()[0].ArgType;
483 }
484 
485 FunctionCallee AMDGPULibCalls::getFunction(Module *M, const FuncInfo &fInfo) {
486  // If we are doing PreLinkOpt, the function is external. So it is safe to
487  // use getOrInsertFunction() at this stage.
488 
490  : AMDGPULibFunc::getFunction(M, fInfo);
491 }
492 
493 bool AMDGPULibCalls::parseFunctionName(const StringRef& FMangledName,
494  FuncInfo *FInfo) {
495  return AMDGPULibFunc::parse(FMangledName, *FInfo);
496 }
497 
499  if (auto Op = dyn_cast<FPMathOperator>(CI))
500  if (Op->isFast())
501  return true;
502  const Function *F = CI->getParent()->getParent();
503  Attribute Attr = F->getFnAttribute("unsafe-fp-math");
504  return Attr.getValueAsString() == "true";
505 }
506 
507 bool AMDGPULibCalls::useNativeFunc(const StringRef F) const {
508  return AllNative ||
509  std::find(UseNative.begin(), UseNative.end(), F) != UseNative.end();
510 }
511 
513  AllNative = useNativeFunc("all") ||
514  (UseNative.getNumOccurrences() && UseNative.size() == 1 &&
515  UseNative.begin()->empty());
516 }
517 
518 bool AMDGPULibCalls::sincosUseNative(CallInst *aCI, const FuncInfo &FInfo) {
519  bool native_sin = useNativeFunc("sin");
520  bool native_cos = useNativeFunc("cos");
521 
522  if (native_sin && native_cos) {
523  Module *M = aCI->getModule();
524  Value *opr0 = aCI->getArgOperand(0);
525 
526  AMDGPULibFunc nf;
527  nf.getLeads()[0].ArgType = FInfo.getLeads()[0].ArgType;
528  nf.getLeads()[0].VectorSize = FInfo.getLeads()[0].VectorSize;
529 
532  FunctionCallee sinExpr = getFunction(M, nf);
533 
536  FunctionCallee cosExpr = getFunction(M, nf);
537  if (sinExpr && cosExpr) {
538  Value *sinval = CallInst::Create(sinExpr, opr0, "splitsin", aCI);
539  Value *cosval = CallInst::Create(cosExpr, opr0, "splitcos", aCI);
540  new StoreInst(cosval, aCI->getArgOperand(1), aCI);
541 
542  DEBUG_WITH_TYPE("usenative", dbgs() << "<useNative> replace " << *aCI
543  << " with native version of sin/cos");
544 
545  replaceCall(sinval);
546  return true;
547  }
548  }
549  return false;
550 }
551 
553  CI = aCI;
554  Function *Callee = aCI->getCalledFunction();
555 
556  FuncInfo FInfo;
557  if (!parseFunctionName(Callee->getName(), &FInfo) || !FInfo.isMangled() ||
558  FInfo.getPrefix() != AMDGPULibFunc::NOPFX ||
559  getArgType(FInfo) == AMDGPULibFunc::F64 || !HasNative(FInfo.getId()) ||
560  !(AllNative || useNativeFunc(FInfo.getName()))) {
561  return false;
562  }
563 
564  if (FInfo.getId() == AMDGPULibFunc::EI_SINCOS)
565  return sincosUseNative(aCI, FInfo);
566 
568  FunctionCallee F = getFunction(aCI->getModule(), FInfo);
569  if (!F)
570  return false;
571 
572  aCI->setCalledFunction(F);
573  DEBUG_WITH_TYPE("usenative", dbgs() << "<useNative> replace " << *aCI
574  << " with native version");
575  return true;
576 }
577 
578 // Clang emits call of __read_pipe_2 or __read_pipe_4 for OpenCL read_pipe
579 // builtin, with appended type size and alignment arguments, where 2 or 4
580 // indicates the original number of arguments. The library has optimized version
581 // of __read_pipe_2/__read_pipe_4 when the type size and alignment has the same
582 // power of 2 value. This function transforms __read_pipe_2 to __read_pipe_2_N
583 // for such cases where N is the size in bytes of the type (N = 1, 2, 4, 8, ...,
584 // 128). The same for __read_pipe_4, write_pipe_2, and write_pipe_4.
585 bool AMDGPULibCalls::fold_read_write_pipe(CallInst *CI, IRBuilder<> &B,
586  FuncInfo &FInfo) {
587  auto *Callee = CI->getCalledFunction();
588  if (!Callee->isDeclaration())
589  return false;
590 
591  assert(Callee->hasName() && "Invalid read_pipe/write_pipe function");
592  auto *M = Callee->getParent();
593  auto &Ctx = M->getContext();
594  std::string Name = Callee->getName();
595  auto NumArg = CI->getNumArgOperands();
596  if (NumArg != 4 && NumArg != 6)
597  return false;
598  auto *PacketSize = CI->getArgOperand(NumArg - 2);
599  auto *PacketAlign = CI->getArgOperand(NumArg - 1);
600  if (!isa<ConstantInt>(PacketSize) || !isa<ConstantInt>(PacketAlign))
601  return false;
602  unsigned Size = cast<ConstantInt>(PacketSize)->getZExtValue();
603  unsigned Align = cast<ConstantInt>(PacketAlign)->getZExtValue();
604  if (Size != Align || !isPowerOf2_32(Size))
605  return false;
606 
607  Type *PtrElemTy;
608  if (Size <= 8)
609  PtrElemTy = Type::getIntNTy(Ctx, Size * 8);
610  else
611  PtrElemTy = VectorType::get(Type::getInt64Ty(Ctx), Size / 8);
612  unsigned PtrArgLoc = CI->getNumArgOperands() - 3;
613  auto PtrArg = CI->getArgOperand(PtrArgLoc);
614  unsigned PtrArgAS = PtrArg->getType()->getPointerAddressSpace();
615  auto *PtrTy = llvm::PointerType::get(PtrElemTy, PtrArgAS);
616 
618  for (unsigned I = 0; I != PtrArgLoc; ++I)
619  ArgTys.push_back(CI->getArgOperand(I)->getType());
620  ArgTys.push_back(PtrTy);
621 
622  Name = Name + "_" + std::to_string(Size);
623  auto *FTy = FunctionType::get(Callee->getReturnType(),
624  ArrayRef<Type *>(ArgTys), false);
625  AMDGPULibFunc NewLibFunc(Name, FTy);
627  if (!F)
628  return false;
629 
630  auto *BCast = B.CreatePointerCast(PtrArg, PtrTy);
632  for (unsigned I = 0; I != PtrArgLoc; ++I)
633  Args.push_back(CI->getArgOperand(I));
634  Args.push_back(BCast);
635 
636  auto *NCI = B.CreateCall(F, Args);
637  NCI->setAttributes(CI->getAttributes());
638  CI->replaceAllUsesWith(NCI);
639  CI->dropAllReferences();
640  CI->eraseFromParent();
641 
642  return true;
643 }
644 
645 // This function returns false if no change; return true otherwise.
647  this->CI = CI;
648  Function *Callee = CI->getCalledFunction();
649 
650  // Ignore indirect calls.
651  if (Callee == 0) return false;
652 
653  BasicBlock *BB = CI->getParent();
655  IRBuilder<> B(Context);
656 
657  // Set the builder to the instruction after the call.
658  B.SetInsertPoint(BB, CI->getIterator());
659 
660  // Copy fast flags from the original call.
661  if (const FPMathOperator *FPOp = dyn_cast<const FPMathOperator>(CI))
662  B.setFastMathFlags(FPOp->getFastMathFlags());
663 
664  switch (Callee->getIntrinsicID()) {
665  default:
666  break;
667  case Intrinsic::amdgcn_wavefrontsize:
668  return !EnablePreLink && fold_wavefrontsize(CI, B);
669  }
670 
671  FuncInfo FInfo;
672  if (!parseFunctionName(Callee->getName(), &FInfo))
673  return false;
674 
675  // Further check the number of arguments to see if they match.
676  if (CI->getNumArgOperands() != FInfo.getNumArgs())
677  return false;
678 
679  if (TDOFold(CI, FInfo))
680  return true;
681 
682  // Under unsafe-math, evaluate calls if possible.
683  // According to Brian Sumner, we can do this for all f32 function calls
684  // using host's double function calls.
685  if (isUnsafeMath(CI) && evaluateCall(CI, FInfo))
686  return true;
687 
688  // Specilized optimizations for each function call
689  switch (FInfo.getId()) {
691  // skip vector function
692  assert ((FInfo.getPrefix() == AMDGPULibFunc::NATIVE ||
693  FInfo.getPrefix() == AMDGPULibFunc::HALF) &&
694  "recip must be an either native or half function");
695  return (getVecSize(FInfo) != 1) ? false : fold_recip(CI, B, FInfo);
696 
698  // skip vector function
699  assert ((FInfo.getPrefix() == AMDGPULibFunc::NATIVE ||
700  FInfo.getPrefix() == AMDGPULibFunc::HALF) &&
701  "divide must be an either native or half function");
702  return (getVecSize(FInfo) != 1) ? false : fold_divide(CI, B, FInfo);
703 
707  return fold_pow(CI, B, FInfo);
708 
710  // skip vector function
711  return (getVecSize(FInfo) != 1) ? false : fold_rootn(CI, B, FInfo);
712 
716  // skip vector function
717  return (getVecSize(FInfo) != 1) ? false : fold_fma_mad(CI, B, FInfo);
718 
720  return isUnsafeMath(CI) && fold_sqrt(CI, B, FInfo);
723  if ((getArgType(FInfo) == AMDGPULibFunc::F32 ||
724  getArgType(FInfo) == AMDGPULibFunc::F64)
725  && (FInfo.getPrefix() == AMDGPULibFunc::NOPFX))
726  return fold_sincos(CI, B, AA);
727 
728  break;
733  return fold_read_write_pipe(CI, B, FInfo);
734 
735  default:
736  break;
737  }
738 
739  return false;
740 }
741 
742 bool AMDGPULibCalls::TDOFold(CallInst *CI, const FuncInfo &FInfo) {
743  // Table-Driven optimization
744  const TableRef tr = getOptTable(FInfo.getId());
745  if (tr.size==0)
746  return false;
747 
748  int const sz = (int)tr.size;
749  const TableEntry * const ftbl = tr.table;
750  Value *opr0 = CI->getArgOperand(0);
751 
752  if (getVecSize(FInfo) > 1) {
753  if (ConstantDataVector *CV = dyn_cast<ConstantDataVector>(opr0)) {
755  for (int eltNo = 0; eltNo < getVecSize(FInfo); ++eltNo) {
756  ConstantFP *eltval = dyn_cast<ConstantFP>(
757  CV->getElementAsConstant((unsigned)eltNo));
758  assert(eltval && "Non-FP arguments in math function!");
759  bool found = false;
760  for (int i=0; i < sz; ++i) {
761  if (eltval->isExactlyValue(ftbl[i].input)) {
762  DVal.push_back(ftbl[i].result);
763  found = true;
764  break;
765  }
766  }
767  if (!found) {
768  // This vector constants not handled yet.
769  return false;
770  }
771  }
772  LLVMContext &context = CI->getParent()->getParent()->getContext();
773  Constant *nval;
774  if (getArgType(FInfo) == AMDGPULibFunc::F32) {
776  for (unsigned i = 0; i < DVal.size(); ++i) {
777  FVal.push_back((float)DVal[i]);
778  }
779  ArrayRef<float> tmp(FVal);
780  nval = ConstantDataVector::get(context, tmp);
781  } else { // F64
782  ArrayRef<double> tmp(DVal);
783  nval = ConstantDataVector::get(context, tmp);
784  }
785  LLVM_DEBUG(errs() << "AMDIC: " << *CI << " ---> " << *nval << "\n");
786  replaceCall(nval);
787  return true;
788  }
789  } else {
790  // Scalar version
791  if (ConstantFP *CF = dyn_cast<ConstantFP>(opr0)) {
792  for (int i = 0; i < sz; ++i) {
793  if (CF->isExactlyValue(ftbl[i].input)) {
794  Value *nval = ConstantFP::get(CF->getType(), ftbl[i].result);
795  LLVM_DEBUG(errs() << "AMDIC: " << *CI << " ---> " << *nval << "\n");
796  replaceCall(nval);
797  return true;
798  }
799  }
800  }
801  }
802 
803  return false;
804 }
805 
806 bool AMDGPULibCalls::replaceWithNative(CallInst *CI, const FuncInfo &FInfo) {
807  Module *M = CI->getModule();
808  if (getArgType(FInfo) != AMDGPULibFunc::F32 ||
809  FInfo.getPrefix() != AMDGPULibFunc::NOPFX ||
810  !HasNative(FInfo.getId()))
811  return false;
812 
813  AMDGPULibFunc nf = FInfo;
815  if (FunctionCallee FPExpr = getFunction(M, nf)) {
816  LLVM_DEBUG(dbgs() << "AMDIC: " << *CI << " ---> ");
817 
818  CI->setCalledFunction(FPExpr);
819 
820  LLVM_DEBUG(dbgs() << *CI << '\n');
821 
822  return true;
823  }
824  return false;
825 }
826 
827 // [native_]half_recip(c) ==> 1.0/c
828 bool AMDGPULibCalls::fold_recip(CallInst *CI, IRBuilder<> &B,
829  const FuncInfo &FInfo) {
830  Value *opr0 = CI->getArgOperand(0);
831  if (ConstantFP *CF = dyn_cast<ConstantFP>(opr0)) {
832  // Just create a normal div. Later, InstCombine will be able
833  // to compute the divide into a constant (avoid check float infinity
834  // or subnormal at this point).
835  Value *nval = B.CreateFDiv(ConstantFP::get(CF->getType(), 1.0),
836  opr0,
837  "recip2div");
838  LLVM_DEBUG(errs() << "AMDIC: " << *CI << " ---> " << *nval << "\n");
839  replaceCall(nval);
840  return true;
841  }
842  return false;
843 }
844 
845 // [native_]half_divide(x, c) ==> x/c
846 bool AMDGPULibCalls::fold_divide(CallInst *CI, IRBuilder<> &B,
847  const FuncInfo &FInfo) {
848  Value *opr0 = CI->getArgOperand(0);
849  Value *opr1 = CI->getArgOperand(1);
850  ConstantFP *CF0 = dyn_cast<ConstantFP>(opr0);
851  ConstantFP *CF1 = dyn_cast<ConstantFP>(opr1);
852 
853  if ((CF0 && CF1) || // both are constants
854  (CF1 && (getArgType(FInfo) == AMDGPULibFunc::F32)))
855  // CF1 is constant && f32 divide
856  {
857  Value *nval1 = B.CreateFDiv(ConstantFP::get(opr1->getType(), 1.0),
858  opr1, "__div2recip");
859  Value *nval = B.CreateFMul(opr0, nval1, "__div2mul");
860  replaceCall(nval);
861  return true;
862  }
863  return false;
864 }
865 
866 namespace llvm {
867 static double log2(double V) {
868 #if _XOPEN_SOURCE >= 600 || defined(_ISOC99_SOURCE) || _POSIX_C_SOURCE >= 200112L
869  return ::log2(V);
870 #else
871  return log(V) / 0.693147180559945309417;
872 #endif
873 }
874 }
875 
876 bool AMDGPULibCalls::fold_pow(CallInst *CI, IRBuilder<> &B,
877  const FuncInfo &FInfo) {
878  assert((FInfo.getId() == AMDGPULibFunc::EI_POW ||
879  FInfo.getId() == AMDGPULibFunc::EI_POWR ||
880  FInfo.getId() == AMDGPULibFunc::EI_POWN) &&
881  "fold_pow: encounter a wrong function call");
882 
883  Value *opr0, *opr1;
884  ConstantFP *CF;
885  ConstantInt *CINT;
886  ConstantAggregateZero *CZero;
887  Type *eltType;
888 
889  opr0 = CI->getArgOperand(0);
890  opr1 = CI->getArgOperand(1);
891  CZero = dyn_cast<ConstantAggregateZero>(opr1);
892  if (getVecSize(FInfo) == 1) {
893  eltType = opr0->getType();
894  CF = dyn_cast<ConstantFP>(opr1);
895  CINT = dyn_cast<ConstantInt>(opr1);
896  } else {
897  VectorType *VTy = dyn_cast<VectorType>(opr0->getType());
898  assert(VTy && "Oprand of vector function should be of vectortype");
899  eltType = VTy->getElementType();
901 
902  // Now, only Handle vector const whose elements have the same value.
903  CF = CDV ? dyn_cast_or_null<ConstantFP>(CDV->getSplatValue()) : nullptr;
904  CINT = CDV ? dyn_cast_or_null<ConstantInt>(CDV->getSplatValue()) : nullptr;
905  }
906 
907  // No unsafe math , no constant argument, do nothing
908  if (!isUnsafeMath(CI) && !CF && !CINT && !CZero)
909  return false;
910 
911  // 0x1111111 means that we don't do anything for this call.
912  int ci_opr1 = (CINT ? (int)CINT->getSExtValue() : 0x1111111);
913 
914  if ((CF && CF->isZero()) || (CINT && ci_opr1 == 0) || CZero) {
915  // pow/powr/pown(x, 0) == 1
916  LLVM_DEBUG(errs() << "AMDIC: " << *CI << " ---> 1\n");
917  Constant *cnval = ConstantFP::get(eltType, 1.0);
918  if (getVecSize(FInfo) > 1) {
919  cnval = ConstantDataVector::getSplat(getVecSize(FInfo), cnval);
920  }
921  replaceCall(cnval);
922  return true;
923  }
924  if ((CF && CF->isExactlyValue(1.0)) || (CINT && ci_opr1 == 1)) {
925  // pow/powr/pown(x, 1.0) = x
926  LLVM_DEBUG(errs() << "AMDIC: " << *CI << " ---> " << *opr0 << "\n");
927  replaceCall(opr0);
928  return true;
929  }
930  if ((CF && CF->isExactlyValue(2.0)) || (CINT && ci_opr1 == 2)) {
931  // pow/powr/pown(x, 2.0) = x*x
932  LLVM_DEBUG(errs() << "AMDIC: " << *CI << " ---> " << *opr0 << " * " << *opr0
933  << "\n");
934  Value *nval = B.CreateFMul(opr0, opr0, "__pow2");
935  replaceCall(nval);
936  return true;
937  }
938  if ((CF && CF->isExactlyValue(-1.0)) || (CINT && ci_opr1 == -1)) {
939  // pow/powr/pown(x, -1.0) = 1.0/x
940  LLVM_DEBUG(errs() << "AMDIC: " << *CI << " ---> 1 / " << *opr0 << "\n");
941  Constant *cnval = ConstantFP::get(eltType, 1.0);
942  if (getVecSize(FInfo) > 1) {
943  cnval = ConstantDataVector::getSplat(getVecSize(FInfo), cnval);
944  }
945  Value *nval = B.CreateFDiv(cnval, opr0, "__powrecip");
946  replaceCall(nval);
947  return true;
948  }
949 
950  Module *M = CI->getModule();
951  if (CF && (CF->isExactlyValue(0.5) || CF->isExactlyValue(-0.5))) {
952  // pow[r](x, [-]0.5) = sqrt(x)
953  bool issqrt = CF->isExactlyValue(0.5);
954  if (FunctionCallee FPExpr =
955  getFunction(M, AMDGPULibFunc(issqrt ? AMDGPULibFunc::EI_SQRT
957  FInfo))) {
958  LLVM_DEBUG(errs() << "AMDIC: " << *CI << " ---> "
959  << FInfo.getName().c_str() << "(" << *opr0 << ")\n");
960  Value *nval = CreateCallEx(B,FPExpr, opr0, issqrt ? "__pow2sqrt"
961  : "__pow2rsqrt");
962  replaceCall(nval);
963  return true;
964  }
965  }
966 
967  if (!isUnsafeMath(CI))
968  return false;
969 
970  // Unsafe Math optimization
971 
972  // Remember that ci_opr1 is set if opr1 is integral
973  if (CF) {
974  double dval = (getArgType(FInfo) == AMDGPULibFunc::F32)
975  ? (double)CF->getValueAPF().convertToFloat()
976  : CF->getValueAPF().convertToDouble();
977  int ival = (int)dval;
978  if ((double)ival == dval) {
979  ci_opr1 = ival;
980  } else
981  ci_opr1 = 0x11111111;
982  }
983 
984  // pow/powr/pown(x, c) = [1/](x*x*..x); where
985  // trunc(c) == c && the number of x == c && |c| <= 12
986  unsigned abs_opr1 = (ci_opr1 < 0) ? -ci_opr1 : ci_opr1;
987  if (abs_opr1 <= 12) {
988  Constant *cnval;
989  Value *nval;
990  if (abs_opr1 == 0) {
991  cnval = ConstantFP::get(eltType, 1.0);
992  if (getVecSize(FInfo) > 1) {
993  cnval = ConstantDataVector::getSplat(getVecSize(FInfo), cnval);
994  }
995  nval = cnval;
996  } else {
997  Value *valx2 = nullptr;
998  nval = nullptr;
999  while (abs_opr1 > 0) {
1000  valx2 = valx2 ? B.CreateFMul(valx2, valx2, "__powx2") : opr0;
1001  if (abs_opr1 & 1) {
1002  nval = nval ? B.CreateFMul(nval, valx2, "__powprod") : valx2;
1003  }
1004  abs_opr1 >>= 1;
1005  }
1006  }
1007 
1008  if (ci_opr1 < 0) {
1009  cnval = ConstantFP::get(eltType, 1.0);
1010  if (getVecSize(FInfo) > 1) {
1011  cnval = ConstantDataVector::getSplat(getVecSize(FInfo), cnval);
1012  }
1013  nval = B.CreateFDiv(cnval, nval, "__1powprod");
1014  }
1015  LLVM_DEBUG(errs() << "AMDIC: " << *CI << " ---> "
1016  << ((ci_opr1 < 0) ? "1/prod(" : "prod(") << *opr0
1017  << ")\n");
1018  replaceCall(nval);
1019  return true;
1020  }
1021 
1022  // powr ---> exp2(y * log2(x))
1023  // pown/pow ---> powr(fabs(x), y) | (x & ((int)y << 31))
1024  FunctionCallee ExpExpr =
1025  getFunction(M, AMDGPULibFunc(AMDGPULibFunc::EI_EXP2, FInfo));
1026  if (!ExpExpr)
1027  return false;
1028 
1029  bool needlog = false;
1030  bool needabs = false;
1031  bool needcopysign = false;
1032  Constant *cnval = nullptr;
1033  if (getVecSize(FInfo) == 1) {
1034  CF = dyn_cast<ConstantFP>(opr0);
1035 
1036  if (CF) {
1037  double V = (getArgType(FInfo) == AMDGPULibFunc::F32)
1038  ? (double)CF->getValueAPF().convertToFloat()
1039  : CF->getValueAPF().convertToDouble();
1040 
1041  V = log2(std::abs(V));
1042  cnval = ConstantFP::get(eltType, V);
1043  needcopysign = (FInfo.getId() != AMDGPULibFunc::EI_POWR) &&
1044  CF->isNegative();
1045  } else {
1046  needlog = true;
1047  needcopysign = needabs = FInfo.getId() != AMDGPULibFunc::EI_POWR &&
1048  (!CF || CF->isNegative());
1049  }
1050  } else {
1052 
1053  if (!CDV) {
1054  needlog = true;
1055  needcopysign = needabs = FInfo.getId() != AMDGPULibFunc::EI_POWR;
1056  } else {
1057  assert ((int)CDV->getNumElements() == getVecSize(FInfo) &&
1058  "Wrong vector size detected");
1059 
1061  for (int i=0; i < getVecSize(FInfo); ++i) {
1062  double V = (getArgType(FInfo) == AMDGPULibFunc::F32)
1063  ? (double)CDV->getElementAsFloat(i)
1064  : CDV->getElementAsDouble(i);
1065  if (V < 0.0) needcopysign = true;
1066  V = log2(std::abs(V));
1067  DVal.push_back(V);
1068  }
1069  if (getArgType(FInfo) == AMDGPULibFunc::F32) {
1070  SmallVector<float, 0> FVal;
1071  for (unsigned i=0; i < DVal.size(); ++i) {
1072  FVal.push_back((float)DVal[i]);
1073  }
1074  ArrayRef<float> tmp(FVal);
1075  cnval = ConstantDataVector::get(M->getContext(), tmp);
1076  } else {
1077  ArrayRef<double> tmp(DVal);
1078  cnval = ConstantDataVector::get(M->getContext(), tmp);
1079  }
1080  }
1081  }
1082 
1083  if (needcopysign && (FInfo.getId() == AMDGPULibFunc::EI_POW)) {
1084  // We cannot handle corner cases for a general pow() function, give up
1085  // unless y is a constant integral value. Then proceed as if it were pown.
1086  if (getVecSize(FInfo) == 1) {
1087  if (const ConstantFP *CF = dyn_cast<ConstantFP>(opr1)) {
1088  double y = (getArgType(FInfo) == AMDGPULibFunc::F32)
1089  ? (double)CF->getValueAPF().convertToFloat()
1090  : CF->getValueAPF().convertToDouble();
1091  if (y != (double)(int64_t)y)
1092  return false;
1093  } else
1094  return false;
1095  } else {
1096  if (const ConstantDataVector *CDV = dyn_cast<ConstantDataVector>(opr1)) {
1097  for (int i=0; i < getVecSize(FInfo); ++i) {
1098  double y = (getArgType(FInfo) == AMDGPULibFunc::F32)
1099  ? (double)CDV->getElementAsFloat(i)
1100  : CDV->getElementAsDouble(i);
1101  if (y != (double)(int64_t)y)
1102  return false;
1103  }
1104  } else
1105  return false;
1106  }
1107  }
1108 
1109  Value *nval;
1110  if (needabs) {
1111  FunctionCallee AbsExpr =
1112  getFunction(M, AMDGPULibFunc(AMDGPULibFunc::EI_FABS, FInfo));
1113  if (!AbsExpr)
1114  return false;
1115  nval = CreateCallEx(B, AbsExpr, opr0, "__fabs");
1116  } else {
1117  nval = cnval ? cnval : opr0;
1118  }
1119  if (needlog) {
1120  FunctionCallee LogExpr =
1121  getFunction(M, AMDGPULibFunc(AMDGPULibFunc::EI_LOG2, FInfo));
1122  if (!LogExpr)
1123  return false;
1124  nval = CreateCallEx(B,LogExpr, nval, "__log2");
1125  }
1126 
1127  if (FInfo.getId() == AMDGPULibFunc::EI_POWN) {
1128  // convert int(32) to fp(f32 or f64)
1129  opr1 = B.CreateSIToFP(opr1, nval->getType(), "pownI2F");
1130  }
1131  nval = B.CreateFMul(opr1, nval, "__ylogx");
1132  nval = CreateCallEx(B,ExpExpr, nval, "__exp2");
1133 
1134  if (needcopysign) {
1135  Value *opr_n;
1136  Type* rTy = opr0->getType();
1137  Type* nTyS = eltType->isDoubleTy() ? B.getInt64Ty() : B.getInt32Ty();
1138  Type *nTy = nTyS;
1139  if (const VectorType *vTy = dyn_cast<VectorType>(rTy))
1140  nTy = VectorType::get(nTyS, vTy->getNumElements());
1141  unsigned size = nTy->getScalarSizeInBits();
1142  opr_n = CI->getArgOperand(1);
1143  if (opr_n->getType()->isIntegerTy())
1144  opr_n = B.CreateZExtOrBitCast(opr_n, nTy, "__ytou");
1145  else
1146  opr_n = B.CreateFPToSI(opr1, nTy, "__ytou");
1147 
1148  Value *sign = B.CreateShl(opr_n, size-1, "__yeven");
1149  sign = B.CreateAnd(B.CreateBitCast(opr0, nTy), sign, "__pow_sign");
1150  nval = B.CreateOr(B.CreateBitCast(nval, nTy), sign);
1151  nval = B.CreateBitCast(nval, opr0->getType());
1152  }
1153 
1154  LLVM_DEBUG(errs() << "AMDIC: " << *CI << " ---> "
1155  << "exp2(" << *opr1 << " * log2(" << *opr0 << "))\n");
1156  replaceCall(nval);
1157 
1158  return true;
1159 }
1160 
1161 bool AMDGPULibCalls::fold_rootn(CallInst *CI, IRBuilder<> &B,
1162  const FuncInfo &FInfo) {
1163  Value *opr0 = CI->getArgOperand(0);
1164  Value *opr1 = CI->getArgOperand(1);
1165 
1166  ConstantInt *CINT = dyn_cast<ConstantInt>(opr1);
1167  if (!CINT) {
1168  return false;
1169  }
1170  int ci_opr1 = (int)CINT->getSExtValue();
1171  if (ci_opr1 == 1) { // rootn(x, 1) = x
1172  LLVM_DEBUG(errs() << "AMDIC: " << *CI << " ---> " << *opr0 << "\n");
1173  replaceCall(opr0);
1174  return true;
1175  }
1176  if (ci_opr1 == 2) { // rootn(x, 2) = sqrt(x)
1177  std::vector<const Type*> ParamsTys;
1178  ParamsTys.push_back(opr0->getType());
1179  Module *M = CI->getModule();
1180  if (FunctionCallee FPExpr =
1181  getFunction(M, AMDGPULibFunc(AMDGPULibFunc::EI_SQRT, FInfo))) {
1182  LLVM_DEBUG(errs() << "AMDIC: " << *CI << " ---> sqrt(" << *opr0 << ")\n");
1183  Value *nval = CreateCallEx(B,FPExpr, opr0, "__rootn2sqrt");
1184  replaceCall(nval);
1185  return true;
1186  }
1187  } else if (ci_opr1 == 3) { // rootn(x, 3) = cbrt(x)
1188  Module *M = CI->getModule();
1189  if (FunctionCallee FPExpr =
1190  getFunction(M, AMDGPULibFunc(AMDGPULibFunc::EI_CBRT, FInfo))) {
1191  LLVM_DEBUG(errs() << "AMDIC: " << *CI << " ---> cbrt(" << *opr0 << ")\n");
1192  Value *nval = CreateCallEx(B,FPExpr, opr0, "__rootn2cbrt");
1193  replaceCall(nval);
1194  return true;
1195  }
1196  } else if (ci_opr1 == -1) { // rootn(x, -1) = 1.0/x
1197  LLVM_DEBUG(errs() << "AMDIC: " << *CI << " ---> 1.0 / " << *opr0 << "\n");
1198  Value *nval = B.CreateFDiv(ConstantFP::get(opr0->getType(), 1.0),
1199  opr0,
1200  "__rootn2div");
1201  replaceCall(nval);
1202  return true;
1203  } else if (ci_opr1 == -2) { // rootn(x, -2) = rsqrt(x)
1204  std::vector<const Type*> ParamsTys;
1205  ParamsTys.push_back(opr0->getType());
1206  Module *M = CI->getModule();
1207  if (FunctionCallee FPExpr =
1208  getFunction(M, AMDGPULibFunc(AMDGPULibFunc::EI_RSQRT, FInfo))) {
1209  LLVM_DEBUG(errs() << "AMDIC: " << *CI << " ---> rsqrt(" << *opr0
1210  << ")\n");
1211  Value *nval = CreateCallEx(B,FPExpr, opr0, "__rootn2rsqrt");
1212  replaceCall(nval);
1213  return true;
1214  }
1215  }
1216  return false;
1217 }
1218 
1219 bool AMDGPULibCalls::fold_fma_mad(CallInst *CI, IRBuilder<> &B,
1220  const FuncInfo &FInfo) {
1221  Value *opr0 = CI->getArgOperand(0);
1222  Value *opr1 = CI->getArgOperand(1);
1223  Value *opr2 = CI->getArgOperand(2);
1224 
1225  ConstantFP *CF0 = dyn_cast<ConstantFP>(opr0);
1226  ConstantFP *CF1 = dyn_cast<ConstantFP>(opr1);
1227  if ((CF0 && CF0->isZero()) || (CF1 && CF1->isZero())) {
1228  // fma/mad(a, b, c) = c if a=0 || b=0
1229  LLVM_DEBUG(errs() << "AMDIC: " << *CI << " ---> " << *opr2 << "\n");
1230  replaceCall(opr2);
1231  return true;
1232  }
1233  if (CF0 && CF0->isExactlyValue(1.0f)) {
1234  // fma/mad(a, b, c) = b+c if a=1
1235  LLVM_DEBUG(errs() << "AMDIC: " << *CI << " ---> " << *opr1 << " + " << *opr2
1236  << "\n");
1237  Value *nval = B.CreateFAdd(opr1, opr2, "fmaadd");
1238  replaceCall(nval);
1239  return true;
1240  }
1241  if (CF1 && CF1->isExactlyValue(1.0f)) {
1242  // fma/mad(a, b, c) = a+c if b=1
1243  LLVM_DEBUG(errs() << "AMDIC: " << *CI << " ---> " << *opr0 << " + " << *opr2
1244  << "\n");
1245  Value *nval = B.CreateFAdd(opr0, opr2, "fmaadd");
1246  replaceCall(nval);
1247  return true;
1248  }
1249  if (ConstantFP *CF = dyn_cast<ConstantFP>(opr2)) {
1250  if (CF->isZero()) {
1251  // fma/mad(a, b, c) = a*b if c=0
1252  LLVM_DEBUG(errs() << "AMDIC: " << *CI << " ---> " << *opr0 << " * "
1253  << *opr1 << "\n");
1254  Value *nval = B.CreateFMul(opr0, opr1, "fmamul");
1255  replaceCall(nval);
1256  return true;
1257  }
1258  }
1259 
1260  return false;
1261 }
1262 
1263 // Get a scalar native builtin signle argument FP function
1264 FunctionCallee AMDGPULibCalls::getNativeFunction(Module *M,
1265  const FuncInfo &FInfo) {
1266  if (getArgType(FInfo) == AMDGPULibFunc::F64 || !HasNative(FInfo.getId()))
1267  return nullptr;
1268  FuncInfo nf = FInfo;
1269  nf.setPrefix(AMDGPULibFunc::NATIVE);
1270  return getFunction(M, nf);
1271 }
1272 
1273 // fold sqrt -> native_sqrt (x)
1274 bool AMDGPULibCalls::fold_sqrt(CallInst *CI, IRBuilder<> &B,
1275  const FuncInfo &FInfo) {
1276  if (getArgType(FInfo) == AMDGPULibFunc::F32 && (getVecSize(FInfo) == 1) &&
1277  (FInfo.getPrefix() != AMDGPULibFunc::NATIVE)) {
1278  if (FunctionCallee FPExpr = getNativeFunction(
1280  Value *opr0 = CI->getArgOperand(0);
1281  LLVM_DEBUG(errs() << "AMDIC: " << *CI << " ---> "
1282  << "sqrt(" << *opr0 << ")\n");
1283  Value *nval = CreateCallEx(B,FPExpr, opr0, "__sqrt");
1284  replaceCall(nval);
1285  return true;
1286  }
1287  }
1288  return false;
1289 }
1290 
1291 // fold sin, cos -> sincos.
1292 bool AMDGPULibCalls::fold_sincos(CallInst *CI, IRBuilder<> &B,
1293  AliasAnalysis *AA) {
1294  AMDGPULibFunc fInfo;
1295  if (!AMDGPULibFunc::parse(CI->getCalledFunction()->getName(), fInfo))
1296  return false;
1297 
1298  assert(fInfo.getId() == AMDGPULibFunc::EI_SIN ||
1299  fInfo.getId() == AMDGPULibFunc::EI_COS);
1300  bool const isSin = fInfo.getId() == AMDGPULibFunc::EI_SIN;
1301 
1302  Value *CArgVal = CI->getArgOperand(0);
1303  BasicBlock * const CBB = CI->getParent();
1304 
1305  int const MaxScan = 30;
1306 
1307  { // fold in load value.
1308  LoadInst *LI = dyn_cast<LoadInst>(CArgVal);
1309  if (LI && LI->getParent() == CBB) {
1310  BasicBlock::iterator BBI = LI->getIterator();
1311  Value *AvailableVal = FindAvailableLoadedValue(LI, CBB, BBI, MaxScan, AA);
1312  if (AvailableVal) {
1313  CArgVal->replaceAllUsesWith(AvailableVal);
1314  if (CArgVal->getNumUses() == 0)
1315  LI->eraseFromParent();
1316  CArgVal = CI->getArgOperand(0);
1317  }
1318  }
1319  }
1320 
1321  Module *M = CI->getModule();
1323  std::string const PairName = fInfo.mangle();
1324 
1325  CallInst *UI = nullptr;
1326  for (User* U : CArgVal->users()) {
1327  CallInst *XI = dyn_cast_or_null<CallInst>(U);
1328  if (!XI || XI == CI || XI->getParent() != CBB)
1329  continue;
1330 
1331  Function *UCallee = XI->getCalledFunction();
1332  if (!UCallee || !UCallee->getName().equals(PairName))
1333  continue;
1334 
1335  BasicBlock::iterator BBI = CI->getIterator();
1336  if (BBI == CI->getParent()->begin())
1337  break;
1338  --BBI;
1339  for (int I = MaxScan; I > 0 && BBI != CBB->begin(); --BBI, --I) {
1340  if (cast<Instruction>(BBI) == XI) {
1341  UI = XI;
1342  break;
1343  }
1344  }
1345  if (UI) break;
1346  }
1347 
1348  if (!UI) return false;
1349 
1350  // Merge the sin and cos.
1351 
1352  // for OpenCL 2.0 we have only generic implementation of sincos
1353  // function.
1356  FunctionCallee Fsincos = getFunction(M, nf);
1357  if (!Fsincos) return false;
1358 
1360  AllocaInst *Alloc = insertAlloca(UI, B, "__sincos_");
1361  B.SetInsertPoint(UI);
1362 
1363  Value *P = Alloc;
1364  Type *PTy = Fsincos.getFunctionType()->getParamType(1);
1365  // The allocaInst allocates the memory in private address space. This need
1366  // to be bitcasted to point to the address space of cos pointer type.
1367  // In OpenCL 2.0 this is generic, while in 1.2 that is private.
1369  P = B.CreateAddrSpaceCast(Alloc, PTy);
1370  CallInst *Call = CreateCallEx2(B, Fsincos, UI->getArgOperand(0), P);
1371 
1372  LLVM_DEBUG(errs() << "AMDIC: fold_sincos (" << *CI << ", " << *UI << ") with "
1373  << *Call << "\n");
1374 
1375  if (!isSin) { // CI->cos, UI->sin
1376  B.SetInsertPoint(&*ItOld);
1377  UI->replaceAllUsesWith(&*Call);
1378  Instruction *Reload = B.CreateLoad(Alloc->getAllocatedType(), Alloc);
1379  CI->replaceAllUsesWith(Reload);
1380  UI->eraseFromParent();
1381  CI->eraseFromParent();
1382  } else { // CI->sin, UI->cos
1383  Instruction *Reload = B.CreateLoad(Alloc->getAllocatedType(), Alloc);
1384  UI->replaceAllUsesWith(Reload);
1385  CI->replaceAllUsesWith(Call);
1386  UI->eraseFromParent();
1387  CI->eraseFromParent();
1388  }
1389  return true;
1390 }
1391 
1392 bool AMDGPULibCalls::fold_wavefrontsize(CallInst *CI, IRBuilder<> &B) {
1393  if (!TM)
1394  return false;
1395 
1396  StringRef CPU = TM->getTargetCPU();
1397  StringRef Features = TM->getTargetFeatureString();
1398  if ((CPU.empty() || CPU.equals_lower("generic")) &&
1399  (Features.empty() ||
1400  Features.find_lower("wavefrontsize") == StringRef::npos))
1401  return false;
1402 
1403  Function *F = CI->getParent()->getParent();
1404  const GCNSubtarget &ST = TM->getSubtarget<GCNSubtarget>(*F);
1405  unsigned N = ST.getWavefrontSize();
1406 
1407  LLVM_DEBUG(errs() << "AMDIC: fold_wavefrontsize (" << *CI << ") with "
1408  << N << "\n");
1409 
1411  CI->eraseFromParent();
1412  return true;
1413 }
1414 
1415 // Get insertion point at entry.
1416 BasicBlock::iterator AMDGPULibCalls::getEntryIns(CallInst * UI) {
1417  Function * Func = UI->getParent()->getParent();
1418  BasicBlock * BB = &Func->getEntryBlock();
1419  assert(BB && "Entry block not found!");
1420  BasicBlock::iterator ItNew = BB->begin();
1421  return ItNew;
1422 }
1423 
1424 // Insert a AllocsInst at the beginning of function entry block.
1425 AllocaInst* AMDGPULibCalls::insertAlloca(CallInst *UI, IRBuilder<> &B,
1426  const char *prefix) {
1427  BasicBlock::iterator ItNew = getEntryIns(UI);
1428  Function *UCallee = UI->getCalledFunction();
1429  Type *RetType = UCallee->getReturnType();
1430  B.SetInsertPoint(&*ItNew);
1431  AllocaInst *Alloc = B.CreateAlloca(RetType, 0,
1432  std::string(prefix) + UI->getName());
1433  Alloc->setAlignment(UCallee->getParent()->getDataLayout()
1434  .getTypeAllocSize(RetType));
1435  return Alloc;
1436 }
1437 
1438 bool AMDGPULibCalls::evaluateScalarMathFunc(FuncInfo &FInfo,
1439  double& Res0, double& Res1,
1440  Constant *copr0, Constant *copr1,
1441  Constant *copr2) {
1442  // By default, opr0/opr1/opr3 holds values of float/double type.
1443  // If they are not float/double, each function has to its
1444  // operand separately.
1445  double opr0=0.0, opr1=0.0, opr2=0.0;
1446  ConstantFP *fpopr0 = dyn_cast_or_null<ConstantFP>(copr0);
1447  ConstantFP *fpopr1 = dyn_cast_or_null<ConstantFP>(copr1);
1448  ConstantFP *fpopr2 = dyn_cast_or_null<ConstantFP>(copr2);
1449  if (fpopr0) {
1450  opr0 = (getArgType(FInfo) == AMDGPULibFunc::F64)
1451  ? fpopr0->getValueAPF().convertToDouble()
1452  : (double)fpopr0->getValueAPF().convertToFloat();
1453  }
1454 
1455  if (fpopr1) {
1456  opr1 = (getArgType(FInfo) == AMDGPULibFunc::F64)
1457  ? fpopr1->getValueAPF().convertToDouble()
1458  : (double)fpopr1->getValueAPF().convertToFloat();
1459  }
1460 
1461  if (fpopr2) {
1462  opr2 = (getArgType(FInfo) == AMDGPULibFunc::F64)
1463  ? fpopr2->getValueAPF().convertToDouble()
1464  : (double)fpopr2->getValueAPF().convertToFloat();
1465  }
1466 
1467  switch (FInfo.getId()) {
1468  default : return false;
1469 
1471  Res0 = acos(opr0);
1472  return true;
1473 
1475  // acosh(x) == log(x + sqrt(x*x - 1))
1476  Res0 = log(opr0 + sqrt(opr0*opr0 - 1.0));
1477  return true;
1478 
1480  Res0 = acos(opr0) / MATH_PI;
1481  return true;
1482 
1484  Res0 = asin(opr0);
1485  return true;
1486 
1488  // asinh(x) == log(x + sqrt(x*x + 1))
1489  Res0 = log(opr0 + sqrt(opr0*opr0 + 1.0));
1490  return true;
1491 
1493  Res0 = asin(opr0) / MATH_PI;
1494  return true;
1495 
1497  Res0 = atan(opr0);
1498  return true;
1499 
1501  // atanh(x) == (log(x+1) - log(x-1))/2;
1502  Res0 = (log(opr0 + 1.0) - log(opr0 - 1.0))/2.0;
1503  return true;
1504 
1506  Res0 = atan(opr0) / MATH_PI;
1507  return true;
1508 
1510  Res0 = (opr0 < 0.0) ? -pow(-opr0, 1.0/3.0) : pow(opr0, 1.0/3.0);
1511  return true;
1512 
1513  case AMDGPULibFunc::EI_COS:
1514  Res0 = cos(opr0);
1515  return true;
1516 
1518  Res0 = cosh(opr0);
1519  return true;
1520 
1522  Res0 = cos(MATH_PI * opr0);
1523  return true;
1524 
1525  case AMDGPULibFunc::EI_EXP:
1526  Res0 = exp(opr0);
1527  return true;
1528 
1530  Res0 = pow(2.0, opr0);
1531  return true;
1532 
1534  Res0 = pow(10.0, opr0);
1535  return true;
1536 
1538  Res0 = exp(opr0) - 1.0;
1539  return true;
1540 
1541  case AMDGPULibFunc::EI_LOG:
1542  Res0 = log(opr0);
1543  return true;
1544 
1546  Res0 = log(opr0) / log(2.0);
1547  return true;
1548 
1550  Res0 = log(opr0) / log(10.0);
1551  return true;
1552 
1554  Res0 = 1.0 / sqrt(opr0);
1555  return true;
1556 
1557  case AMDGPULibFunc::EI_SIN:
1558  Res0 = sin(opr0);
1559  return true;
1560 
1562  Res0 = sinh(opr0);
1563  return true;
1564 
1566  Res0 = sin(MATH_PI * opr0);
1567  return true;
1568 
1570  Res0 = sqrt(opr0);
1571  return true;
1572 
1573  case AMDGPULibFunc::EI_TAN:
1574  Res0 = tan(opr0);
1575  return true;
1576 
1578  Res0 = tanh(opr0);
1579  return true;
1580 
1582  Res0 = tan(MATH_PI * opr0);
1583  return true;
1584 
1586  Res0 = 1.0 / opr0;
1587  return true;
1588 
1589  // two-arg functions
1591  Res0 = opr0 / opr1;
1592  return true;
1593 
1594  case AMDGPULibFunc::EI_POW:
1596  Res0 = pow(opr0, opr1);
1597  return true;
1598 
1599  case AMDGPULibFunc::EI_POWN: {
1600  if (ConstantInt *iopr1 = dyn_cast_or_null<ConstantInt>(copr1)) {
1601  double val = (double)iopr1->getSExtValue();
1602  Res0 = pow(opr0, val);
1603  return true;
1604  }
1605  return false;
1606  }
1607 
1608  case AMDGPULibFunc::EI_ROOTN: {
1609  if (ConstantInt *iopr1 = dyn_cast_or_null<ConstantInt>(copr1)) {
1610  double val = (double)iopr1->getSExtValue();
1611  Res0 = pow(opr0, 1.0 / val);
1612  return true;
1613  }
1614  return false;
1615  }
1616 
1617  // with ptr arg
1619  Res0 = sin(opr0);
1620  Res1 = cos(opr0);
1621  return true;
1622 
1623  // three-arg functions
1624  case AMDGPULibFunc::EI_FMA:
1625  case AMDGPULibFunc::EI_MAD:
1626  Res0 = opr0 * opr1 + opr2;
1627  return true;
1628  }
1629 
1630  return false;
1631 }
1632 
1633 bool AMDGPULibCalls::evaluateCall(CallInst *aCI, FuncInfo &FInfo) {
1634  int numArgs = (int)aCI->getNumArgOperands();
1635  if (numArgs > 3)
1636  return false;
1637 
1638  Constant *copr0 = nullptr;
1639  Constant *copr1 = nullptr;
1640  Constant *copr2 = nullptr;
1641  if (numArgs > 0) {
1642  if ((copr0 = dyn_cast<Constant>(aCI->getArgOperand(0))) == nullptr)
1643  return false;
1644  }
1645 
1646  if (numArgs > 1) {
1647  if ((copr1 = dyn_cast<Constant>(aCI->getArgOperand(1))) == nullptr) {
1648  if (FInfo.getId() != AMDGPULibFunc::EI_SINCOS)
1649  return false;
1650  }
1651  }
1652 
1653  if (numArgs > 2) {
1654  if ((copr2 = dyn_cast<Constant>(aCI->getArgOperand(2))) == nullptr)
1655  return false;
1656  }
1657 
1658  // At this point, all arguments to aCI are constants.
1659 
1660  // max vector size is 16, and sincos will generate two results.
1661  double DVal0[16], DVal1[16];
1662  bool hasTwoResults = (FInfo.getId() == AMDGPULibFunc::EI_SINCOS);
1663  if (getVecSize(FInfo) == 1) {
1664  if (!evaluateScalarMathFunc(FInfo, DVal0[0],
1665  DVal1[0], copr0, copr1, copr2)) {
1666  return false;
1667  }
1668  } else {
1669  ConstantDataVector *CDV0 = dyn_cast_or_null<ConstantDataVector>(copr0);
1670  ConstantDataVector *CDV1 = dyn_cast_or_null<ConstantDataVector>(copr1);
1671  ConstantDataVector *CDV2 = dyn_cast_or_null<ConstantDataVector>(copr2);
1672  for (int i=0; i < getVecSize(FInfo); ++i) {
1673  Constant *celt0 = CDV0 ? CDV0->getElementAsConstant(i) : nullptr;
1674  Constant *celt1 = CDV1 ? CDV1->getElementAsConstant(i) : nullptr;
1675  Constant *celt2 = CDV2 ? CDV2->getElementAsConstant(i) : nullptr;
1676  if (!evaluateScalarMathFunc(FInfo, DVal0[i],
1677  DVal1[i], celt0, celt1, celt2)) {
1678  return false;
1679  }
1680  }
1681  }
1682 
1683  LLVMContext &context = CI->getParent()->getParent()->getContext();
1684  Constant *nval0, *nval1;
1685  if (getVecSize(FInfo) == 1) {
1686  nval0 = ConstantFP::get(CI->getType(), DVal0[0]);
1687  if (hasTwoResults)
1688  nval1 = ConstantFP::get(CI->getType(), DVal1[0]);
1689  } else {
1690  if (getArgType(FInfo) == AMDGPULibFunc::F32) {
1691  SmallVector <float, 0> FVal0, FVal1;
1692  for (int i=0; i < getVecSize(FInfo); ++i)
1693  FVal0.push_back((float)DVal0[i]);
1694  ArrayRef<float> tmp0(FVal0);
1695  nval0 = ConstantDataVector::get(context, tmp0);
1696  if (hasTwoResults) {
1697  for (int i=0; i < getVecSize(FInfo); ++i)
1698  FVal1.push_back((float)DVal1[i]);
1699  ArrayRef<float> tmp1(FVal1);
1700  nval1 = ConstantDataVector::get(context, tmp1);
1701  }
1702  } else {
1703  ArrayRef<double> tmp0(DVal0);
1704  nval0 = ConstantDataVector::get(context, tmp0);
1705  if (hasTwoResults) {
1706  ArrayRef<double> tmp1(DVal1);
1707  nval1 = ConstantDataVector::get(context, tmp1);
1708  }
1709  }
1710  }
1711 
1712  if (hasTwoResults) {
1713  // sincos
1714  assert(FInfo.getId() == AMDGPULibFunc::EI_SINCOS &&
1715  "math function with ptr arg not supported yet");
1716  new StoreInst(nval1, aCI->getArgOperand(1), aCI);
1717  }
1718 
1719  replaceCall(nval0);
1720  return true;
1721 }
1722 
1723 // Public interface to the Simplify LibCalls pass.
1725  const TargetMachine *TM) {
1726  return new AMDGPUSimplifyLibCalls(Opt, TM);
1727 }
1728 
1730  return new AMDGPUUseNativeCalls();
1731 }
1732 
1733 static bool setFastFlags(Function &F, const TargetOptions &Options) {
1734  AttrBuilder B;
1735 
1736  if (Options.UnsafeFPMath || Options.NoInfsFPMath)
1737  B.addAttribute("no-infs-fp-math", "true");
1738  if (Options.UnsafeFPMath || Options.NoNaNsFPMath)
1739  B.addAttribute("no-nans-fp-math", "true");
1740  if (Options.UnsafeFPMath) {
1741  B.addAttribute("less-precise-fpmad", "true");
1742  B.addAttribute("unsafe-fp-math", "true");
1743  }
1744 
1745  if (!B.hasAttributes())
1746  return false;
1747 
1749 
1750  return true;
1751 }
1752 
1754  if (skipFunction(F))
1755  return false;
1756 
1757  bool Changed = false;
1758  auto AA = &getAnalysis<AAResultsWrapperPass>().getAAResults();
1759 
1760  LLVM_DEBUG(dbgs() << "AMDIC: process function ";
1761  F.printAsOperand(dbgs(), false, F.getParent()); dbgs() << '\n';);
1762 
1763  if (!EnablePreLink)
1764  Changed |= setFastFlags(F, Options);
1765 
1766  for (auto &BB : F) {
1767  for (BasicBlock::iterator I = BB.begin(), E = BB.end(); I != E; ) {
1768  // Ignore non-calls.
1769  CallInst *CI = dyn_cast<CallInst>(I);
1770  ++I;
1771  if (!CI) continue;
1772 
1773  // Ignore indirect calls.
1774  Function *Callee = CI->getCalledFunction();
1775  if (Callee == 0) continue;
1776 
1777  LLVM_DEBUG(dbgs() << "AMDIC: try folding " << *CI << "\n";
1778  dbgs().flush());
1779  if(Simplifier.fold(CI, AA))
1780  Changed = true;
1781  }
1782  }
1783  return Changed;
1784 }
1785 
1787  if (skipFunction(F) || UseNative.empty())
1788  return false;
1789 
1790  bool Changed = false;
1791  for (auto &BB : F) {
1792  for (BasicBlock::iterator I = BB.begin(), E = BB.end(); I != E; ) {
1793  // Ignore non-calls.
1794  CallInst *CI = dyn_cast<CallInst>(I);
1795  ++I;
1796  if (!CI) continue;
1797 
1798  // Ignore indirect calls.
1799  Function *Callee = CI->getCalledFunction();
1800  if (Callee == 0) continue;
1801 
1802  if(Simplifier.useNative(CI))
1803  Changed = true;
1804  }
1805  }
1806  return Changed;
1807 }
A vector constant whose element type is a simple 1/2/4/8-byte integer or float/double, and whose elements are just simple data values (i.e.
Definition: Constants.h:761
SymbolTableList< Instruction >::iterator eraseFromParent()
This method unlinks &#39;this&#39; from the containing basic block and deletes it.
Definition: Instruction.cpp:67
static const TableEntry tbl_sqrt[]
static const TableEntry tbl_log10[]
raw_ostream & errs()
This returns a reference to a raw_ostream for standard error.
static PassRegistry * getPassRegistry()
getPassRegistry - Access the global registry object, which is automatically initialized at applicatio...
LLVMContext & Context
AllocaInst * CreateAlloca(Type *Ty, unsigned AddrSpace, Value *ArraySize=nullptr, const Twine &Name="")
Definition: IRBuilder.h:1563
AMDGPU specific subclass of TargetSubtarget.
void dropAllReferences()
Drop all references to operands.
Definition: User.h:294
EFuncId getId() const
Value * CreateAddrSpaceCast(Value *V, Type *DestTy, const Twine &Name="")
Definition: IRBuilder.h:1969
This class represents lattice values for constants.
Definition: AllocatorList.h:23
Type * getParamType(unsigned i) const
Parameter type accessors.
Definition: DerivedTypes.h:135
Constant * getElementAsConstant(unsigned i) const
Return a Constant for a specified index&#39;s element.
Definition: Constants.cpp:2773
LoadInst * CreateLoad(Type *Ty, Value *Ptr, const char *Name)
Provided to resolve &#39;CreateLoad(Ty, Ptr, "...")&#39; correctly, instead of converting the string to &#39;bool...
Definition: IRBuilder.h:1576
LLVM_NODISCARD bool equals_lower(StringRef RHS) const
equals_lower - Check for string equality, ignoring case.
Definition: StringRef.h:181
A Module instance is used to store all the information related to an LLVM module. ...
Definition: Module.h:65
void setAlignment(unsigned Align)
amdgpu Simplify well known AMD library false FunctionCallee Value const Twine & Name
static const TableEntry tbl_cospi[]
amdgpu simplifylib
static CallInst * Create(FunctionType *Ty, Value *F, const Twine &NameStr="", Instruction *InsertBefore=nullptr)
A handy container for a FunctionType+Callee-pointer pair, which can be passed around as a single enti...
Definition: DerivedTypes.h:165
static const TableEntry tbl_cbrt[]
static bool parse(StringRef MangledName, AMDGPULibFunc &Ptr)
void initializeAMDGPUSimplifyLibCallsPass(PassRegistry &)
This class represents a function call, abstracting a target machine&#39;s calling convention.
static PointerType * get(Type *ElementType, unsigned AddressSpace)
This constructs a pointer to an object of the specified type in a numbered address space...
Definition: Type.cpp:632
bool isUnsafeMath(const CallInst *CI) const
float convertToFloat() const
Definition: APFloat.h:1113
F(f)
unsigned getPointerAddressSpace() const
Get the address space of this pointer or pointer vector type.
Definition: DerivedTypes.h:580
An instruction for reading from memory.
Definition: Instructions.h:167
static const TableEntry tbl_erf[]
static IntegerType * getInt64Ty(LLVMContext &C)
Definition: Type.cpp:176
AttrBuilder & addAttribute(Attribute::AttrKind Val)
Add an attribute to the builder.
void setPrefix(ENamePrefix PFX)
LLVMContext & getContext() const
Get the context in which this basic block lives.
Definition: BasicBlock.cpp:32
static const TableEntry tbl_acosh[]
static FunctionCallee getOrInsertFunction(llvm::Module *M, const AMDGPULibFunc &fInfo)
Address space for private memory.
Definition: AMDGPU.h:275
iterator begin()
Instruction iterator methods.
Definition: BasicBlock.h:268
#define DEBUG_WITH_TYPE(TYPE, X)
DEBUG_WITH_TYPE macro - This macro should be used by passes to emit debug information.
Definition: Debug.h:64
IntegerType * getInt32Ty()
Fetch the type representing a 32-bit integer.
Definition: IRBuilder.h:383
Value * getArgOperand(unsigned i) const
Definition: InstrTypes.h:1241
static const TableEntry tbl_atanh[]
std::string getName() const
Get unmangled name for mangled library function and name for unmangled library function.
AnalysisUsage & addRequired()
#define INITIALIZE_PASS_DEPENDENCY(depName)
Definition: PassSupport.h:50
bool hasAttributes() const
Return true if the builder has IR-level attributes.
const DataLayout & getDataLayout() const
Get the data layout for the module&#39;s target platform.
Definition: Module.cpp:369
Twine - A lightweight data structure for efficiently representing the concatenation of temporary valu...
Definition: Twine.h:80
IntegerType * getInt64Ty()
Fetch the type representing a 64-bit integer.
Definition: IRBuilder.h:388
Value * FindAvailableLoadedValue(LoadInst *Load, BasicBlock *ScanBB, BasicBlock::iterator &ScanFrom, unsigned MaxInstsToScan=DefMaxInstsToScan, AliasAnalysis *AA=nullptr, bool *IsLoadCSE=nullptr, unsigned *NumScanedInst=nullptr)
Scan backwards to see if we have the value of the given load available locally within a small number ...
Definition: Loads.cpp:361
LLVMContext & getContext() const
Get the global data context.
Definition: Module.h:244
bool isIntegerTy() const
True if this is an instance of IntegerType.
Definition: Type.h:196
This provides a uniform API for creating instructions and inserting them into a basic block: either a...
Definition: IRBuilder.h:779
Param * getLeads()
Get leading parameters for mangled lib functions.
static const TableEntry tbl_sinh[]
LLVM_NODISCARD bool empty() const
empty - Check if the string is empty.
Definition: StringRef.h:140
unsigned getNumArgs() const
INITIALIZE_PASS_BEGIN(AMDGPUSimplifyLibCalls, "amdgpu-simplifylib", "Simplify well-known AMD library calls", false, false) INITIALIZE_PASS_END(AMDGPUSimplifyLibCalls
All zero aggregate value.
Definition: Constants.h:340
void setId(EFuncId Id)
Value * CreateBitCast(Value *V, Type *DestTy, const Twine &Name="")
Definition: IRBuilder.h:1964
unsigned NoNaNsFPMath
NoNaNsFPMath - This flag is enabled when the -enable-no-nans-fp-math flag is specified on the command...
Type * getType() const
All values are typed, get the type of this value.
Definition: Value.h:245
amdgpu Simplify well known AMD library false
static const TableEntry tbl_sinpi[]
void setCalledFunction(Function *Fn)
Sets the function called, including updating the function type.
Definition: InstrTypes.h:1323
bool isNegative() const
Return true if the sign bit is set.
Definition: Constants.h:308
static const TableEntry tbl_asinh[]
static const TableEntry tbl_tanpi[]
ArrayRef - Represent a constant reference to an array (0 or more elements consecutively in memory)...
Definition: APInt.h:32
Value * CreateSIToFP(Value *V, Type *DestTy, const Twine &Name="")
Definition: IRBuilder.h:1934
An instruction for storing to memory.
Definition: Instructions.h:320
void replaceAllUsesWith(Value *V)
Change all uses of this to point to a new Value.
Definition: Value.cpp:429
Value * CreateZExtOrBitCast(Value *V, Type *DestTy, const Twine &Name="")
Definition: IRBuilder.h:1974
void SetInsertPoint(BasicBlock *TheBB)
This specifies that created instructions should be appended to the end of the specified block...
Definition: IRBuilder.h:132
static const TableEntry tbl_log2[]
Value * CreateOr(Value *LHS, Value *RHS, const Twine &Name="")
Definition: IRBuilder.h:1294
static AMDGPULibFunc::EType getArgType(const AMDGPULibFunc &FInfo)
const BasicBlock & getEntryBlock() const
Definition: Function.h:664
static bool runOnFunction(Function &F, bool PostInlining)
#define P(N)
initializer< Ty > init(const Ty &Val)
Definition: CommandLine.h:432
TableRef(const TableEntry(&tbl)[N])
Type * getReturnType() const
Returns the type of the ret val.
Definition: Function.h:168
bool fold(CallInst *CI, AliasAnalysis *AA=nullptr)
unsigned UnsafeFPMath
UnsafeFPMath - This flag is enabled when the -enable-unsafe-fp-math flag is specified on the command ...
static GCRegistry::Add< OcamlGC > B("ocaml", "ocaml 3.10-compatible GC")
Value * CreateFMul(Value *L, Value *R, const Twine &Name="", MDNode *FPMD=nullptr)
Definition: IRBuilder.h:1383
static const TableEntry tbl_exp2[]
constexpr bool isPowerOf2_32(uint32_t Value)
Return true if the argument is a power of two > 0.
Definition: MathExtras.h:428
LLVM Basic Block Representation.
Definition: BasicBlock.h:57
static Function * getFunction(llvm::Module *M, const AMDGPULibFunc &fInfo)
Constant * getSplatValue() const
If this is a splat constant, meaning that all of the elements have the same value, return that value.
Definition: Constants.cpp:2810
The instances of the Type class are immutable: once they are created, they are never changed...
Definition: Type.h:45
This is an important class for using LLVM in a threaded context.
Definition: LLVMContext.h:64
static GCRegistry::Add< CoreCLRGC > E("coreclr", "CoreCLR-compatible GC")
This is an important base class in LLVM.
Definition: Constant.h:41
This file contains the declarations for the subclasses of Constant, which represent the different fla...
ConstantFP - Floating Point Values [float, double].
Definition: Constants.h:263
static cl::opt< bool > EnablePreLink("amdgpu-prelink", cl::desc("Enable pre-link mode optimizations"), cl::init(false), cl::Hidden)
double convertToDouble() const
Definition: APFloat.h:1112
float getElementAsFloat(unsigned i) const
If this is an sequential container of floats, return the specified element as a float.
Definition: Constants.cpp:2761
static const TableEntry tbl_atan[]
static const TableEntry tbl_sin[]
bool isMangled() const
Represent the analysis usage information of a pass.
static const TableEntry tbl_rsqrt[]
static const TableEntry tbl_acospi[]
void setCallingConv(CallingConv::ID CC)
Definition: InstrTypes.h:1348
FunctionPass class - This class is used to implement most global optimizations.
Definition: Pass.h:284
amdgpu Simplify well known AMD library false FunctionCallee Value * Arg
static FunctionType * get(Type *Result, ArrayRef< Type *> Params, bool isVarArg)
This static method is the primary way of constructing a FunctionType.
Definition: Type.cpp:296
Wrapper class for AMDGPULIbFuncImpl.
static const TableEntry tbl_expm1[]
self_iterator getIterator()
Definition: ilist_node.h:81
static int getVecSize(const AMDGPULibFunc &FInfo)
static Constant * getSplat(unsigned NumElts, Constant *Elt)
Return a ConstantVector with the specified constant in each element.
Definition: Constants.cpp:2650
unsigned NoInfsFPMath
NoInfsFPMath - This flag is enabled when the -enable-no-infs-fp-math flag is specified on the command...
static double log2(double V)
LLVMContext & getContext() const
getContext - Return a reference to the LLVMContext associated with this function. ...
Definition: Function.cpp:205
static unsigned getEPtrKindFromAddrSpace(unsigned AS)
size_t size() const
Definition: SmallVector.h:52
void initializeAMDGPUUseNativeCallsPass(PassRegistry &)
auto find(R &&Range, const T &Val) -> decltype(adl_begin(Range))
Provide wrappers to std::find which take ranges instead of having to pass begin/end explicitly...
Definition: STLExtras.h:1186
void printAsOperand(raw_ostream &O, bool PrintType=true, const Module *M=nullptr) const
Print the name of this Value out to the specified raw_ostream.
Definition: AsmWriter.cpp:4355
INITIALIZE_PASS_END(RegBankSelect, DEBUG_TYPE, "Assign register bank of generic virtual registers", false, false) RegBankSelect
This struct is a compact representation of a valid (non-zero power of two) alignment.
Definition: Alignment.h:40
static const TableEntry tbl_atanpi[]
Type * getAllocatedType() const
Return the type that is being allocated by the instruction.
Definition: Instructions.h:105
const APFloat & getValueAPF() const
Definition: Constants.h:302
unsigned getWavefrontSize() const
static const TableEntry tbl_exp10[]
AMDGPULibCalls(const TargetMachine *TM_=nullptr)
Iterator for intrusive lists based on ilist_node.
TargetPassConfig.
This is the shared class of boolean and integer constants.
Definition: Constants.h:83
static const TableEntry tbl_asinpi[]
static bool setFastFlags(Function &F, const TargetOptions &Options)
unsigned getScalarSizeInBits() const LLVM_READONLY
If this is a vector type, return the getPrimitiveSizeInBits value for the element type...
Definition: Type.cpp:129
ENamePrefix getPrefix() const
This is a &#39;vector&#39; (really, a variable-sized array), optimized for the case when the array is small...
Definition: SmallVector.h:837
Utility class for floating point operations which can have information about relaxed accuracy require...
Definition: Operator.h:245
Module.h This file contains the declarations for the Module class.
LLVM_NODISCARD size_t find_lower(char C, size_t From=0) const
Search for the first character C in the string, ignoring case.
Definition: StringRef.cpp:57
static IntegerType * getIntNTy(LLVMContext &C, unsigned N)
Definition: Type.cpp:179
static Constant * get(Type *Ty, uint64_t V, bool isSigned=false)
If Ty is a vector type, return a Constant with a splat of the given value.
Definition: Constants.cpp:653
static Constant * get(Type *Ty, double V)
This returns a ConstantFP, or a vector containing a splat of a ConstantFP, for the specified value in...
Definition: Constants.cpp:716
const TableEntry * table
AttributeList getAttributes() const
Return the parameter attributes for this call.
Definition: InstrTypes.h:1366
static const TableEntry tbl_log[]
Intrinsic::ID getIntrinsicID() const LLVM_READONLY
getIntrinsicID - This method returns the ID number of the specified function, or Intrinsic::not_intri...
Definition: Function.h:193
raw_ostream & dbgs()
dbgs() - This returns a reference to a raw_ostream for debugging messages.
Definition: Debug.cpp:132
Class to represent vector types.
Definition: DerivedTypes.h:427
const Module * getModule() const
Return the module owning the function this instruction belongs to or nullptr it the function does not...
Definition: Instruction.cpp:55
static const TableEntry tbl_cosh[]
amdgpu Simplify well known AMD library false FunctionCallee Callee
iterator_range< user_iterator > users()
Definition: Value.h:419
InstListType::iterator iterator
Instruction iterators...
Definition: BasicBlock.h:89
Value * CreateShl(Value *LHS, Value *RHS, const Twine &Name="", bool HasNUW=false, bool HasNSW=false)
Definition: IRBuilder.h:1207
Value * CreatePointerCast(Value *V, Type *DestTy, const Twine &Name="")
Definition: IRBuilder.h:2010
LLVM_NODISCARD bool equals(StringRef RHS) const
equals - Check for string equality, this is more efficient than compare() when the relative ordering ...
Definition: StringRef.h:174
uint64_t getTypeAllocSize(Type *Ty) const
Returns the offset in bytes between successive objects of the specified type, including alignment pad...
Definition: DataLayout.h:470
unsigned getNumUses() const
This method computes the number of uses of this Value.
Definition: Value.cpp:160
unsigned getNumArgOperands() const
Definition: InstrTypes.h:1239
static const TableEntry tbl_exp[]
static const size_t npos
Definition: StringRef.h:50
StringRef getValueAsString() const
Return the attribute&#39;s value as a string.
Definition: Attributes.cpp:223
static const TableEntry tbl_tanh[]
bool isZero() const
Return true if the value is positive or negative zero.
Definition: Constants.h:305
static VectorType * get(Type *ElementType, ElementCount EC)
This static method is the primary way to construct an VectorType.
Definition: Type.cpp:609
double getElementAsDouble(unsigned i) const
If this is an sequential container of doubles, return the specified element as a double.
Definition: Constants.cpp:2767
StringRef getName() const
Return a constant reference to the value&#39;s name.
Definition: Value.cpp:214
Function * getCalledFunction() const
Returns the function called, or null if this is an indirect function invocation.
Definition: InstrTypes.h:1287
const Function * getParent() const
Return the enclosing method, or null if none.
Definition: BasicBlock.h:106
#define I(x, y, z)
Definition: MD5.cpp:58
static TableRef getOptTable(AMDGPULibFunc::EFuncId id)
#define N
#define MATH_PI
APFloat abs(APFloat X)
Returns the absolute value of the argument.
Definition: APFloat.h:1228
static const TableEntry tbl_asin[]
Value * CreateFPToSI(Value *V, Type *DestTy, const Twine &Name="")
Definition: IRBuilder.h:1923
void setAttributes(AttributeList A)
Set the parameter attributes for this call.
Definition: InstrTypes.h:1370
FunctionType * getFunctionType()
Definition: DerivedTypes.h:182
static cl::list< std::string > UseNative("amdgpu-use-native", cl::desc("Comma separated list of functions to replace with native, or all"), cl::CommaSeparated, cl::ValueOptional, cl::Hidden)
LLVM_NODISCARD std::enable_if<!is_simple_type< Y >::value, typename cast_retty< X, const Y >::ret_type >::type dyn_cast(const Y &Val)
Definition: Casting.h:332
uint32_t Size
Definition: Profile.cpp:46
CallInst * CreateCall(FunctionType *FTy, Value *Callee, ArrayRef< Value *> Args=None, const Twine &Name="", MDNode *FPMathTag=nullptr)
Definition: IRBuilder.h:2237
#define MATH_SQRT2
static const TableEntry tbl_tan[]
static const TableEntry tbl_cos[]
Value * CreateFAdd(Value *L, Value *R, const Twine &Name="", MDNode *FPMD=nullptr)
Definition: IRBuilder.h:1333
Address space for flat memory.
Definition: AMDGPU.h:269
Value * CreateAnd(Value *LHS, Value *RHS, const Twine &Name="")
Definition: IRBuilder.h:1268
static bool HasNative(AMDGPULibFunc::EFuncId id)
Value * CreateFDiv(Value *L, Value *R, const Twine &Name="", MDNode *FPMD=nullptr)
Definition: IRBuilder.h:1408
const std::string to_string(const T &Value)
Definition: ScopedPrinter.h:61
void replaceCall(Value *With)
void addAttributes(unsigned i, const AttrBuilder &Attrs)
adds the attributes to the list of attributes.
Definition: Function.cpp:392
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
FunctionPass * createAMDGPUSimplifyLibCallsPass(const TargetOptions &, const TargetMachine *)
FunctionPass * createAMDGPUUseNativeCallsPass()
aarch64 promote const
Module * getParent()
Get the module that this global value is contained inside of...
Definition: GlobalValue.h:575
LLVM Value Representation.
Definition: Value.h:73
amdgpu Simplify well known AMD library calls
static const TableEntry tbl_erfc[]
void setFastMathFlags(FastMathFlags NewFMF)
Set the fast-math flags to be used with generated fp-math operators.
Definition: IRBuilder.h:225
BasicBlock::iterator GetInsertPoint() const
Definition: IRBuilder.h:127
Attribute getFnAttribute(Attribute::AttrKind Kind) const
Return the attribute for the given attribute kind.
Definition: Function.h:333
std::string mangle() const
Primary interface to the complete machine description for the target machine.
Definition: TargetMachine.h:65
Type * getElementType() const
Definition: DerivedTypes.h:394
bool isExactlyValue(const APFloat &V) const
We don&#39;t rely on operator== working on double values, as it returns true for things that are clearly ...
Definition: Constants.cpp:855
StringRef - Represent a constant reference to a string, i.e.
Definition: StringRef.h:48
unsigned getNumElements() const
Return the number of elements in the array or vector.
Definition: Constants.cpp:2456
A wrapper pass to provide the legacy pass manager access to a suitably prepared AAResults object...
int64_t getSExtValue() const
Return the constant as a 64-bit integer value after it has been sign extended as appropriate for the ...
Definition: Constants.h:156
static Constant * get(LLVMContext &Context, ArrayRef< uint8_t > Elts)
get() constructors - Return a constant with vector type with an element count and element type matchi...
Definition: Constants.cpp:2595
#define LLVM_DEBUG(X)
Definition: Debug.h:122
bool isDoubleTy() const
Return true if this is &#39;double&#39;, a 64-bit IEEE fp type.
Definition: Type.h:149
static const TableEntry tbl_tgamma[]
constexpr char Args[]
Key for Kernel::Metadata::mArgs.
bool useNative(CallInst *CI)
static const TableEntry tbl_acos[]
const BasicBlock * getParent() const
Definition: Instruction.h:66
an instruction to allocate memory on the stack
Definition: Instructions.h:59
#define MATH_E